├── .gitignore ├── LICENSE ├── README.md ├── example └── main.odin ├── mimalloc ├── build_windows.cmd ├── c │ ├── .gitattributes │ ├── .gitignore │ ├── LICENSE │ ├── include │ │ ├── mimalloc-new-delete.h │ │ ├── mimalloc-override.h │ │ ├── mimalloc.h │ │ └── mimalloc │ │ │ ├── atomic.h │ │ │ ├── internal.h │ │ │ ├── prim.h │ │ │ ├── track.h │ │ │ └── types.h │ ├── readme.md │ └── src │ │ ├── alloc-aligned.c │ │ ├── alloc-override.c │ │ ├── alloc-posix.c │ │ ├── alloc.c │ │ ├── arena.c │ │ ├── bitmap.c │ │ ├── bitmap.h │ │ ├── heap.c │ │ ├── init.c │ │ ├── options.c │ │ ├── os.c │ │ ├── page-queue.c │ │ ├── page.c │ │ ├── prim │ │ ├── osx │ │ │ ├── alloc-override-zone.c │ │ │ └── prim.c │ │ ├── prim.c │ │ ├── readme.md │ │ ├── unix │ │ │ └── prim.c │ │ ├── wasi │ │ │ └── prim.c │ │ └── windows │ │ │ ├── etw-mimalloc.wprp │ │ │ ├── etw.h │ │ │ ├── etw.man │ │ │ ├── prim.c │ │ │ └── readme.md │ │ ├── random.c │ │ ├── segment-map.c │ │ ├── segment.c │ │ ├── static.c │ │ └── stats.c ├── mimalloc.odin └── wrapper.odin ├── odinfmt.json └── ols.json /.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | *.lib 3 | *.pdb 4 | *.ilk 5 | *.exp -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Jakub Tomšů 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # odin-mimalloc 2 | Odin bindngs for [mimalloc](https://github.com/microsoft/mimalloc) 3 | 4 | mimalloc (pronounced "me-malloc") is a general purpose allocator with excellent performance characteristics. 5 | 6 | WIP - only builds on windows and not all functions were tested. 7 | 8 | # Usage 9 | [Mimalloc API documentation](https://microsoft.github.io/mimalloc/) 10 | 11 | Simply copy the `mimalloc` folder somewhere into your project. Then you can just import it: 12 | ```odin 13 | import mi "mimalloc" 14 | ``` 15 | Now you can use all the mimalloc functions. No global init required. 16 | 17 | How to override the default `context.allocator`: 18 | ```odin 19 | context.allocator = mi.global_allocator() 20 | ``` 21 | 22 | Or with a custom heap: 23 | ```odin 24 | heap := mi.heap_new() 25 | context.allocator = mi.heap_allocator(heap) 26 | // ... 27 | mi.heap_destroy(heap) 28 | // Here you might want to restore the context.allocator 29 | ``` 30 | 31 | -------------------------------------------------------------------------------- /example/main.odin: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "core:fmt" 4 | import mi "../mimalloc" 5 | 6 | main :: proc() { 7 | fmt.println("Hello") 8 | 9 | a := mi.malloc(16) 10 | fmt.println(a) 11 | a = mi.realloc(a, 1000) 12 | fmt.println(a) 13 | mi.free(a) 14 | 15 | context.allocator = mi.global_allocator() 16 | b: [dynamic]int 17 | 18 | for i in 0 ..< 20 do append(&b, i) 19 | fmt.println(b) 20 | 21 | delete(b) 22 | 23 | { 24 | heap := mi.heap_new() 25 | defer mi.heap_destroy(heap) 26 | 27 | context.allocator = mi.heap_allocator(heap) 28 | 29 | c: [dynamic]f32 30 | for i in 0 ..< 100 do append(&c, 1.0 / f32(i)) 31 | fmt.println(c) 32 | } 33 | 34 | // mi.stats_print() 35 | mi.option_enable(.show_stats) 36 | } 37 | -------------------------------------------------------------------------------- /mimalloc/build_windows.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | cl /c c\src\static.c /I c\include /Fomimalloc.obj /D_DEBUG /DEBUG 4 | lib /OUT:mimalloc_windows_x64_debug.lib mimalloc.obj 5 | 6 | cl /c c\src\static.c /I c\include /Fomimalloc.obj /O2 7 | lib /OUT:mimalloc_windows_x64_release.lib mimalloc.obj 8 | 9 | del mimalloc.obj -------------------------------------------------------------------------------- /mimalloc/c/.gitattributes: -------------------------------------------------------------------------------- 1 | # default behavior is to always use unix style line endings 2 | * text eol=lf 3 | *.png binary 4 | *.pdn binary 5 | *.jpg binary 6 | *.sln binary 7 | *.suo binary 8 | *.vcproj binary 9 | *.patch binary 10 | *.dll binary 11 | *.lib binary 12 | *.exe binary 13 | -------------------------------------------------------------------------------- /mimalloc/c/.gitignore: -------------------------------------------------------------------------------- 1 | ide/vs20??/*.db 2 | ide/vs20??/*.opendb 3 | ide/vs20??/*.user 4 | ide/vs20??/*.vcxproj.filters 5 | ide/vs20??/.vs 6 | ide/vs20??/VTune* 7 | out/ 8 | docs/ 9 | *.zip 10 | -------------------------------------------------------------------------------- /mimalloc/c/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /mimalloc/c/include/mimalloc-new-delete.h: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2020 Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #pragma once 8 | #ifndef MIMALLOC_NEW_DELETE_H 9 | #define MIMALLOC_NEW_DELETE_H 10 | 11 | // ---------------------------------------------------------------------------- 12 | // This header provides convenient overrides for the new and 13 | // delete operations in C++. 14 | // 15 | // This header should be included in only one source file! 16 | // 17 | // On Windows, or when linking dynamically with mimalloc, these 18 | // can be more performant than the standard new-delete operations. 19 | // See 20 | // --------------------------------------------------------------------------- 21 | #if defined(__cplusplus) 22 | #include 23 | #include 24 | 25 | #if defined(_MSC_VER) && defined(_Ret_notnull_) && defined(_Post_writable_byte_size_) 26 | // stay consistent with VCRT definitions 27 | #define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict _Ret_notnull_ _Post_writable_byte_size_(n) 28 | #define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict _Ret_maybenull_ _Success_(return != NULL) _Post_writable_byte_size_(n) 29 | #else 30 | #define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict 31 | #define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict 32 | #endif 33 | 34 | void operator delete(void* p) noexcept { mi_free(p); }; 35 | void operator delete[](void* p) noexcept { mi_free(p); }; 36 | 37 | void operator delete (void* p, const std::nothrow_t&) noexcept { mi_free(p); } 38 | void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); } 39 | 40 | mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); } 41 | mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); } 42 | 43 | mi_decl_new_nothrow(n) void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); } 44 | mi_decl_new_nothrow(n) void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); } 45 | 46 | #if (__cplusplus >= 201402L || _MSC_VER >= 1916) 47 | void operator delete (void* p, std::size_t n) noexcept { mi_free_size(p,n); }; 48 | void operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); }; 49 | #endif 50 | 51 | #if (__cplusplus > 201402L || defined(__cpp_aligned_new)) 52 | void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } 53 | void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } 54 | void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; 55 | void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; 56 | void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); } 57 | void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); } 58 | 59 | void* operator new (std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } 60 | void* operator new[](std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } 61 | void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } 62 | void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } 63 | #endif 64 | #endif 65 | 66 | #endif // MIMALLOC_NEW_DELETE_H 67 | -------------------------------------------------------------------------------- /mimalloc/c/include/mimalloc-override.h: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2020 Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #pragma once 8 | #ifndef MIMALLOC_OVERRIDE_H 9 | #define MIMALLOC_OVERRIDE_H 10 | 11 | /* ---------------------------------------------------------------------------- 12 | This header can be used to statically redirect malloc/free and new/delete 13 | to the mimalloc variants. This can be useful if one can include this file on 14 | each source file in a project (but be careful when using external code to 15 | not accidentally mix pointers from different allocators). 16 | -----------------------------------------------------------------------------*/ 17 | 18 | #include 19 | 20 | // Standard C allocation 21 | #define malloc(n) mi_malloc(n) 22 | #define calloc(n,c) mi_calloc(n,c) 23 | #define realloc(p,n) mi_realloc(p,n) 24 | #define free(p) mi_free(p) 25 | 26 | #define strdup(s) mi_strdup(s) 27 | #define strndup(s,n) mi_strndup(s,n) 28 | #define realpath(f,n) mi_realpath(f,n) 29 | 30 | // Microsoft extensions 31 | #define _expand(p,n) mi_expand(p,n) 32 | #define _msize(p) mi_usable_size(p) 33 | #define _recalloc(p,n,c) mi_recalloc(p,n,c) 34 | 35 | #define _strdup(s) mi_strdup(s) 36 | #define _strndup(s,n) mi_strndup(s,n) 37 | #define _wcsdup(s) (wchar_t*)mi_wcsdup((const unsigned short*)(s)) 38 | #define _mbsdup(s) mi_mbsdup(s) 39 | #define _dupenv_s(b,n,v) mi_dupenv_s(b,n,v) 40 | #define _wdupenv_s(b,n,v) mi_wdupenv_s((unsigned short*)(b),n,(const unsigned short*)(v)) 41 | 42 | // Various Posix and Unix variants 43 | #define reallocf(p,n) mi_reallocf(p,n) 44 | #define malloc_size(p) mi_usable_size(p) 45 | #define malloc_usable_size(p) mi_usable_size(p) 46 | #define cfree(p) mi_free(p) 47 | 48 | #define valloc(n) mi_valloc(n) 49 | #define pvalloc(n) mi_pvalloc(n) 50 | #define reallocarray(p,s,n) mi_reallocarray(p,s,n) 51 | #define reallocarr(p,s,n) mi_reallocarr(p,s,n) 52 | #define memalign(a,n) mi_memalign(a,n) 53 | #define aligned_alloc(a,n) mi_aligned_alloc(a,n) 54 | #define posix_memalign(p,a,n) mi_posix_memalign(p,a,n) 55 | #define _posix_memalign(p,a,n) mi_posix_memalign(p,a,n) 56 | 57 | // Microsoft aligned variants 58 | #define _aligned_malloc(n,a) mi_malloc_aligned(n,a) 59 | #define _aligned_realloc(p,n,a) mi_realloc_aligned(p,n,a) 60 | #define _aligned_recalloc(p,s,n,a) mi_aligned_recalloc(p,s,n,a) 61 | #define _aligned_msize(p,a,o) mi_usable_size(p) 62 | #define _aligned_free(p) mi_free(p) 63 | #define _aligned_offset_malloc(n,a,o) mi_malloc_aligned_at(n,a,o) 64 | #define _aligned_offset_realloc(p,n,a,o) mi_realloc_aligned_at(p,n,a,o) 65 | #define _aligned_offset_recalloc(p,s,n,a,o) mi_recalloc_aligned_at(p,s,n,a,o) 66 | 67 | #endif // MIMALLOC_OVERRIDE_H 68 | -------------------------------------------------------------------------------- /mimalloc/c/include/mimalloc/atomic.h: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2023 Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #pragma once 8 | #ifndef MIMALLOC_ATOMIC_H 9 | #define MIMALLOC_ATOMIC_H 10 | 11 | // -------------------------------------------------------------------------------------------- 12 | // Atomics 13 | // We need to be portable between C, C++, and MSVC. 14 | // We base the primitives on the C/C++ atomics and create a mimimal wrapper for MSVC in C compilation mode. 15 | // This is why we try to use only `uintptr_t` and `*` as atomic types. 16 | // To gain better insight in the range of used atomics, we use explicitly named memory order operations 17 | // instead of passing the memory order as a parameter. 18 | // ----------------------------------------------------------------------------------------------- 19 | 20 | #if defined(__cplusplus) 21 | // Use C++ atomics 22 | #include 23 | #define _Atomic(tp) std::atomic 24 | #define mi_atomic(name) std::atomic_##name 25 | #define mi_memory_order(name) std::memory_order_##name 26 | #if !defined(ATOMIC_VAR_INIT) || (__cplusplus >= 202002L) // c++20, see issue #571 27 | #define MI_ATOMIC_VAR_INIT(x) x 28 | #else 29 | #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) 30 | #endif 31 | #elif defined(_MSC_VER) 32 | // Use MSVC C wrapper for C11 atomics 33 | #define _Atomic(tp) tp 34 | #define MI_ATOMIC_VAR_INIT(x) x 35 | #define mi_atomic(name) mi_atomic_##name 36 | #define mi_memory_order(name) mi_memory_order_##name 37 | #else 38 | // Use C11 atomics 39 | #include 40 | #define mi_atomic(name) atomic_##name 41 | #define mi_memory_order(name) memory_order_##name 42 | #if !defined(ATOMIC_VAR_INIT) || (__STDC_VERSION__ >= 201710L) // c17, see issue #735 43 | #define MI_ATOMIC_VAR_INIT(x) x 44 | #else 45 | #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) 46 | #endif 47 | #endif 48 | 49 | // Various defines for all used memory orders in mimalloc 50 | #define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail) \ 51 | mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail) 52 | 53 | #define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail) \ 54 | mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail) 55 | 56 | #define mi_atomic_load_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire)) 57 | #define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed)) 58 | #define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) 59 | #define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) 60 | #define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release)) 61 | #define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel)) 62 | #define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) 63 | #define mi_atomic_cas_weak_acq_rel(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire)) 64 | #define mi_atomic_cas_strong_release(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) 65 | #define mi_atomic_cas_strong_acq_rel(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire)) 66 | 67 | #define mi_atomic_add_relaxed(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed)) 68 | #define mi_atomic_sub_relaxed(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed)) 69 | #define mi_atomic_add_acq_rel(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel)) 70 | #define mi_atomic_sub_acq_rel(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel)) 71 | #define mi_atomic_and_acq_rel(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel)) 72 | #define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel)) 73 | 74 | #define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(uintptr_t)1) 75 | #define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(uintptr_t)1) 76 | #define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1) 77 | #define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1) 78 | 79 | static inline void mi_atomic_yield(void); 80 | static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add); 81 | static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub); 82 | 83 | 84 | #if defined(__cplusplus) || !defined(_MSC_VER) 85 | 86 | // In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value) 87 | // We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well 88 | #define mi_atomic_load_ptr_acquire(tp,p) mi_atomic_load_acquire(p) 89 | #define mi_atomic_load_ptr_relaxed(tp,p) mi_atomic_load_relaxed(p) 90 | 91 | // In C++ we need to add casts to help resolve templates if NULL is passed 92 | #if defined(__cplusplus) 93 | #define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,(tp*)x) 94 | #define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,(tp*)x) 95 | #define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des) 96 | #define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des) 97 | #define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des) 98 | #define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x) 99 | #define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x) 100 | #else 101 | #define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,x) 102 | #define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,x) 103 | #define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des) 104 | #define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des) 105 | #define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des) 106 | #define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x) 107 | #define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x) 108 | #endif 109 | 110 | // These are used by the statistics 111 | static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) { 112 | return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed)); 113 | } 114 | static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { 115 | int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p); 116 | while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, ¤t, x)) { /* nothing */ }; 117 | } 118 | 119 | // Used by timers 120 | #define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire)) 121 | #define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed)) 122 | #define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) 123 | #define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) 124 | 125 | #define mi_atomic_casi64_strong_acq_rel(p,e,d) mi_atomic_cas_strong_acq_rel(p,e,d) 126 | #define mi_atomic_addi64_acq_rel(p,i) mi_atomic_add_acq_rel(p,i) 127 | 128 | 129 | #elif defined(_MSC_VER) 130 | 131 | // MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics. 132 | #define WIN32_LEAN_AND_MEAN 133 | #include 134 | #include 135 | #ifdef _WIN64 136 | typedef LONG64 msc_intptr_t; 137 | #define MI_64(f) f##64 138 | #else 139 | typedef LONG msc_intptr_t; 140 | #define MI_64(f) f 141 | #endif 142 | 143 | typedef enum mi_memory_order_e { 144 | mi_memory_order_relaxed, 145 | mi_memory_order_consume, 146 | mi_memory_order_acquire, 147 | mi_memory_order_release, 148 | mi_memory_order_acq_rel, 149 | mi_memory_order_seq_cst 150 | } mi_memory_order; 151 | 152 | static inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) { 153 | (void)(mo); 154 | return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add); 155 | } 156 | static inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) { 157 | (void)(mo); 158 | return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub)); 159 | } 160 | static inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { 161 | (void)(mo); 162 | return (uintptr_t)MI_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x); 163 | } 164 | static inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { 165 | (void)(mo); 166 | return (uintptr_t)MI_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x); 167 | } 168 | static inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) { 169 | (void)(mo1); (void)(mo2); 170 | uintptr_t read = (uintptr_t)MI_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected)); 171 | if (read == *expected) { 172 | return true; 173 | } 174 | else { 175 | *expected = read; 176 | return false; 177 | } 178 | } 179 | static inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) { 180 | return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2); 181 | } 182 | static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) { 183 | (void)(mo); 184 | return (uintptr_t)MI_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange); 185 | } 186 | static inline void mi_atomic_thread_fence(mi_memory_order mo) { 187 | (void)(mo); 188 | _Atomic(uintptr_t) x = 0; 189 | mi_atomic_exchange_explicit(&x, 1, mo); 190 | } 191 | static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) { 192 | (void)(mo); 193 | #if defined(_M_IX86) || defined(_M_X64) 194 | return *p; 195 | #else 196 | uintptr_t x = *p; 197 | if (mo > mi_memory_order_relaxed) { 198 | while (!mi_atomic_compare_exchange_weak_explicit(p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ }; 199 | } 200 | return x; 201 | #endif 202 | } 203 | static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { 204 | (void)(mo); 205 | #if defined(_M_IX86) || defined(_M_X64) 206 | *p = x; 207 | #else 208 | mi_atomic_exchange_explicit(p, x, mo); 209 | #endif 210 | } 211 | static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) { 212 | (void)(mo); 213 | #if defined(_M_X64) 214 | return *p; 215 | #else 216 | int64_t old = *p; 217 | int64_t x = old; 218 | while ((old = InterlockedCompareExchange64(p, x, old)) != x) { 219 | x = old; 220 | } 221 | return x; 222 | #endif 223 | } 224 | static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) { 225 | (void)(mo); 226 | #if defined(x_M_IX86) || defined(_M_X64) 227 | *p = x; 228 | #else 229 | InterlockedExchange64(p, x); 230 | #endif 231 | } 232 | 233 | // These are used by the statistics 234 | static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) { 235 | #ifdef _WIN64 236 | return (int64_t)mi_atomic_addi((int64_t*)p, add); 237 | #else 238 | int64_t current; 239 | int64_t sum; 240 | do { 241 | current = *p; 242 | sum = current + add; 243 | } while (_InterlockedCompareExchange64(p, sum, current) != current); 244 | return current; 245 | #endif 246 | } 247 | static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) { 248 | int64_t current; 249 | do { 250 | current = *p; 251 | } while (current < x && _InterlockedCompareExchange64(p, x, current) != current); 252 | } 253 | 254 | static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) { 255 | mi_atomic_addi64_relaxed(p, i); 256 | } 257 | 258 | static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) { 259 | int64_t read = _InterlockedCompareExchange64(p, des, *exp); 260 | if (read == *exp) { 261 | return true; 262 | } 263 | else { 264 | *exp = read; 265 | return false; 266 | } 267 | } 268 | 269 | // The pointer macros cast to `uintptr_t`. 270 | #define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p)) 271 | #define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p)) 272 | #define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) 273 | #define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) 274 | #define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) 275 | #define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) 276 | #define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) 277 | #define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x) 278 | #define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x) 279 | 280 | #define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire)) 281 | #define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed)) 282 | #define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release)) 283 | #define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed)) 284 | 285 | 286 | #endif 287 | 288 | 289 | // Atomically add a signed value; returns the previous value. 290 | static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) { 291 | return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add); 292 | } 293 | 294 | // Atomically subtract a signed value; returns the previous value. 295 | static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) { 296 | return (intptr_t)mi_atomic_addi(p, -sub); 297 | } 298 | 299 | typedef _Atomic(uintptr_t) mi_atomic_once_t; 300 | 301 | // Returns true only on the first invocation 302 | static inline bool mi_atomic_once( mi_atomic_once_t* once ) { 303 | if (mi_atomic_load_relaxed(once) != 0) return false; // quick test 304 | uintptr_t expected = 0; 305 | return mi_atomic_cas_strong_acq_rel(once, &expected, (uintptr_t)1); // try to set to 1 306 | } 307 | 308 | typedef _Atomic(uintptr_t) mi_atomic_guard_t; 309 | 310 | // Allows only one thread to execute at a time 311 | #define mi_atomic_guard(guard) \ 312 | uintptr_t _mi_guard_expected = 0; \ 313 | for(bool _mi_guard_once = true; \ 314 | _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \ 315 | (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) ) 316 | 317 | 318 | 319 | // Yield 320 | #if defined(__cplusplus) 321 | #include 322 | static inline void mi_atomic_yield(void) { 323 | std::this_thread::yield(); 324 | } 325 | #elif defined(_WIN32) 326 | #define WIN32_LEAN_AND_MEAN 327 | #include 328 | static inline void mi_atomic_yield(void) { 329 | YieldProcessor(); 330 | } 331 | #elif defined(__SSE2__) 332 | #include 333 | static inline void mi_atomic_yield(void) { 334 | _mm_pause(); 335 | } 336 | #elif (defined(__GNUC__) || defined(__clang__)) && \ 337 | (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__armel__) || defined(__ARMEL__) || \ 338 | defined(__aarch64__) || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__)) || defined(__POWERPC__) 339 | #if defined(__x86_64__) || defined(__i386__) 340 | static inline void mi_atomic_yield(void) { 341 | __asm__ volatile ("pause" ::: "memory"); 342 | } 343 | #elif defined(__aarch64__) 344 | static inline void mi_atomic_yield(void) { 345 | __asm__ volatile("wfe"); 346 | } 347 | #elif (defined(__arm__) && __ARM_ARCH__ >= 7) 348 | static inline void mi_atomic_yield(void) { 349 | __asm__ volatile("yield" ::: "memory"); 350 | } 351 | #elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__) 352 | #ifdef __APPLE__ 353 | static inline void mi_atomic_yield(void) { 354 | __asm__ volatile ("or r27,r27,r27" ::: "memory"); 355 | } 356 | #else 357 | static inline void mi_atomic_yield(void) { 358 | __asm__ __volatile__ ("or 27,27,27" ::: "memory"); 359 | } 360 | #endif 361 | #elif defined(__armel__) || defined(__ARMEL__) 362 | static inline void mi_atomic_yield(void) { 363 | __asm__ volatile ("nop" ::: "memory"); 364 | } 365 | #endif 366 | #elif defined(__sun) 367 | // Fallback for other archs 368 | #include 369 | static inline void mi_atomic_yield(void) { 370 | smt_pause(); 371 | } 372 | #elif defined(__wasi__) 373 | #include 374 | static inline void mi_atomic_yield(void) { 375 | sched_yield(); 376 | } 377 | #else 378 | #include 379 | static inline void mi_atomic_yield(void) { 380 | sleep(0); 381 | } 382 | #endif 383 | 384 | 385 | #endif // __MIMALLOC_ATOMIC_H 386 | -------------------------------------------------------------------------------- /mimalloc/c/include/mimalloc/prim.h: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2023, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #pragma once 8 | #ifndef MIMALLOC_PRIM_H 9 | #define MIMALLOC_PRIM_H 10 | 11 | 12 | // -------------------------------------------------------------------------- 13 | // This file specifies the primitive portability API. 14 | // Each OS/host needs to implement these primitives, see `src/prim` 15 | // for implementations on Window, macOS, WASI, and Linux/Unix. 16 | // 17 | // note: on all primitive functions, we always have result parameters != NUL, and: 18 | // addr != NULL and page aligned 19 | // size > 0 and page aligned 20 | // return value is an error code an int where 0 is success. 21 | // -------------------------------------------------------------------------- 22 | 23 | // OS memory configuration 24 | typedef struct mi_os_mem_config_s { 25 | size_t page_size; // 4KiB 26 | size_t large_page_size; // 2MiB 27 | size_t alloc_granularity; // smallest allocation size (on Windows 64KiB) 28 | bool has_overcommit; // can we reserve more memory than can be actually committed? 29 | bool must_free_whole; // must allocated blocks be freed as a whole (false for mmap, true for VirtualAlloc) 30 | bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory) 31 | } mi_os_mem_config_t; 32 | 33 | // Initialize 34 | void _mi_prim_mem_init( mi_os_mem_config_t* config ); 35 | 36 | // Free OS memory 37 | int _mi_prim_free(void* addr, size_t size ); 38 | 39 | // Allocate OS memory. Return NULL on error. 40 | // The `try_alignment` is just a hint and the returned pointer does not have to be aligned. 41 | // If `commit` is false, the virtual memory range only needs to be reserved (with no access) 42 | // which will later be committed explicitly using `_mi_prim_commit`. 43 | // `is_zero` is set to true if the memory was zero initialized (as on most OS's) 44 | // pre: !commit => !allow_large 45 | // try_alignment >= _mi_os_page_size() and a power of 2 46 | int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr); 47 | 48 | // Commit memory. Returns error code or 0 on success. 49 | // For example, on Linux this would make the memory PROT_READ|PROT_WRITE. 50 | // `is_zero` is set to true if the memory was zero initialized (e.g. on Windows) 51 | int _mi_prim_commit(void* addr, size_t size, bool* is_zero); 52 | 53 | // Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true 54 | // if the memory would need to be re-committed. For example, on Windows this is always true, 55 | // but on Linux we could use MADV_DONTNEED to decommit which does not need a recommit. 56 | // pre: needs_recommit != NULL 57 | int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit); 58 | 59 | // Reset memory. The range keeps being accessible but the content might be reset. 60 | // Returns error code or 0 on success. 61 | int _mi_prim_reset(void* addr, size_t size); 62 | 63 | // Protect memory. Returns error code or 0 on success. 64 | int _mi_prim_protect(void* addr, size_t size, bool protect); 65 | 66 | // Allocate huge (1GiB) pages possibly associated with a NUMA node. 67 | // `is_zero` is set to true if the memory was zero initialized (as on most OS's) 68 | // pre: size > 0 and a multiple of 1GiB. 69 | // numa_node is either negative (don't care), or a numa node number. 70 | int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr); 71 | 72 | // Return the current NUMA node 73 | size_t _mi_prim_numa_node(void); 74 | 75 | // Return the number of logical NUMA nodes 76 | size_t _mi_prim_numa_node_count(void); 77 | 78 | // Clock ticks 79 | mi_msecs_t _mi_prim_clock_now(void); 80 | 81 | // Return process information (only for statistics) 82 | typedef struct mi_process_info_s { 83 | mi_msecs_t elapsed; 84 | mi_msecs_t utime; 85 | mi_msecs_t stime; 86 | size_t current_rss; 87 | size_t peak_rss; 88 | size_t current_commit; 89 | size_t peak_commit; 90 | size_t page_faults; 91 | } mi_process_info_t; 92 | 93 | void _mi_prim_process_info(mi_process_info_t* pinfo); 94 | 95 | // Default stderr output. (only for warnings etc. with verbose enabled) 96 | // msg != NULL && _mi_strlen(msg) > 0 97 | void _mi_prim_out_stderr( const char* msg ); 98 | 99 | // Get an environment variable. (only for options) 100 | // name != NULL, result != NULL, result_size >= 64 101 | bool _mi_prim_getenv(const char* name, char* result, size_t result_size); 102 | 103 | 104 | // Fill a buffer with strong randomness; return `false` on error or if 105 | // there is no strong randomization available. 106 | bool _mi_prim_random_buf(void* buf, size_t buf_len); 107 | 108 | // Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination. 109 | void _mi_prim_thread_init_auto_done(void); 110 | 111 | // Called on process exit and may take action to clean up resources associated with the thread auto done. 112 | void _mi_prim_thread_done_auto_done(void); 113 | 114 | // Called when the default heap for a thread changes 115 | void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); 116 | 117 | 118 | //------------------------------------------------------------------- 119 | // Thread id: `_mi_prim_thread_id()` 120 | // 121 | // Getting the thread id should be performant as it is called in the 122 | // fast path of `_mi_free` and we specialize for various platforms as 123 | // inlined definitions. Regular code should call `init.c:_mi_thread_id()`. 124 | // We only require _mi_prim_thread_id() to return a unique id 125 | // for each thread (unequal to zero). 126 | //------------------------------------------------------------------- 127 | 128 | // defined in `init.c`; do not use these directly 129 | extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from 130 | extern bool _mi_process_is_initialized; // has mi_process_init been called? 131 | 132 | static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept; 133 | 134 | #if defined(_WIN32) 135 | 136 | #define WIN32_LEAN_AND_MEAN 137 | #include 138 | static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { 139 | // Windows: works on Intel and ARM in both 32- and 64-bit 140 | return (uintptr_t)NtCurrentTeb(); 141 | } 142 | 143 | // We use assembly for a fast thread id on the main platforms. The TLS layout depends on 144 | // both the OS and libc implementation so we use specific tests for each main platform. 145 | // If you test on another platform and it works please send a PR :-) 146 | // see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. 147 | #elif defined(__GNUC__) && ( \ 148 | (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ 149 | || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \ 150 | || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ 151 | || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ 152 | || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ 153 | ) 154 | 155 | static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept { 156 | void* res; 157 | const size_t ofs = (slot*sizeof(void*)); 158 | #if defined(__i386__) 159 | __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS 160 | #elif defined(__APPLE__) && defined(__x86_64__) 161 | __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS 162 | #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) 163 | __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI 164 | #elif defined(__x86_64__) 165 | __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS 166 | #elif defined(__arm__) 167 | void** tcb; MI_UNUSED(ofs); 168 | __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); 169 | res = tcb[slot]; 170 | #elif defined(__aarch64__) 171 | void** tcb; MI_UNUSED(ofs); 172 | #if defined(__APPLE__) // M1, issue #343 173 | __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); 174 | #else 175 | __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); 176 | #endif 177 | res = tcb[slot]; 178 | #endif 179 | return res; 180 | } 181 | 182 | // setting a tls slot is only used on macOS for now 183 | static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { 184 | const size_t ofs = (slot*sizeof(void*)); 185 | #if defined(__i386__) 186 | __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS 187 | #elif defined(__APPLE__) && defined(__x86_64__) 188 | __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS 189 | #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) 190 | __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI 191 | #elif defined(__x86_64__) 192 | __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS 193 | #elif defined(__arm__) 194 | void** tcb; MI_UNUSED(ofs); 195 | __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); 196 | tcb[slot] = value; 197 | #elif defined(__aarch64__) 198 | void** tcb; MI_UNUSED(ofs); 199 | #if defined(__APPLE__) // M1, issue #343 200 | __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); 201 | #else 202 | __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); 203 | #endif 204 | tcb[slot] = value; 205 | #endif 206 | } 207 | 208 | static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { 209 | #if defined(__BIONIC__) 210 | // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id 211 | // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86 212 | return (uintptr_t)mi_prim_tls_slot(1); 213 | #else 214 | // in all our other targets, slot 0 is the thread id 215 | // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h 216 | // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36 217 | return (uintptr_t)mi_prim_tls_slot(0); 218 | #endif 219 | } 220 | 221 | #else 222 | 223 | // otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms). 224 | static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { 225 | return (uintptr_t)&_mi_heap_default; 226 | } 227 | 228 | #endif 229 | 230 | 231 | 232 | /* ---------------------------------------------------------------------------------------- 233 | The thread local default heap: `_mi_prim_get_default_heap()` 234 | This is inlined here as it is on the fast path for allocation functions. 235 | 236 | On most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a 237 | __thread local variable (`_mi_heap_default`). With the initial-exec TLS model this ensures 238 | that the storage will always be available (allocated on the thread stacks). 239 | 240 | On some platforms though we cannot use that when overriding `malloc` since the underlying 241 | TLS implementation (or the loader) will call itself `malloc` on a first access and recurse. 242 | We try to circumvent this in an efficient way: 243 | - macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the 244 | loader itself calls `malloc` even before the modules are initialized. 245 | - OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS). 246 | - DragonFly: defaults are working but seem slow compared to freeBSD (see PR #323) 247 | ------------------------------------------------------------------------------------------- */ 248 | 249 | static inline mi_heap_t* mi_prim_get_default_heap(void); 250 | 251 | #if defined(MI_MALLOC_OVERRIDE) 252 | #if defined(__APPLE__) // macOS 253 | #define MI_TLS_SLOT 89 // seems unused? 254 | // #define MI_TLS_RECURSE_GUARD 1 255 | // other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89) 256 | // see 257 | #elif defined(__OpenBSD__) 258 | // use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16) 259 | // see 260 | #define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24) 261 | // #elif defined(__DragonFly__) 262 | // #warning "mimalloc is not working correctly on DragonFly yet." 263 | // #define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) 264 | #elif defined(__ANDROID__) 265 | // See issue #381 266 | #define MI_TLS_PTHREAD 267 | #endif 268 | #endif 269 | 270 | 271 | #if defined(MI_TLS_SLOT) 272 | 273 | static inline mi_heap_t* mi_prim_get_default_heap(void) { 274 | mi_heap_t* heap = (mi_heap_t*)mi_prim_tls_slot(MI_TLS_SLOT); 275 | if mi_unlikely(heap == NULL) { 276 | #ifdef __GNUC__ 277 | __asm(""); // prevent conditional load of the address of _mi_heap_empty 278 | #endif 279 | heap = (mi_heap_t*)&_mi_heap_empty; 280 | } 281 | return heap; 282 | } 283 | 284 | #elif defined(MI_TLS_PTHREAD_SLOT_OFS) 285 | 286 | static inline mi_heap_t** mi_prim_tls_pthread_heap_slot(void) { 287 | pthread_t self = pthread_self(); 288 | #if defined(__DragonFly__) 289 | if (self==NULL) return NULL; 290 | #endif 291 | return (mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS); 292 | } 293 | 294 | static inline mi_heap_t* mi_prim_get_default_heap(void) { 295 | mi_heap_t** pheap = mi_prim_tls_pthread_heap_slot(); 296 | if mi_unlikely(pheap == NULL) return _mi_heap_main_get(); 297 | mi_heap_t* heap = *pheap; 298 | if mi_unlikely(heap == NULL) return (mi_heap_t*)&_mi_heap_empty; 299 | return heap; 300 | } 301 | 302 | #elif defined(MI_TLS_PTHREAD) 303 | 304 | extern pthread_key_t _mi_heap_default_key; 305 | static inline mi_heap_t* mi_prim_get_default_heap(void) { 306 | mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? _mi_heap_main_get() : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key)); 307 | return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); 308 | } 309 | 310 | #else // default using a thread local variable; used on most platforms. 311 | 312 | static inline mi_heap_t* mi_prim_get_default_heap(void) { 313 | #if defined(MI_TLS_RECURSE_GUARD) 314 | if (mi_unlikely(!_mi_process_is_initialized)) return _mi_heap_main_get(); 315 | #endif 316 | return _mi_heap_default; 317 | } 318 | 319 | #endif // mi_prim_get_default_heap() 320 | 321 | 322 | 323 | #endif // MIMALLOC_PRIM_H 324 | -------------------------------------------------------------------------------- /mimalloc/c/include/mimalloc/track.h: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2023, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #pragma once 8 | #ifndef MIMALLOC_TRACK_H 9 | #define MIMALLOC_TRACK_H 10 | 11 | /* ------------------------------------------------------------------------------------------------------ 12 | Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers. 13 | These can be defined for tracking allocation: 14 | 15 | #define mi_track_malloc_size(p,reqsize,size,zero) 16 | #define mi_track_free_size(p,_size) 17 | 18 | The macros are set up such that the size passed to `mi_track_free_size` 19 | always matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`). 20 | The `reqsize` is what the user requested, and `size >= reqsize`. 21 | The `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled, 22 | or otherwise it is the usable block size which may be larger than the original request. 23 | Use `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc). 24 | The `zero` parameter is `true` if the allocated block is zero initialized. 25 | 26 | Optional: 27 | 28 | #define mi_track_align(p,alignedp,offset,size) 29 | #define mi_track_resize(p,oldsize,newsize) 30 | #define mi_track_init() 31 | 32 | The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block. 33 | The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`). 34 | The `mi_track_resize` is currently unused but could be called on reallocations within a block. 35 | `mi_track_init` is called at program start. 36 | 37 | The following macros are for tools like asan and valgrind to track whether memory is 38 | defined, undefined, or not accessible at all: 39 | 40 | #define mi_track_mem_defined(p,size) 41 | #define mi_track_mem_undefined(p,size) 42 | #define mi_track_mem_noaccess(p,size) 43 | 44 | -------------------------------------------------------------------------------------------------------*/ 45 | 46 | #if MI_TRACK_VALGRIND 47 | // valgrind tool 48 | 49 | #define MI_TRACK_ENABLED 1 50 | #define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on heap_destroy 51 | #define MI_TRACK_TOOL "valgrind" 52 | 53 | #include 54 | #include 55 | 56 | #define mi_track_malloc_size(p,reqsize,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero) 57 | #define mi_track_free_size(p,_size) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/) 58 | #define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/) 59 | #define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size) 60 | #define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size) 61 | #define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size) 62 | 63 | #elif MI_TRACK_ASAN 64 | // address sanitizer 65 | 66 | #define MI_TRACK_ENABLED 1 67 | #define MI_TRACK_HEAP_DESTROY 0 68 | #define MI_TRACK_TOOL "asan" 69 | 70 | #include 71 | 72 | #define mi_track_malloc_size(p,reqsize,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size) 73 | #define mi_track_free_size(p,size) ASAN_POISON_MEMORY_REGION(p,size) 74 | #define mi_track_mem_defined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) 75 | #define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) 76 | #define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size) 77 | 78 | #elif MI_TRACK_ETW 79 | // windows event tracing 80 | 81 | #define MI_TRACK_ENABLED 1 82 | #define MI_TRACK_HEAP_DESTROY 1 83 | #define MI_TRACK_TOOL "ETW" 84 | 85 | #define WIN32_LEAN_AND_MEAN 86 | #include 87 | #include "../src/prim/windows/etw.h" 88 | 89 | #define mi_track_init() EventRegistermicrosoft_windows_mimalloc(); 90 | #define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size) 91 | #define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)(p), size) 92 | 93 | #else 94 | // no tracking 95 | 96 | #define MI_TRACK_ENABLED 0 97 | #define MI_TRACK_HEAP_DESTROY 0 98 | #define MI_TRACK_TOOL "none" 99 | 100 | #define mi_track_malloc_size(p,reqsize,size,zero) 101 | #define mi_track_free_size(p,_size) 102 | 103 | #endif 104 | 105 | // ------------------- 106 | // Utility definitions 107 | 108 | #ifndef mi_track_resize 109 | #define mi_track_resize(p,oldsize,newsize) mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false) 110 | #endif 111 | 112 | #ifndef mi_track_align 113 | #define mi_track_align(p,alignedp,offset,size) mi_track_mem_noaccess(p,offset) 114 | #endif 115 | 116 | #ifndef mi_track_init 117 | #define mi_track_init() 118 | #endif 119 | 120 | #ifndef mi_track_mem_defined 121 | #define mi_track_mem_defined(p,size) 122 | #endif 123 | 124 | #ifndef mi_track_mem_undefined 125 | #define mi_track_mem_undefined(p,size) 126 | #endif 127 | 128 | #ifndef mi_track_mem_noaccess 129 | #define mi_track_mem_noaccess(p,size) 130 | #endif 131 | 132 | 133 | #if MI_PADDING 134 | #define mi_track_malloc(p,reqsize,zero) \ 135 | if ((p)!=NULL) { \ 136 | mi_assert_internal(mi_usable_size(p)==(reqsize)); \ 137 | mi_track_malloc_size(p,reqsize,reqsize,zero); \ 138 | } 139 | #else 140 | #define mi_track_malloc(p,reqsize,zero) \ 141 | if ((p)!=NULL) { \ 142 | mi_assert_internal(mi_usable_size(p)>=(reqsize)); \ 143 | mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \ 144 | } 145 | #endif 146 | 147 | #endif 148 | -------------------------------------------------------------------------------- /mimalloc/c/src/alloc-aligned.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2021, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | #include "mimalloc.h" 9 | #include "mimalloc/internal.h" 10 | #include "mimalloc/prim.h" // mi_prim_get_default_heap 11 | 12 | #include // memset 13 | 14 | // ------------------------------------------------------ 15 | // Aligned Allocation 16 | // ------------------------------------------------------ 17 | 18 | // Fallback primitive aligned allocation -- split out for better codegen 19 | static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept 20 | { 21 | mi_assert_internal(size <= PTRDIFF_MAX); 22 | mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment)); 23 | 24 | const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)` 25 | const size_t padsize = size + MI_PADDING_SIZE; 26 | 27 | // use regular allocation if it is guaranteed to fit the alignment constraints 28 | if (offset==0 && alignment<=padsize && padsize<=MI_MAX_ALIGN_GUARANTEE && (padsize&align_mask)==0) { 29 | void* p = _mi_heap_malloc_zero(heap, size, zero); 30 | mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); 31 | return p; 32 | } 33 | 34 | void* p; 35 | size_t oversize; 36 | if mi_unlikely(alignment > MI_ALIGNMENT_MAX) { 37 | // use OS allocation for very large alignment and allocate inside a huge page (dedicated segment with 1 page) 38 | // This can support alignments >= MI_SEGMENT_SIZE by ensuring the object can be aligned at a point in the 39 | // first (and single) page such that the segment info is `MI_SEGMENT_SIZE` bytes before it (so it can be found by aligning the pointer down) 40 | if mi_unlikely(offset != 0) { 41 | // todo: cannot support offset alignment for very large alignments yet 42 | #if MI_DEBUG > 0 43 | _mi_error_message(EOVERFLOW, "aligned allocation with a very large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset); 44 | #endif 45 | return NULL; 46 | } 47 | oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); 48 | p = _mi_heap_malloc_zero_ex(heap, oversize, false, alignment); // the page block size should be large enough to align in the single huge page block 49 | // zero afterwards as only the area from the aligned_p may be committed! 50 | if (p == NULL) return NULL; 51 | } 52 | else { 53 | // otherwise over-allocate 54 | oversize = size + alignment - 1; 55 | p = _mi_heap_malloc_zero(heap, oversize, zero); 56 | if (p == NULL) return NULL; 57 | } 58 | 59 | // .. and align within the allocation 60 | const uintptr_t poffset = ((uintptr_t)p + offset) & align_mask; 61 | const uintptr_t adjust = (poffset == 0 ? 0 : alignment - poffset); 62 | mi_assert_internal(adjust < alignment); 63 | void* aligned_p = (void*)((uintptr_t)p + adjust); 64 | if (aligned_p != p) { 65 | mi_page_t* page = _mi_ptr_page(p); 66 | mi_page_set_has_aligned(page, true); 67 | _mi_padding_shrink(page, (mi_block_t*)p, adjust + size); 68 | } 69 | // todo: expand padding if overallocated ? 70 | 71 | mi_assert_internal(mi_page_usable_block_size(_mi_ptr_page(p)) >= adjust + size); 72 | mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p), _mi_ptr_page(aligned_p), aligned_p)); 73 | mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); 74 | mi_assert_internal(mi_usable_size(aligned_p)>=size); 75 | mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); 76 | 77 | // now zero the block if needed 78 | if (alignment > MI_ALIGNMENT_MAX) { 79 | // for the tracker, on huge aligned allocations only from the start of the large block is defined 80 | mi_track_mem_undefined(aligned_p, size); 81 | if (zero) { 82 | _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p)); 83 | } 84 | } 85 | 86 | if (p != aligned_p) { 87 | mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p)); 88 | } 89 | return aligned_p; 90 | } 91 | 92 | // Primitive aligned allocation 93 | static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept 94 | { 95 | // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size. 96 | if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see ) 97 | #if MI_DEBUG > 0 98 | _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n", size, alignment); 99 | #endif 100 | return NULL; 101 | } 102 | 103 | if mi_unlikely(size > PTRDIFF_MAX) { // we don't allocate more than PTRDIFF_MAX (see ) 104 | #if MI_DEBUG > 0 105 | _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); 106 | #endif 107 | return NULL; 108 | } 109 | const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` 110 | const size_t padsize = size + MI_PADDING_SIZE; // note: cannot overflow due to earlier size > PTRDIFF_MAX check 111 | 112 | // try first if there happens to be a small block available with just the right alignment 113 | if mi_likely(padsize <= MI_SMALL_SIZE_MAX && alignment <= padsize) { 114 | mi_page_t* page = _mi_heap_get_free_small_page(heap, padsize); 115 | const bool is_aligned = (((uintptr_t)page->free+offset) & align_mask)==0; 116 | if mi_likely(page->free != NULL && is_aligned) 117 | { 118 | #if MI_STAT>1 119 | mi_heap_stat_increase(heap, malloc, size); 120 | #endif 121 | void* p = _mi_page_malloc(heap, page, padsize, zero); // TODO: inline _mi_page_malloc 122 | mi_assert_internal(p != NULL); 123 | mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); 124 | mi_track_malloc(p,size,zero); 125 | return p; 126 | } 127 | } 128 | // fallback 129 | return mi_heap_malloc_zero_aligned_at_fallback(heap, size, alignment, offset, zero); 130 | } 131 | 132 | 133 | // ------------------------------------------------------ 134 | // Optimized mi_heap_malloc_aligned / mi_malloc_aligned 135 | // ------------------------------------------------------ 136 | 137 | mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 138 | return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false); 139 | } 140 | 141 | mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { 142 | if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) return NULL; 143 | #if !MI_PADDING 144 | // without padding, any small sized allocation is naturally aligned (see also `_mi_segment_page_start`) 145 | if mi_likely(_mi_is_power_of_two(size) && size >= alignment && size <= MI_SMALL_SIZE_MAX) 146 | #else 147 | // with padding, we can only guarantee this for fixed alignments 148 | if mi_likely((alignment == sizeof(void*) || (alignment == MI_MAX_ALIGN_SIZE && size > (MI_MAX_ALIGN_SIZE/2))) 149 | && size <= MI_SMALL_SIZE_MAX) 150 | #endif 151 | { 152 | // fast path for common alignment and size 153 | return mi_heap_malloc_small(heap, size); 154 | } 155 | else { 156 | return mi_heap_malloc_aligned_at(heap, size, alignment, 0); 157 | } 158 | } 159 | 160 | // ensure a definition is emitted 161 | #if defined(__cplusplus) 162 | static void* _mi_heap_malloc_aligned = (void*)&mi_heap_malloc_aligned; 163 | #endif 164 | 165 | // ------------------------------------------------------ 166 | // Aligned Allocation 167 | // ------------------------------------------------------ 168 | 169 | mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 170 | return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true); 171 | } 172 | 173 | mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { 174 | return mi_heap_zalloc_aligned_at(heap, size, alignment, 0); 175 | } 176 | 177 | mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 178 | size_t total; 179 | if (mi_count_size_overflow(count, size, &total)) return NULL; 180 | return mi_heap_zalloc_aligned_at(heap, total, alignment, offset); 181 | } 182 | 183 | mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { 184 | return mi_heap_calloc_aligned_at(heap,count,size,alignment,0); 185 | } 186 | 187 | mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 188 | return mi_heap_malloc_aligned_at(mi_prim_get_default_heap(), size, alignment, offset); 189 | } 190 | 191 | mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { 192 | return mi_heap_malloc_aligned(mi_prim_get_default_heap(), size, alignment); 193 | } 194 | 195 | mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 196 | return mi_heap_zalloc_aligned_at(mi_prim_get_default_heap(), size, alignment, offset); 197 | } 198 | 199 | mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { 200 | return mi_heap_zalloc_aligned(mi_prim_get_default_heap(), size, alignment); 201 | } 202 | 203 | mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 204 | return mi_heap_calloc_aligned_at(mi_prim_get_default_heap(), count, size, alignment, offset); 205 | } 206 | 207 | mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept { 208 | return mi_heap_calloc_aligned(mi_prim_get_default_heap(), count, size, alignment); 209 | } 210 | 211 | 212 | // ------------------------------------------------------ 213 | // Aligned re-allocation 214 | // ------------------------------------------------------ 215 | 216 | static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { 217 | mi_assert(alignment > 0); 218 | if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero); 219 | if (p == NULL) return mi_heap_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero); 220 | size_t size = mi_usable_size(p); 221 | if (newsize <= size && newsize >= (size - (size / 2)) 222 | && (((uintptr_t)p + offset) % alignment) == 0) { 223 | return p; // reallocation still fits, is aligned and not more than 50% waste 224 | } 225 | else { 226 | // note: we don't zero allocate upfront so we only zero initialize the expanded part 227 | void* newp = mi_heap_malloc_aligned_at(heap,newsize,alignment,offset); 228 | if (newp != NULL) { 229 | if (zero && newsize > size) { 230 | // also set last word in the previous allocation to zero to ensure any padding is zero-initialized 231 | size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); 232 | _mi_memzero((uint8_t*)newp + start, newsize - start); 233 | } 234 | _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize)); 235 | mi_free(p); // only free if successful 236 | } 237 | return newp; 238 | } 239 | } 240 | 241 | static void* mi_heap_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept { 242 | mi_assert(alignment > 0); 243 | if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero); 244 | size_t offset = ((uintptr_t)p % alignment); // use offset of previous allocation (p can be NULL) 245 | return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero); 246 | } 247 | 248 | mi_decl_nodiscard void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { 249 | return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false); 250 | } 251 | 252 | mi_decl_nodiscard void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { 253 | return mi_heap_realloc_zero_aligned(heap,p,newsize,alignment,false); 254 | } 255 | 256 | mi_decl_nodiscard void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { 257 | return mi_heap_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true); 258 | } 259 | 260 | mi_decl_nodiscard void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { 261 | return mi_heap_realloc_zero_aligned(heap, p, newsize, alignment, true); 262 | } 263 | 264 | mi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 265 | size_t total; 266 | if (mi_count_size_overflow(newcount, size, &total)) return NULL; 267 | return mi_heap_rezalloc_aligned_at(heap, p, total, alignment, offset); 268 | } 269 | 270 | mi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { 271 | size_t total; 272 | if (mi_count_size_overflow(newcount, size, &total)) return NULL; 273 | return mi_heap_rezalloc_aligned(heap, p, total, alignment); 274 | } 275 | 276 | mi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { 277 | return mi_heap_realloc_aligned_at(mi_prim_get_default_heap(), p, newsize, alignment, offset); 278 | } 279 | 280 | mi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { 281 | return mi_heap_realloc_aligned(mi_prim_get_default_heap(), p, newsize, alignment); 282 | } 283 | 284 | mi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { 285 | return mi_heap_rezalloc_aligned_at(mi_prim_get_default_heap(), p, newsize, alignment, offset); 286 | } 287 | 288 | mi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { 289 | return mi_heap_rezalloc_aligned(mi_prim_get_default_heap(), p, newsize, alignment); 290 | } 291 | 292 | mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 293 | return mi_heap_recalloc_aligned_at(mi_prim_get_default_heap(), p, newcount, size, alignment, offset); 294 | } 295 | 296 | mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { 297 | return mi_heap_recalloc_aligned(mi_prim_get_default_heap(), p, newcount, size, alignment); 298 | } 299 | -------------------------------------------------------------------------------- /mimalloc/c/src/alloc-override.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2021, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | #if !defined(MI_IN_ALLOC_C) 9 | #error "this file should be included from 'alloc.c' (so aliases can work)" 10 | #endif 11 | 12 | #if defined(MI_MALLOC_OVERRIDE) && defined(_WIN32) && !(defined(MI_SHARED_LIB) && defined(_DLL)) 13 | #error "It is only possible to override "malloc" on Windows when building as a DLL (and linking the C runtime as a DLL)" 14 | #endif 15 | 16 | #if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) 17 | 18 | #if defined(__APPLE__) 19 | #include 20 | mi_decl_externc void vfree(void* p); 21 | mi_decl_externc size_t malloc_size(const void* p); 22 | mi_decl_externc size_t malloc_good_size(size_t size); 23 | #endif 24 | 25 | // helper definition for C override of C++ new 26 | typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; 27 | 28 | // ------------------------------------------------------ 29 | // Override system malloc 30 | // ------------------------------------------------------ 31 | 32 | #if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !MI_TRACK_ENABLED 33 | // gcc, clang: use aliasing to alias the exported function to one of our `mi_` functions 34 | #if (defined(__GNUC__) && __GNUC__ >= 9) 35 | #pragma GCC diagnostic ignored "-Wattributes" // or we get warnings that nodiscard is ignored on a forward 36 | #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))); 37 | #else 38 | #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))); 39 | #endif 40 | #define MI_FORWARD1(fun,x) MI_FORWARD(fun) 41 | #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) 42 | #define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun) 43 | #define MI_FORWARD0(fun,x) MI_FORWARD(fun) 44 | #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun) 45 | #else 46 | // otherwise use forwarding by calling our `mi_` function 47 | #define MI_FORWARD1(fun,x) { return fun(x); } 48 | #define MI_FORWARD2(fun,x,y) { return fun(x,y); } 49 | #define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); } 50 | #define MI_FORWARD0(fun,x) { fun(x); } 51 | #define MI_FORWARD02(fun,x,y) { fun(x,y); } 52 | #endif 53 | 54 | 55 | #if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) 56 | // define MI_OSX_IS_INTERPOSED as we should not provide forwarding definitions for 57 | // functions that are interposed (or the interposing does not work) 58 | #define MI_OSX_IS_INTERPOSED 59 | 60 | mi_decl_externc size_t mi_malloc_size_checked(void *p) { 61 | if (!mi_is_in_heap_region(p)) return 0; 62 | return mi_usable_size(p); 63 | } 64 | 65 | // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` 66 | // See: 67 | struct mi_interpose_s { 68 | const void* replacement; 69 | const void* target; 70 | }; 71 | #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } 72 | #define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) 73 | 74 | __attribute__((used)) static struct mi_interpose_s _mi_interposes[] __attribute__((section("__DATA, __interpose"))) = 75 | { 76 | MI_INTERPOSE_MI(malloc), 77 | MI_INTERPOSE_MI(calloc), 78 | MI_INTERPOSE_MI(realloc), 79 | MI_INTERPOSE_MI(strdup), 80 | MI_INTERPOSE_MI(strndup), 81 | MI_INTERPOSE_MI(realpath), 82 | MI_INTERPOSE_MI(posix_memalign), 83 | MI_INTERPOSE_MI(reallocf), 84 | MI_INTERPOSE_MI(valloc), 85 | MI_INTERPOSE_FUN(malloc_size,mi_malloc_size_checked), 86 | MI_INTERPOSE_MI(malloc_good_size), 87 | #if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 88 | MI_INTERPOSE_MI(aligned_alloc), 89 | #endif 90 | #ifdef MI_OSX_ZONE 91 | // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely 92 | MI_INTERPOSE_MI(free), 93 | MI_INTERPOSE_FUN(vfree,mi_free), 94 | #else 95 | // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) 96 | MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us 97 | MI_INTERPOSE_FUN(vfree,mi_cfree), 98 | #endif 99 | }; 100 | 101 | #ifdef __cplusplus 102 | extern "C" { 103 | #endif 104 | void _ZdlPv(void* p); // delete 105 | void _ZdaPv(void* p); // delete[] 106 | void _ZdlPvm(void* p, size_t n); // delete 107 | void _ZdaPvm(void* p, size_t n); // delete[] 108 | void* _Znwm(size_t n); // new 109 | void* _Znam(size_t n); // new[] 110 | void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow 111 | void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow 112 | #ifdef __cplusplus 113 | } 114 | #endif 115 | __attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[] __attribute__((section("__DATA, __interpose"))) = 116 | { 117 | MI_INTERPOSE_FUN(_ZdlPv,mi_free), 118 | MI_INTERPOSE_FUN(_ZdaPv,mi_free), 119 | MI_INTERPOSE_FUN(_ZdlPvm,mi_free_size), 120 | MI_INTERPOSE_FUN(_ZdaPvm,mi_free_size), 121 | MI_INTERPOSE_FUN(_Znwm,mi_new), 122 | MI_INTERPOSE_FUN(_Znam,mi_new), 123 | MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow), 124 | MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow), 125 | }; 126 | 127 | #elif defined(_MSC_VER) 128 | // cannot override malloc unless using a dll. 129 | // we just override new/delete which does work in a static library. 130 | #else 131 | // On all other systems forward to our API 132 | mi_decl_export void* malloc(size_t size) MI_FORWARD1(mi_malloc, size) 133 | mi_decl_export void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n) 134 | mi_decl_export void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize) 135 | mi_decl_export void free(void* p) MI_FORWARD0(mi_free, p) 136 | #endif 137 | 138 | #if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) 139 | #pragma GCC visibility push(default) 140 | #endif 141 | 142 | // ------------------------------------------------------ 143 | // Override new/delete 144 | // This is not really necessary as they usually call 145 | // malloc/free anyway, but it improves performance. 146 | // ------------------------------------------------------ 147 | #ifdef __cplusplus 148 | // ------------------------------------------------------ 149 | // With a C++ compiler we override the new/delete operators. 150 | // see 151 | // ------------------------------------------------------ 152 | #include 153 | 154 | #ifndef MI_OSX_IS_INTERPOSED 155 | void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p) 156 | void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p) 157 | 158 | void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) 159 | void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) 160 | 161 | void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); } 162 | void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); } 163 | 164 | #if (__cplusplus >= 201402L || _MSC_VER >= 1916) 165 | void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) 166 | void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) 167 | #endif 168 | #endif 169 | 170 | #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5)) 171 | void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } 172 | void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } 173 | void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; 174 | void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; 175 | void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); } 176 | void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); } 177 | 178 | void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } 179 | void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } 180 | void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } 181 | void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } 182 | #endif 183 | 184 | #elif (defined(__GNUC__) || defined(__clang__)) 185 | // ------------------------------------------------------ 186 | // Override by defining the mangled C++ names of the operators (as 187 | // used by GCC and CLang). 188 | // See 189 | // ------------------------------------------------------ 190 | 191 | void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete 192 | void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[] 193 | void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) 194 | void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) 195 | void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } 196 | void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } 197 | void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } 198 | void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } 199 | 200 | #if (MI_INTPTR_SIZE==8) 201 | void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit 202 | void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit 203 | void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } 204 | void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } 205 | void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) 206 | void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) 207 | void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } 208 | void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } 209 | #elif (MI_INTPTR_SIZE==4) 210 | void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit 211 | void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit 212 | void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } 213 | void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } 214 | void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) 215 | void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) 216 | void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } 217 | void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } 218 | #else 219 | #error "define overloads for new/delete for this platform (just for performance, can be skipped)" 220 | #endif 221 | #endif // __cplusplus 222 | 223 | // ------------------------------------------------------ 224 | // Further Posix & Unix functions definitions 225 | // ------------------------------------------------------ 226 | 227 | #ifdef __cplusplus 228 | extern "C" { 229 | #endif 230 | 231 | #ifndef MI_OSX_IS_INTERPOSED 232 | // Forward Posix/Unix calls as well 233 | void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) 234 | size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) 235 | #if !defined(__ANDROID__) && !defined(__FreeBSD__) 236 | size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) 237 | #else 238 | size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) 239 | #endif 240 | 241 | // No forwarding here due to aliasing/name mangling issues 242 | void* valloc(size_t size) { return mi_valloc(size); } 243 | void vfree(void* p) { mi_free(p); } 244 | size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); } 245 | int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } 246 | 247 | // `aligned_alloc` is only available when __USE_ISOC11 is defined. 248 | // Note: it seems __USE_ISOC11 is not defined in musl (and perhaps other libc's) so we only check 249 | // for it if using glibc. 250 | // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot 251 | // override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. 252 | // Fortunately, in the case where `aligned_alloc` is declared as `static inline` it 253 | // uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves. 254 | #if !defined(__GLIBC__) || __USE_ISOC11 255 | void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } 256 | #endif 257 | #endif 258 | 259 | // no forwarding here due to aliasing/name mangling issues 260 | void cfree(void* p) { mi_free(p); } 261 | void* pvalloc(size_t size) { return mi_pvalloc(size); } 262 | void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } 263 | int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); } 264 | void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } 265 | void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } 266 | 267 | #if defined(__wasi__) 268 | // forward __libc interface (see PR #667) 269 | void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc, size) 270 | void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc, count, size) 271 | void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc, p, size) 272 | void __libc_free(void* p) MI_FORWARD0(mi_free, p) 273 | void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } 274 | 275 | #elif defined(__GLIBC__) && defined(__linux__) 276 | // forward __libc interface (needed for glibc-based Linux distributions) 277 | void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) 278 | void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size) 279 | void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size) 280 | void __libc_free(void* p) MI_FORWARD0(mi_free,p) 281 | void __libc_cfree(void* p) MI_FORWARD0(mi_free,p) 282 | 283 | void* __libc_valloc(size_t size) { return mi_valloc(size); } 284 | void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } 285 | void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); } 286 | int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); } 287 | #endif 288 | 289 | #ifdef __cplusplus 290 | } 291 | #endif 292 | 293 | #if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) 294 | #pragma GCC visibility pop 295 | #endif 296 | 297 | #endif // MI_MALLOC_OVERRIDE && !_WIN32 298 | -------------------------------------------------------------------------------- /mimalloc/c/src/alloc-posix.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2021, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | // ------------------------------------------------------------------------ 9 | // mi prefixed publi definitions of various Posix, Unix, and C++ functions 10 | // for convenience and used when overriding these functions. 11 | // ------------------------------------------------------------------------ 12 | #include "mimalloc.h" 13 | #include "mimalloc/internal.h" 14 | 15 | // ------------------------------------------------------ 16 | // Posix & Unix functions definitions 17 | // ------------------------------------------------------ 18 | 19 | #include 20 | #include // memset 21 | #include // getenv 22 | 23 | #ifdef _MSC_VER 24 | #pragma warning(disable:4996) // getenv _wgetenv 25 | #endif 26 | 27 | #ifndef EINVAL 28 | #define EINVAL 22 29 | #endif 30 | #ifndef ENOMEM 31 | #define ENOMEM 12 32 | #endif 33 | 34 | 35 | mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept { 36 | // if (!mi_is_in_heap_region(p)) return 0; 37 | return mi_usable_size(p); 38 | } 39 | 40 | mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { 41 | // if (!mi_is_in_heap_region(p)) return 0; 42 | return mi_usable_size(p); 43 | } 44 | 45 | mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept { 46 | return mi_good_size(size); 47 | } 48 | 49 | void mi_cfree(void* p) mi_attr_noexcept { 50 | if (mi_is_in_heap_region(p)) { 51 | mi_free(p); 52 | } 53 | } 54 | 55 | int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept { 56 | // Note: The spec dictates we should not modify `*p` on an error. (issue#27) 57 | // 58 | if (p == NULL) return EINVAL; 59 | if ((alignment % sizeof(void*)) != 0) return EINVAL; // natural alignment 60 | // it is also required that alignment is a power of 2 and > 0; this is checked in `mi_malloc_aligned` 61 | if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2 62 | void* q = mi_malloc_aligned(size, alignment); 63 | if (q==NULL && size != 0) return ENOMEM; 64 | mi_assert_internal(((uintptr_t)q % alignment) == 0); 65 | *p = q; 66 | return 0; 67 | } 68 | 69 | mi_decl_nodiscard mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept { 70 | void* p = mi_malloc_aligned(size, alignment); 71 | mi_assert_internal(((uintptr_t)p % alignment) == 0); 72 | return p; 73 | } 74 | 75 | mi_decl_nodiscard mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept { 76 | return mi_memalign( _mi_os_page_size(), size ); 77 | } 78 | 79 | mi_decl_nodiscard mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept { 80 | size_t psize = _mi_os_page_size(); 81 | if (size >= SIZE_MAX - psize) return NULL; // overflow 82 | size_t asize = _mi_align_up(size, psize); 83 | return mi_malloc_aligned(asize, psize); 84 | } 85 | 86 | mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept { 87 | // C11 requires the size to be an integral multiple of the alignment, see . 88 | // unfortunately, it turns out quite some programs pass a size that is not an integral multiple so skip this check.. 89 | /* if mi_unlikely((size & (alignment - 1)) != 0) { // C11 requires alignment>0 && integral multiple, see 90 | #if MI_DEBUG > 0 91 | _mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\n", size, alignment); 92 | #endif 93 | return NULL; 94 | } 95 | */ 96 | // C11 also requires alignment to be a power-of-two (and > 0) which is checked in mi_malloc_aligned 97 | void* p = mi_malloc_aligned(size, alignment); 98 | mi_assert_internal(((uintptr_t)p % alignment) == 0); 99 | return p; 100 | } 101 | 102 | mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD 103 | void* newp = mi_reallocn(p,count,size); 104 | if (newp==NULL) { errno = ENOMEM; } 105 | return newp; 106 | } 107 | 108 | mi_decl_nodiscard int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD 109 | mi_assert(p != NULL); 110 | if (p == NULL) { 111 | errno = EINVAL; 112 | return EINVAL; 113 | } 114 | void** op = (void**)p; 115 | void* newp = mi_reallocarray(*op, count, size); 116 | if mi_unlikely(newp == NULL) { return errno; } 117 | *op = newp; 118 | return 0; 119 | } 120 | 121 | void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft 122 | void* res = mi_expand(p, newsize); 123 | if (res == NULL) { errno = ENOMEM; } 124 | return res; 125 | } 126 | 127 | mi_decl_nodiscard mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept { 128 | if (s==NULL) return NULL; 129 | size_t len; 130 | for(len = 0; s[len] != 0; len++) { } 131 | size_t size = (len+1)*sizeof(unsigned short); 132 | unsigned short* p = (unsigned short*)mi_malloc(size); 133 | if (p != NULL) { 134 | _mi_memcpy(p,s,size); 135 | } 136 | return p; 137 | } 138 | 139 | mi_decl_nodiscard mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept { 140 | return (unsigned char*)mi_strdup((const char*)s); 141 | } 142 | 143 | int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept { 144 | if (buf==NULL || name==NULL) return EINVAL; 145 | if (size != NULL) *size = 0; 146 | char* p = getenv(name); // mscver warning 4996 147 | if (p==NULL) { 148 | *buf = NULL; 149 | } 150 | else { 151 | *buf = mi_strdup(p); 152 | if (*buf==NULL) return ENOMEM; 153 | if (size != NULL) *size = _mi_strlen(p); 154 | } 155 | return 0; 156 | } 157 | 158 | int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept { 159 | if (buf==NULL || name==NULL) return EINVAL; 160 | if (size != NULL) *size = 0; 161 | #if !defined(_WIN32) || (defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP)) 162 | // not supported 163 | *buf = NULL; 164 | return EINVAL; 165 | #else 166 | unsigned short* p = (unsigned short*)_wgetenv((const wchar_t*)name); // msvc warning 4996 167 | if (p==NULL) { 168 | *buf = NULL; 169 | } 170 | else { 171 | *buf = mi_wcsdup(p); 172 | if (*buf==NULL) return ENOMEM; 173 | if (size != NULL) *size = wcslen((const wchar_t*)p); 174 | } 175 | return 0; 176 | #endif 177 | } 178 | 179 | mi_decl_nodiscard void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft 180 | return mi_recalloc_aligned_at(p, newcount, size, alignment, offset); 181 | } 182 | 183 | mi_decl_nodiscard void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft 184 | return mi_recalloc_aligned(p, newcount, size, alignment); 185 | } 186 | -------------------------------------------------------------------------------- /mimalloc/c/src/bitmap.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2019-2023 Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | /* ---------------------------------------------------------------------------- 9 | Concurrent bitmap that can set/reset sequences of bits atomically, 10 | represeted as an array of fields where each field is a machine word (`size_t`) 11 | 12 | There are two api's; the standard one cannot have sequences that cross 13 | between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). 14 | 15 | The `_across` postfixed functions do allow sequences that can cross over 16 | between the fields. (This is used in arena allocation) 17 | ---------------------------------------------------------------------------- */ 18 | 19 | #include "mimalloc.h" 20 | #include "mimalloc/internal.h" 21 | #include "bitmap.h" 22 | 23 | /* ----------------------------------------------------------- 24 | Bitmap definition 25 | ----------------------------------------------------------- */ 26 | 27 | // The bit mask for a given number of blocks at a specified bit index. 28 | static inline size_t mi_bitmap_mask_(size_t count, size_t bitidx) { 29 | mi_assert_internal(count + bitidx <= MI_BITMAP_FIELD_BITS); 30 | mi_assert_internal(count > 0); 31 | if (count >= MI_BITMAP_FIELD_BITS) return MI_BITMAP_FIELD_FULL; 32 | if (count == 0) return 0; 33 | return ((((size_t)1 << count) - 1) << bitidx); 34 | } 35 | 36 | 37 | /* ----------------------------------------------------------- 38 | Claim a bit sequence atomically 39 | ----------------------------------------------------------- */ 40 | 41 | // Try to atomically claim a sequence of `count` bits in a single 42 | // field at `idx` in `bitmap`. Returns `true` on success. 43 | inline bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx) 44 | { 45 | mi_assert_internal(bitmap_idx != NULL); 46 | mi_assert_internal(count <= MI_BITMAP_FIELD_BITS); 47 | mi_assert_internal(count > 0); 48 | mi_bitmap_field_t* field = &bitmap[idx]; 49 | size_t map = mi_atomic_load_relaxed(field); 50 | if (map==MI_BITMAP_FIELD_FULL) return false; // short cut 51 | 52 | // search for 0-bit sequence of length count 53 | const size_t mask = mi_bitmap_mask_(count, 0); 54 | const size_t bitidx_max = MI_BITMAP_FIELD_BITS - count; 55 | 56 | #ifdef MI_HAVE_FAST_BITSCAN 57 | size_t bitidx = mi_ctz(~map); // quickly find the first zero bit if possible 58 | #else 59 | size_t bitidx = 0; // otherwise start at 0 60 | #endif 61 | size_t m = (mask << bitidx); // invariant: m == mask shifted by bitidx 62 | 63 | // scan linearly for a free range of zero bits 64 | while (bitidx <= bitidx_max) { 65 | const size_t mapm = (map & m); 66 | if (mapm == 0) { // are the mask bits free at bitidx? 67 | mi_assert_internal((m >> bitidx) == mask); // no overflow? 68 | const size_t newmap = (map | m); 69 | mi_assert_internal((newmap^map) >> bitidx == mask); 70 | if (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)) { // TODO: use weak cas here? 71 | // no success, another thread claimed concurrently.. keep going (with updated `map`) 72 | continue; 73 | } 74 | else { 75 | // success, we claimed the bits! 76 | *bitmap_idx = mi_bitmap_index_create(idx, bitidx); 77 | return true; 78 | } 79 | } 80 | else { 81 | // on to the next bit range 82 | #ifdef MI_HAVE_FAST_BITSCAN 83 | mi_assert_internal(mapm != 0); 84 | const size_t shift = (count == 1 ? 1 : (MI_INTPTR_BITS - mi_clz(mapm) - bitidx)); 85 | mi_assert_internal(shift > 0 && shift <= count); 86 | #else 87 | const size_t shift = 1; 88 | #endif 89 | bitidx += shift; 90 | m <<= shift; 91 | } 92 | } 93 | // no bits found 94 | return false; 95 | } 96 | 97 | // Find `count` bits of 0 and set them to 1 atomically; returns `true` on success. 98 | // Starts at idx, and wraps around to search in all `bitmap_fields` fields. 99 | // `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. 100 | bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) { 101 | size_t idx = start_field_idx; 102 | for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) { 103 | if (idx >= bitmap_fields) { idx = 0; } // wrap 104 | if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) { 105 | return true; 106 | } 107 | } 108 | return false; 109 | } 110 | 111 | // Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled 112 | bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, 113 | const size_t start_field_idx, const size_t count, 114 | mi_bitmap_pred_fun_t pred_fun, void* pred_arg, 115 | mi_bitmap_index_t* bitmap_idx) { 116 | size_t idx = start_field_idx; 117 | for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) { 118 | if (idx >= bitmap_fields) idx = 0; // wrap 119 | if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) { 120 | if (pred_fun == NULL || pred_fun(*bitmap_idx, pred_arg)) { 121 | return true; 122 | } 123 | // predicate returned false, unclaim and look further 124 | _mi_bitmap_unclaim(bitmap, bitmap_fields, count, *bitmap_idx); 125 | } 126 | } 127 | return false; 128 | } 129 | 130 | // Set `count` bits at `bitmap_idx` to 0 atomically 131 | // Returns `true` if all `count` bits were 1 previously. 132 | bool _mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { 133 | const size_t idx = mi_bitmap_index_field(bitmap_idx); 134 | const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); 135 | const size_t mask = mi_bitmap_mask_(count, bitidx); 136 | mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); 137 | // mi_assert_internal((bitmap[idx] & mask) == mask); 138 | const size_t prev = mi_atomic_and_acq_rel(&bitmap[idx], ~mask); 139 | return ((prev & mask) == mask); 140 | } 141 | 142 | 143 | // Set `count` bits at `bitmap_idx` to 1 atomically 144 | // Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. 145 | bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero) { 146 | const size_t idx = mi_bitmap_index_field(bitmap_idx); 147 | const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); 148 | const size_t mask = mi_bitmap_mask_(count, bitidx); 149 | mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); 150 | //mi_assert_internal(any_zero != NULL || (bitmap[idx] & mask) == 0); 151 | size_t prev = mi_atomic_or_acq_rel(&bitmap[idx], mask); 152 | if (any_zero != NULL) { *any_zero = ((prev & mask) != mask); } 153 | return ((prev & mask) == 0); 154 | } 155 | 156 | // Returns `true` if all `count` bits were 1. `any_ones` is `true` if there was at least one bit set to one. 157 | static bool mi_bitmap_is_claimedx(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_ones) { 158 | const size_t idx = mi_bitmap_index_field(bitmap_idx); 159 | const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); 160 | const size_t mask = mi_bitmap_mask_(count, bitidx); 161 | mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); 162 | const size_t field = mi_atomic_load_relaxed(&bitmap[idx]); 163 | if (any_ones != NULL) { *any_ones = ((field & mask) != 0); } 164 | return ((field & mask) == mask); 165 | } 166 | 167 | // Try to set `count` bits at `bitmap_idx` from 0 to 1 atomically. 168 | // Returns `true` if successful when all previous `count` bits were 0. 169 | bool _mi_bitmap_try_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { 170 | const size_t idx = mi_bitmap_index_field(bitmap_idx); 171 | const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); 172 | const size_t mask = mi_bitmap_mask_(count, bitidx); 173 | mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); 174 | size_t expected = mi_atomic_load_relaxed(&bitmap[idx]); 175 | do { 176 | if ((expected & mask) != 0) return false; 177 | } 178 | while (!mi_atomic_cas_strong_acq_rel(&bitmap[idx], &expected, expected | mask)); 179 | mi_assert_internal((expected & mask) == 0); 180 | return true; 181 | } 182 | 183 | 184 | bool _mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { 185 | return mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, NULL); 186 | } 187 | 188 | bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { 189 | bool any_ones; 190 | mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, &any_ones); 191 | return any_ones; 192 | } 193 | 194 | 195 | //-------------------------------------------------------------------------- 196 | // the `_across` functions work on bitmaps where sequences can cross over 197 | // between the fields. This is used in arena allocation 198 | //-------------------------------------------------------------------------- 199 | 200 | // Try to atomically claim a sequence of `count` bits starting from the field 201 | // at `idx` in `bitmap` and crossing into subsequent fields. Returns `true` on success. 202 | // Only needs to consider crossing into the next fields (see `mi_bitmap_try_find_from_claim_across`) 203 | static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t idx, const size_t count, const size_t retries, mi_bitmap_index_t* bitmap_idx) 204 | { 205 | mi_assert_internal(bitmap_idx != NULL); 206 | 207 | // check initial trailing zeros 208 | mi_bitmap_field_t* field = &bitmap[idx]; 209 | size_t map = mi_atomic_load_relaxed(field); 210 | const size_t initial = mi_clz(map); // count of initial zeros starting at idx 211 | mi_assert_internal(initial <= MI_BITMAP_FIELD_BITS); 212 | if (initial == 0) return false; 213 | if (initial >= count) return _mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx); // no need to cross fields (this case won't happen for us) 214 | if (_mi_divide_up(count - initial, MI_BITMAP_FIELD_BITS) >= (bitmap_fields - idx)) return false; // not enough entries 215 | 216 | // scan ahead 217 | size_t found = initial; 218 | size_t mask = 0; // mask bits for the final field 219 | while(found < count) { 220 | field++; 221 | map = mi_atomic_load_relaxed(field); 222 | const size_t mask_bits = (found + MI_BITMAP_FIELD_BITS <= count ? MI_BITMAP_FIELD_BITS : (count - found)); 223 | mi_assert_internal(mask_bits > 0 && mask_bits <= MI_BITMAP_FIELD_BITS); 224 | mask = mi_bitmap_mask_(mask_bits, 0); 225 | if ((map & mask) != 0) return false; // some part is already claimed 226 | found += mask_bits; 227 | } 228 | mi_assert_internal(field < &bitmap[bitmap_fields]); 229 | 230 | // we found a range of contiguous zeros up to the final field; mask contains mask in the final field 231 | // now try to claim the range atomically 232 | mi_bitmap_field_t* const final_field = field; 233 | const size_t final_mask = mask; 234 | mi_bitmap_field_t* const initial_field = &bitmap[idx]; 235 | const size_t initial_idx = MI_BITMAP_FIELD_BITS - initial; 236 | const size_t initial_mask = mi_bitmap_mask_(initial, initial_idx); 237 | 238 | // initial field 239 | size_t newmap; 240 | field = initial_field; 241 | map = mi_atomic_load_relaxed(field); 242 | do { 243 | newmap = (map | initial_mask); 244 | if ((map & initial_mask) != 0) { goto rollback; }; 245 | } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); 246 | 247 | // intermediate fields 248 | while (++field < final_field) { 249 | newmap = MI_BITMAP_FIELD_FULL; 250 | map = 0; 251 | if (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)) { goto rollback; } 252 | } 253 | 254 | // final field 255 | mi_assert_internal(field == final_field); 256 | map = mi_atomic_load_relaxed(field); 257 | do { 258 | newmap = (map | final_mask); 259 | if ((map & final_mask) != 0) { goto rollback; } 260 | } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); 261 | 262 | // claimed! 263 | *bitmap_idx = mi_bitmap_index_create(idx, initial_idx); 264 | return true; 265 | 266 | rollback: 267 | // roll back intermediate fields 268 | // (we just failed to claim `field` so decrement first) 269 | while (--field > initial_field) { 270 | newmap = 0; 271 | map = MI_BITMAP_FIELD_FULL; 272 | mi_assert_internal(mi_atomic_load_relaxed(field) == map); 273 | mi_atomic_store_release(field, newmap); 274 | } 275 | if (field == initial_field) { // (if we failed on the initial field, `field + 1 == initial_field`) 276 | map = mi_atomic_load_relaxed(field); 277 | do { 278 | mi_assert_internal((map & initial_mask) == initial_mask); 279 | newmap = (map & ~initial_mask); 280 | } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); 281 | } 282 | // retry? (we make a recursive call instead of goto to be able to use const declarations) 283 | if (retries <= 2) { 284 | return mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, retries+1, bitmap_idx); 285 | } 286 | else { 287 | return false; 288 | } 289 | } 290 | 291 | 292 | // Find `count` bits of zeros and set them to 1 atomically; returns `true` on success. 293 | // Starts at idx, and wraps around to search in all `bitmap_fields` fields. 294 | bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) { 295 | mi_assert_internal(count > 0); 296 | if (count <= 2) { 297 | // we don't bother with crossover fields for small counts 298 | return _mi_bitmap_try_find_from_claim(bitmap, bitmap_fields, start_field_idx, count, bitmap_idx); 299 | } 300 | 301 | // visit the fields 302 | size_t idx = start_field_idx; 303 | for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) { 304 | if (idx >= bitmap_fields) { idx = 0; } // wrap 305 | // first try to claim inside a field 306 | if (count <= MI_BITMAP_FIELD_BITS) { 307 | if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) { 308 | return true; 309 | } 310 | } 311 | // if that fails, then try to claim across fields 312 | if (mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, 0, bitmap_idx)) { 313 | return true; 314 | } 315 | } 316 | return false; 317 | } 318 | 319 | // Helper for masks across fields; returns the mid count, post_mask may be 0 320 | static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, size_t* pre_mask, size_t* mid_mask, size_t* post_mask) { 321 | MI_UNUSED(bitmap_fields); 322 | const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); 323 | if mi_likely(bitidx + count <= MI_BITMAP_FIELD_BITS) { 324 | *pre_mask = mi_bitmap_mask_(count, bitidx); 325 | *mid_mask = 0; 326 | *post_mask = 0; 327 | mi_assert_internal(mi_bitmap_index_field(bitmap_idx) < bitmap_fields); 328 | return 0; 329 | } 330 | else { 331 | const size_t pre_bits = MI_BITMAP_FIELD_BITS - bitidx; 332 | mi_assert_internal(pre_bits < count); 333 | *pre_mask = mi_bitmap_mask_(pre_bits, bitidx); 334 | count -= pre_bits; 335 | const size_t mid_count = (count / MI_BITMAP_FIELD_BITS); 336 | *mid_mask = MI_BITMAP_FIELD_FULL; 337 | count %= MI_BITMAP_FIELD_BITS; 338 | *post_mask = (count==0 ? 0 : mi_bitmap_mask_(count, 0)); 339 | mi_assert_internal(mi_bitmap_index_field(bitmap_idx) + mid_count + (count==0 ? 0 : 1) < bitmap_fields); 340 | return mid_count; 341 | } 342 | } 343 | 344 | // Set `count` bits at `bitmap_idx` to 0 atomically 345 | // Returns `true` if all `count` bits were 1 previously. 346 | bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { 347 | size_t idx = mi_bitmap_index_field(bitmap_idx); 348 | size_t pre_mask; 349 | size_t mid_mask; 350 | size_t post_mask; 351 | size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); 352 | bool all_one = true; 353 | mi_bitmap_field_t* field = &bitmap[idx]; 354 | size_t prev = mi_atomic_and_acq_rel(field++, ~pre_mask); // clear first part 355 | if ((prev & pre_mask) != pre_mask) all_one = false; 356 | while(mid_count-- > 0) { 357 | prev = mi_atomic_and_acq_rel(field++, ~mid_mask); // clear mid part 358 | if ((prev & mid_mask) != mid_mask) all_one = false; 359 | } 360 | if (post_mask!=0) { 361 | prev = mi_atomic_and_acq_rel(field, ~post_mask); // clear end part 362 | if ((prev & post_mask) != post_mask) all_one = false; 363 | } 364 | return all_one; 365 | } 366 | 367 | // Set `count` bits at `bitmap_idx` to 1 atomically 368 | // Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. 369 | bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero) { 370 | size_t idx = mi_bitmap_index_field(bitmap_idx); 371 | size_t pre_mask; 372 | size_t mid_mask; 373 | size_t post_mask; 374 | size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); 375 | bool all_zero = true; 376 | bool any_zero = false; 377 | _Atomic(size_t)*field = &bitmap[idx]; 378 | size_t prev = mi_atomic_or_acq_rel(field++, pre_mask); 379 | if ((prev & pre_mask) != 0) all_zero = false; 380 | if ((prev & pre_mask) != pre_mask) any_zero = true; 381 | while (mid_count-- > 0) { 382 | prev = mi_atomic_or_acq_rel(field++, mid_mask); 383 | if ((prev & mid_mask) != 0) all_zero = false; 384 | if ((prev & mid_mask) != mid_mask) any_zero = true; 385 | } 386 | if (post_mask!=0) { 387 | prev = mi_atomic_or_acq_rel(field, post_mask); 388 | if ((prev & post_mask) != 0) all_zero = false; 389 | if ((prev & post_mask) != post_mask) any_zero = true; 390 | } 391 | if (pany_zero != NULL) { *pany_zero = any_zero; } 392 | return all_zero; 393 | } 394 | 395 | 396 | // Returns `true` if all `count` bits were 1. 397 | // `any_ones` is `true` if there was at least one bit set to one. 398 | static bool mi_bitmap_is_claimedx_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_ones) { 399 | size_t idx = mi_bitmap_index_field(bitmap_idx); 400 | size_t pre_mask; 401 | size_t mid_mask; 402 | size_t post_mask; 403 | size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); 404 | bool all_ones = true; 405 | bool any_ones = false; 406 | mi_bitmap_field_t* field = &bitmap[idx]; 407 | size_t prev = mi_atomic_load_relaxed(field++); 408 | if ((prev & pre_mask) != pre_mask) all_ones = false; 409 | if ((prev & pre_mask) != 0) any_ones = true; 410 | while (mid_count-- > 0) { 411 | prev = mi_atomic_load_relaxed(field++); 412 | if ((prev & mid_mask) != mid_mask) all_ones = false; 413 | if ((prev & mid_mask) != 0) any_ones = true; 414 | } 415 | if (post_mask!=0) { 416 | prev = mi_atomic_load_relaxed(field); 417 | if ((prev & post_mask) != post_mask) all_ones = false; 418 | if ((prev & post_mask) != 0) any_ones = true; 419 | } 420 | if (pany_ones != NULL) { *pany_ones = any_ones; } 421 | return all_ones; 422 | } 423 | 424 | bool _mi_bitmap_is_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { 425 | return mi_bitmap_is_claimedx_across(bitmap, bitmap_fields, count, bitmap_idx, NULL); 426 | } 427 | 428 | bool _mi_bitmap_is_any_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { 429 | bool any_ones; 430 | mi_bitmap_is_claimedx_across(bitmap, bitmap_fields, count, bitmap_idx, &any_ones); 431 | return any_ones; 432 | } 433 | -------------------------------------------------------------------------------- /mimalloc/c/src/bitmap.h: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2019-2023 Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | /* ---------------------------------------------------------------------------- 9 | Concurrent bitmap that can set/reset sequences of bits atomically, 10 | represeted as an array of fields where each field is a machine word (`size_t`) 11 | 12 | There are two api's; the standard one cannot have sequences that cross 13 | between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). 14 | (this is used in region allocation) 15 | 16 | The `_across` postfixed functions do allow sequences that can cross over 17 | between the fields. (This is used in arena allocation) 18 | ---------------------------------------------------------------------------- */ 19 | #pragma once 20 | #ifndef MI_BITMAP_H 21 | #define MI_BITMAP_H 22 | 23 | /* ----------------------------------------------------------- 24 | Bitmap definition 25 | ----------------------------------------------------------- */ 26 | 27 | #define MI_BITMAP_FIELD_BITS (8*MI_SIZE_SIZE) 28 | #define MI_BITMAP_FIELD_FULL (~((size_t)0)) // all bits set 29 | 30 | // An atomic bitmap of `size_t` fields 31 | typedef _Atomic(size_t) mi_bitmap_field_t; 32 | typedef mi_bitmap_field_t* mi_bitmap_t; 33 | 34 | // A bitmap index is the index of the bit in a bitmap. 35 | typedef size_t mi_bitmap_index_t; 36 | 37 | // Create a bit index. 38 | static inline mi_bitmap_index_t mi_bitmap_index_create(size_t idx, size_t bitidx) { 39 | mi_assert_internal(bitidx < MI_BITMAP_FIELD_BITS); 40 | return (idx*MI_BITMAP_FIELD_BITS) + bitidx; 41 | } 42 | 43 | // Create a bit index. 44 | static inline mi_bitmap_index_t mi_bitmap_index_create_from_bit(size_t full_bitidx) { 45 | return mi_bitmap_index_create(full_bitidx / MI_BITMAP_FIELD_BITS, full_bitidx % MI_BITMAP_FIELD_BITS); 46 | } 47 | 48 | // Get the field index from a bit index. 49 | static inline size_t mi_bitmap_index_field(mi_bitmap_index_t bitmap_idx) { 50 | return (bitmap_idx / MI_BITMAP_FIELD_BITS); 51 | } 52 | 53 | // Get the bit index in a bitmap field 54 | static inline size_t mi_bitmap_index_bit_in_field(mi_bitmap_index_t bitmap_idx) { 55 | return (bitmap_idx % MI_BITMAP_FIELD_BITS); 56 | } 57 | 58 | // Get the full bit index 59 | static inline size_t mi_bitmap_index_bit(mi_bitmap_index_t bitmap_idx) { 60 | return bitmap_idx; 61 | } 62 | 63 | /* ----------------------------------------------------------- 64 | Claim a bit sequence atomically 65 | ----------------------------------------------------------- */ 66 | 67 | // Try to atomically claim a sequence of `count` bits in a single 68 | // field at `idx` in `bitmap`. Returns `true` on success. 69 | bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx); 70 | 71 | // Starts at idx, and wraps around to search in all `bitmap_fields` fields. 72 | // For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. 73 | bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx); 74 | 75 | // Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled 76 | typedef bool (mi_cdecl *mi_bitmap_pred_fun_t)(mi_bitmap_index_t bitmap_idx, void* pred_arg); 77 | bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_pred_fun_t pred_fun, void* pred_arg, mi_bitmap_index_t* bitmap_idx); 78 | 79 | // Set `count` bits at `bitmap_idx` to 0 atomically 80 | // Returns `true` if all `count` bits were 1 previously. 81 | bool _mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); 82 | 83 | // Try to set `count` bits at `bitmap_idx` from 0 to 1 atomically. 84 | // Returns `true` if successful when all previous `count` bits were 0. 85 | bool _mi_bitmap_try_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); 86 | 87 | // Set `count` bits at `bitmap_idx` to 1 atomically 88 | // Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. 89 | bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero); 90 | 91 | bool _mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); 92 | bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); 93 | 94 | 95 | //-------------------------------------------------------------------------- 96 | // the `_across` functions work on bitmaps where sequences can cross over 97 | // between the fields. This is used in arena allocation 98 | //-------------------------------------------------------------------------- 99 | 100 | // Find `count` bits of zeros and set them to 1 atomically; returns `true` on success. 101 | // Starts at idx, and wraps around to search in all `bitmap_fields` fields. 102 | bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx); 103 | 104 | // Set `count` bits at `bitmap_idx` to 0 atomically 105 | // Returns `true` if all `count` bits were 1 previously. 106 | bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); 107 | 108 | // Set `count` bits at `bitmap_idx` to 1 atomically 109 | // Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. 110 | bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero); 111 | 112 | bool _mi_bitmap_is_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); 113 | bool _mi_bitmap_is_any_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); 114 | 115 | #endif 116 | -------------------------------------------------------------------------------- /mimalloc/c/src/page-queue.c: -------------------------------------------------------------------------------- 1 | /*---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2020, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | /* ----------------------------------------------------------- 9 | Definition of page queues for each block size 10 | ----------------------------------------------------------- */ 11 | 12 | #ifndef MI_IN_PAGE_C 13 | #error "this file should be included from 'page.c'" 14 | #endif 15 | 16 | /* ----------------------------------------------------------- 17 | Minimal alignment in machine words (i.e. `sizeof(void*)`) 18 | ----------------------------------------------------------- */ 19 | 20 | #if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE) 21 | #error "define alignment for more than 4x word size for this platform" 22 | #elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE) 23 | #define MI_ALIGN4W // 4 machine words minimal alignment 24 | #elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE) 25 | #define MI_ALIGN2W // 2 machine words minimal alignment 26 | #else 27 | // ok, default alignment is 1 word 28 | #endif 29 | 30 | 31 | /* ----------------------------------------------------------- 32 | Queue query 33 | ----------------------------------------------------------- */ 34 | 35 | 36 | static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) { 37 | return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+sizeof(uintptr_t))); 38 | } 39 | 40 | static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) { 41 | return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+(2*sizeof(uintptr_t)))); 42 | } 43 | 44 | static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { 45 | return (pq->block_size > MI_MEDIUM_OBJ_SIZE_MAX); 46 | } 47 | 48 | /* ----------------------------------------------------------- 49 | Bins 50 | ----------------------------------------------------------- */ 51 | 52 | // Return the bin for a given field size. 53 | // Returns MI_BIN_HUGE if the size is too large. 54 | // We use `wsize` for the size in "machine word sizes", 55 | // i.e. byte size == `wsize*sizeof(void*)`. 56 | static inline uint8_t mi_bin(size_t size) { 57 | size_t wsize = _mi_wsize_from_size(size); 58 | uint8_t bin; 59 | if (wsize <= 1) { 60 | bin = 1; 61 | } 62 | #if defined(MI_ALIGN4W) 63 | else if (wsize <= 4) { 64 | bin = (uint8_t)((wsize+1)&~1); // round to double word sizes 65 | } 66 | #elif defined(MI_ALIGN2W) 67 | else if (wsize <= 8) { 68 | bin = (uint8_t)((wsize+1)&~1); // round to double word sizes 69 | } 70 | #else 71 | else if (wsize <= 8) { 72 | bin = (uint8_t)wsize; 73 | } 74 | #endif 75 | else if (wsize > MI_MEDIUM_OBJ_WSIZE_MAX) { 76 | bin = MI_BIN_HUGE; 77 | } 78 | else { 79 | #if defined(MI_ALIGN4W) 80 | if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes 81 | #endif 82 | wsize--; 83 | // find the highest bit 84 | uint8_t b = (uint8_t)mi_bsr(wsize); // note: wsize != 0 85 | // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation). 86 | // - adjust with 3 because we use do not round the first 8 sizes 87 | // which each get an exact bin 88 | bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3; 89 | mi_assert_internal(bin < MI_BIN_HUGE); 90 | } 91 | mi_assert_internal(bin > 0 && bin <= MI_BIN_HUGE); 92 | return bin; 93 | } 94 | 95 | 96 | 97 | /* ----------------------------------------------------------- 98 | Queue of pages with free blocks 99 | ----------------------------------------------------------- */ 100 | 101 | uint8_t _mi_bin(size_t size) { 102 | return mi_bin(size); 103 | } 104 | 105 | size_t _mi_bin_size(uint8_t bin) { 106 | return _mi_heap_empty.pages[bin].block_size; 107 | } 108 | 109 | // Good size for allocation 110 | size_t mi_good_size(size_t size) mi_attr_noexcept { 111 | if (size <= MI_MEDIUM_OBJ_SIZE_MAX) { 112 | return _mi_bin_size(mi_bin(size)); 113 | } 114 | else { 115 | return _mi_align_up(size,_mi_os_page_size()); 116 | } 117 | } 118 | 119 | #if (MI_DEBUG>1) 120 | static bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page) { 121 | mi_assert_internal(page != NULL); 122 | mi_page_t* list = queue->first; 123 | while (list != NULL) { 124 | mi_assert_internal(list->next == NULL || list->next->prev == list); 125 | mi_assert_internal(list->prev == NULL || list->prev->next == list); 126 | if (list == page) break; 127 | list = list->next; 128 | } 129 | return (list == page); 130 | } 131 | 132 | #endif 133 | 134 | #if (MI_DEBUG>1) 135 | static bool mi_heap_contains_queue(const mi_heap_t* heap, const mi_page_queue_t* pq) { 136 | return (pq >= &heap->pages[0] && pq <= &heap->pages[MI_BIN_FULL]); 137 | } 138 | #endif 139 | 140 | static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) { 141 | uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : mi_bin(page->xblock_size)); 142 | mi_heap_t* heap = mi_page_heap(page); 143 | mi_assert_internal(heap != NULL && bin <= MI_BIN_FULL); 144 | mi_page_queue_t* pq = &heap->pages[bin]; 145 | mi_assert_internal(bin >= MI_BIN_HUGE || page->xblock_size == pq->block_size); 146 | mi_assert_expensive(mi_page_queue_contains(pq, page)); 147 | return pq; 148 | } 149 | 150 | static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) { 151 | uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : mi_bin(page->xblock_size)); 152 | mi_assert_internal(bin <= MI_BIN_FULL); 153 | mi_page_queue_t* pq = &heap->pages[bin]; 154 | mi_assert_internal(mi_page_is_in_full(page) || page->xblock_size == pq->block_size); 155 | return pq; 156 | } 157 | 158 | // The current small page array is for efficiency and for each 159 | // small size (up to 256) it points directly to the page for that 160 | // size without having to compute the bin. This means when the 161 | // current free page queue is updated for a small bin, we need to update a 162 | // range of entries in `_mi_page_small_free`. 163 | static inline void mi_heap_queue_first_update(mi_heap_t* heap, const mi_page_queue_t* pq) { 164 | mi_assert_internal(mi_heap_contains_queue(heap,pq)); 165 | size_t size = pq->block_size; 166 | if (size > MI_SMALL_SIZE_MAX) return; 167 | 168 | mi_page_t* page = pq->first; 169 | if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty; 170 | 171 | // find index in the right direct page array 172 | size_t start; 173 | size_t idx = _mi_wsize_from_size(size); 174 | mi_page_t** pages_free = heap->pages_free_direct; 175 | 176 | if (pages_free[idx] == page) return; // already set 177 | 178 | // find start slot 179 | if (idx<=1) { 180 | start = 0; 181 | } 182 | else { 183 | // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped 184 | uint8_t bin = mi_bin(size); 185 | const mi_page_queue_t* prev = pq - 1; 186 | while( bin == mi_bin(prev->block_size) && prev > &heap->pages[0]) { 187 | prev--; 188 | } 189 | start = 1 + _mi_wsize_from_size(prev->block_size); 190 | if (start > idx) start = idx; 191 | } 192 | 193 | // set size range to the right page 194 | mi_assert(start <= idx); 195 | for (size_t sz = start; sz <= idx; sz++) { 196 | pages_free[sz] = page; 197 | } 198 | } 199 | 200 | /* 201 | static bool mi_page_queue_is_empty(mi_page_queue_t* queue) { 202 | return (queue->first == NULL); 203 | } 204 | */ 205 | 206 | static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { 207 | mi_assert_internal(page != NULL); 208 | mi_assert_expensive(mi_page_queue_contains(queue, page)); 209 | mi_assert_internal(page->xblock_size == queue->block_size || (page->xblock_size > MI_MEDIUM_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); 210 | mi_heap_t* heap = mi_page_heap(page); 211 | 212 | if (page->prev != NULL) page->prev->next = page->next; 213 | if (page->next != NULL) page->next->prev = page->prev; 214 | if (page == queue->last) queue->last = page->prev; 215 | if (page == queue->first) { 216 | queue->first = page->next; 217 | // update first 218 | mi_assert_internal(mi_heap_contains_queue(heap, queue)); 219 | mi_heap_queue_first_update(heap,queue); 220 | } 221 | heap->page_count--; 222 | page->next = NULL; 223 | page->prev = NULL; 224 | // mi_atomic_store_ptr_release(mi_atomic_cast(void*, &page->heap), NULL); 225 | mi_page_set_in_full(page,false); 226 | } 227 | 228 | 229 | static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_t* page) { 230 | mi_assert_internal(mi_page_heap(page) == heap); 231 | mi_assert_internal(!mi_page_queue_contains(queue, page)); 232 | #if MI_HUGE_PAGE_ABANDON 233 | mi_assert_internal(_mi_page_segment(page)->kind != MI_SEGMENT_HUGE); 234 | #endif 235 | mi_assert_internal(page->xblock_size == queue->block_size || 236 | (page->xblock_size > MI_MEDIUM_OBJ_SIZE_MAX) || 237 | (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); 238 | 239 | mi_page_set_in_full(page, mi_page_queue_is_full(queue)); 240 | // mi_atomic_store_ptr_release(mi_atomic_cast(void*, &page->heap), heap); 241 | page->next = queue->first; 242 | page->prev = NULL; 243 | if (queue->first != NULL) { 244 | mi_assert_internal(queue->first->prev == NULL); 245 | queue->first->prev = page; 246 | queue->first = page; 247 | } 248 | else { 249 | queue->first = queue->last = page; 250 | } 251 | 252 | // update direct 253 | mi_heap_queue_first_update(heap, queue); 254 | heap->page_count++; 255 | } 256 | 257 | 258 | static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { 259 | mi_assert_internal(page != NULL); 260 | mi_assert_expensive(mi_page_queue_contains(from, page)); 261 | mi_assert_expensive(!mi_page_queue_contains(to, page)); 262 | 263 | mi_assert_internal((page->xblock_size == to->block_size && page->xblock_size == from->block_size) || 264 | (page->xblock_size == to->block_size && mi_page_queue_is_full(from)) || 265 | (page->xblock_size == from->block_size && mi_page_queue_is_full(to)) || 266 | (page->xblock_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(to)) || 267 | (page->xblock_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_full(to))); 268 | 269 | mi_heap_t* heap = mi_page_heap(page); 270 | if (page->prev != NULL) page->prev->next = page->next; 271 | if (page->next != NULL) page->next->prev = page->prev; 272 | if (page == from->last) from->last = page->prev; 273 | if (page == from->first) { 274 | from->first = page->next; 275 | // update first 276 | mi_assert_internal(mi_heap_contains_queue(heap, from)); 277 | mi_heap_queue_first_update(heap, from); 278 | } 279 | 280 | page->prev = to->last; 281 | page->next = NULL; 282 | if (to->last != NULL) { 283 | mi_assert_internal(heap == mi_page_heap(to->last)); 284 | to->last->next = page; 285 | to->last = page; 286 | } 287 | else { 288 | to->first = page; 289 | to->last = page; 290 | mi_heap_queue_first_update(heap, to); 291 | } 292 | 293 | mi_page_set_in_full(page, mi_page_queue_is_full(to)); 294 | } 295 | 296 | // Only called from `mi_heap_absorb`. 297 | size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append) { 298 | mi_assert_internal(mi_heap_contains_queue(heap,pq)); 299 | mi_assert_internal(pq->block_size == append->block_size); 300 | 301 | if (append->first==NULL) return 0; 302 | 303 | // set append pages to new heap and count 304 | size_t count = 0; 305 | for (mi_page_t* page = append->first; page != NULL; page = page->next) { 306 | // inline `mi_page_set_heap` to avoid wrong assertion during absorption; 307 | // in this case it is ok to be delayed freeing since both "to" and "from" heap are still alive. 308 | mi_atomic_store_release(&page->xheap, (uintptr_t)heap); 309 | // set the flag to delayed free (not overriding NEVER_DELAYED_FREE) which has as a 310 | // side effect that it spins until any DELAYED_FREEING is finished. This ensures 311 | // that after appending only the new heap will be used for delayed free operations. 312 | _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, false); 313 | count++; 314 | } 315 | 316 | if (pq->last==NULL) { 317 | // take over afresh 318 | mi_assert_internal(pq->first==NULL); 319 | pq->first = append->first; 320 | pq->last = append->last; 321 | mi_heap_queue_first_update(heap, pq); 322 | } 323 | else { 324 | // append to end 325 | mi_assert_internal(pq->last!=NULL); 326 | mi_assert_internal(append->first!=NULL); 327 | pq->last->next = append->first; 328 | append->first->prev = pq->last; 329 | pq->last = append->last; 330 | } 331 | return count; 332 | } 333 | -------------------------------------------------------------------------------- /mimalloc/c/src/prim/osx/alloc-override-zone.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2022, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | #include "mimalloc.h" 9 | #include "mimalloc/internal.h" 10 | 11 | #if defined(MI_MALLOC_OVERRIDE) 12 | 13 | #if !defined(__APPLE__) 14 | #error "this file should only be included on macOS" 15 | #endif 16 | 17 | /* ------------------------------------------------------ 18 | Override system malloc on macOS 19 | This is done through the malloc zone interface. 20 | It seems to be most robust in combination with interposing 21 | though or otherwise we may get zone errors as there are could 22 | be allocations done by the time we take over the 23 | zone. 24 | ------------------------------------------------------ */ 25 | 26 | #include 27 | #include 28 | #include // memset 29 | #include 30 | 31 | #ifdef __cplusplus 32 | extern "C" { 33 | #endif 34 | 35 | #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) 36 | // only available from OSX 10.6 37 | extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import)); 38 | #endif 39 | 40 | /* ------------------------------------------------------ 41 | malloc zone members 42 | ------------------------------------------------------ */ 43 | 44 | static size_t zone_size(malloc_zone_t* zone, const void* p) { 45 | MI_UNUSED(zone); 46 | if (!mi_is_in_heap_region(p)){ return 0; } // not our pointer, bail out 47 | return mi_usable_size(p); 48 | } 49 | 50 | static void* zone_malloc(malloc_zone_t* zone, size_t size) { 51 | MI_UNUSED(zone); 52 | return mi_malloc(size); 53 | } 54 | 55 | static void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) { 56 | MI_UNUSED(zone); 57 | return mi_calloc(count, size); 58 | } 59 | 60 | static void* zone_valloc(malloc_zone_t* zone, size_t size) { 61 | MI_UNUSED(zone); 62 | return mi_malloc_aligned(size, _mi_os_page_size()); 63 | } 64 | 65 | static void zone_free(malloc_zone_t* zone, void* p) { 66 | MI_UNUSED(zone); 67 | mi_cfree(p); 68 | } 69 | 70 | static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) { 71 | MI_UNUSED(zone); 72 | return mi_realloc(p, newsize); 73 | } 74 | 75 | static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) { 76 | MI_UNUSED(zone); 77 | return mi_malloc_aligned(size,alignment); 78 | } 79 | 80 | static void zone_destroy(malloc_zone_t* zone) { 81 | MI_UNUSED(zone); 82 | // todo: ignore for now? 83 | } 84 | 85 | static unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) { 86 | size_t i; 87 | for (i = 0; i < count; i++) { 88 | ps[i] = zone_malloc(zone, size); 89 | if (ps[i] == NULL) break; 90 | } 91 | return i; 92 | } 93 | 94 | static void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) { 95 | for(size_t i = 0; i < count; i++) { 96 | zone_free(zone, ps[i]); 97 | ps[i] = NULL; 98 | } 99 | } 100 | 101 | static size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) { 102 | MI_UNUSED(zone); MI_UNUSED(size); 103 | mi_collect(false); 104 | return 0; 105 | } 106 | 107 | static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) { 108 | MI_UNUSED(size); 109 | zone_free(zone,p); 110 | } 111 | 112 | static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) { 113 | MI_UNUSED(zone); 114 | return mi_is_in_heap_region(p); 115 | } 116 | 117 | 118 | /* ------------------------------------------------------ 119 | Introspection members 120 | ------------------------------------------------------ */ 121 | 122 | static kern_return_t intro_enumerator(task_t task, void* p, 123 | unsigned type_mask, vm_address_t zone_address, 124 | memory_reader_t reader, 125 | vm_range_recorder_t recorder) 126 | { 127 | // todo: enumerate all memory 128 | MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address); 129 | MI_UNUSED(reader); MI_UNUSED(recorder); 130 | return KERN_SUCCESS; 131 | } 132 | 133 | static size_t intro_good_size(malloc_zone_t* zone, size_t size) { 134 | MI_UNUSED(zone); 135 | return mi_good_size(size); 136 | } 137 | 138 | static boolean_t intro_check(malloc_zone_t* zone) { 139 | MI_UNUSED(zone); 140 | return true; 141 | } 142 | 143 | static void intro_print(malloc_zone_t* zone, boolean_t verbose) { 144 | MI_UNUSED(zone); MI_UNUSED(verbose); 145 | mi_stats_print(NULL); 146 | } 147 | 148 | static void intro_log(malloc_zone_t* zone, void* p) { 149 | MI_UNUSED(zone); MI_UNUSED(p); 150 | // todo? 151 | } 152 | 153 | static void intro_force_lock(malloc_zone_t* zone) { 154 | MI_UNUSED(zone); 155 | // todo? 156 | } 157 | 158 | static void intro_force_unlock(malloc_zone_t* zone) { 159 | MI_UNUSED(zone); 160 | // todo? 161 | } 162 | 163 | static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) { 164 | MI_UNUSED(zone); 165 | // todo... 166 | stats->blocks_in_use = 0; 167 | stats->size_in_use = 0; 168 | stats->max_size_in_use = 0; 169 | stats->size_allocated = 0; 170 | } 171 | 172 | static boolean_t intro_zone_locked(malloc_zone_t* zone) { 173 | MI_UNUSED(zone); 174 | return false; 175 | } 176 | 177 | 178 | /* ------------------------------------------------------ 179 | At process start, override the default allocator 180 | ------------------------------------------------------ */ 181 | 182 | #if defined(__GNUC__) && !defined(__clang__) 183 | #pragma GCC diagnostic ignored "-Wmissing-field-initializers" 184 | #endif 185 | 186 | #if defined(__clang__) 187 | #pragma clang diagnostic ignored "-Wc99-extensions" 188 | #endif 189 | 190 | static malloc_introspection_t mi_introspect = { 191 | .enumerator = &intro_enumerator, 192 | .good_size = &intro_good_size, 193 | .check = &intro_check, 194 | .print = &intro_print, 195 | .log = &intro_log, 196 | .force_lock = &intro_force_lock, 197 | .force_unlock = &intro_force_unlock, 198 | #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__) 199 | .statistics = &intro_statistics, 200 | .zone_locked = &intro_zone_locked, 201 | #endif 202 | }; 203 | 204 | static malloc_zone_t mi_malloc_zone = { 205 | // note: even with designators, the order is important for C++ compilation 206 | //.reserved1 = NULL, 207 | //.reserved2 = NULL, 208 | .size = &zone_size, 209 | .malloc = &zone_malloc, 210 | .calloc = &zone_calloc, 211 | .valloc = &zone_valloc, 212 | .free = &zone_free, 213 | .realloc = &zone_realloc, 214 | .destroy = &zone_destroy, 215 | .zone_name = "mimalloc", 216 | .batch_malloc = &zone_batch_malloc, 217 | .batch_free = &zone_batch_free, 218 | .introspect = &mi_introspect, 219 | #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__) 220 | #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14) 221 | .version = 10, 222 | #else 223 | .version = 9, 224 | #endif 225 | // switch to version 9+ on OSX 10.6 to support memalign. 226 | .memalign = &zone_memalign, 227 | .free_definite_size = &zone_free_definite_size, 228 | .pressure_relief = &zone_pressure_relief, 229 | #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14) 230 | .claimed_address = &zone_claimed_address, 231 | #endif 232 | #else 233 | .version = 4, 234 | #endif 235 | }; 236 | 237 | #ifdef __cplusplus 238 | } 239 | #endif 240 | 241 | 242 | #if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT) 243 | 244 | // ------------------------------------------------------ 245 | // Override malloc_xxx and malloc_zone_xxx api's to use only 246 | // our mimalloc zone. Since even the loader uses malloc 247 | // on macOS, this ensures that all allocations go through 248 | // mimalloc (as all calls are interposed). 249 | // The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`, 250 | // Here, we also override macOS specific API's like 251 | // `malloc_zone_calloc` etc. see 252 | // ------------------------------------------------------ 253 | 254 | static inline malloc_zone_t* mi_get_default_zone(void) 255 | { 256 | static bool init; 257 | if mi_unlikely(!init) { 258 | init = true; 259 | malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see ) 260 | } 261 | return &mi_malloc_zone; 262 | } 263 | 264 | mi_decl_externc int malloc_jumpstart(uintptr_t cookie); 265 | mi_decl_externc void _malloc_fork_prepare(void); 266 | mi_decl_externc void _malloc_fork_parent(void); 267 | mi_decl_externc void _malloc_fork_child(void); 268 | 269 | 270 | static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) { 271 | MI_UNUSED(size); MI_UNUSED(flags); 272 | return mi_get_default_zone(); 273 | } 274 | 275 | static malloc_zone_t* mi_malloc_default_zone (void) { 276 | return mi_get_default_zone(); 277 | } 278 | 279 | static malloc_zone_t* mi_malloc_default_purgeable_zone(void) { 280 | return mi_get_default_zone(); 281 | } 282 | 283 | static void mi_malloc_destroy_zone(malloc_zone_t* zone) { 284 | MI_UNUSED(zone); 285 | // nothing. 286 | } 287 | 288 | static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) { 289 | MI_UNUSED(task); MI_UNUSED(mr); 290 | if (addresses != NULL) *addresses = NULL; 291 | if (count != NULL) *count = 0; 292 | return KERN_SUCCESS; 293 | } 294 | 295 | static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) { 296 | return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name); 297 | } 298 | 299 | static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) { 300 | MI_UNUSED(zone); MI_UNUSED(name); 301 | } 302 | 303 | static int mi_malloc_jumpstart(uintptr_t cookie) { 304 | MI_UNUSED(cookie); 305 | return 1; // or 0 for no error? 306 | } 307 | 308 | static void mi__malloc_fork_prepare(void) { 309 | // nothing 310 | } 311 | static void mi__malloc_fork_parent(void) { 312 | // nothing 313 | } 314 | static void mi__malloc_fork_child(void) { 315 | // nothing 316 | } 317 | 318 | static void mi_malloc_printf(const char* fmt, ...) { 319 | MI_UNUSED(fmt); 320 | } 321 | 322 | static bool zone_check(malloc_zone_t* zone) { 323 | MI_UNUSED(zone); 324 | return true; 325 | } 326 | 327 | static malloc_zone_t* zone_from_ptr(const void* p) { 328 | MI_UNUSED(p); 329 | return mi_get_default_zone(); 330 | } 331 | 332 | static void zone_log(malloc_zone_t* zone, void* p) { 333 | MI_UNUSED(zone); MI_UNUSED(p); 334 | } 335 | 336 | static void zone_print(malloc_zone_t* zone, bool b) { 337 | MI_UNUSED(zone); MI_UNUSED(b); 338 | } 339 | 340 | static void zone_print_ptr_info(void* p) { 341 | MI_UNUSED(p); 342 | } 343 | 344 | static void zone_register(malloc_zone_t* zone) { 345 | MI_UNUSED(zone); 346 | } 347 | 348 | static void zone_unregister(malloc_zone_t* zone) { 349 | MI_UNUSED(zone); 350 | } 351 | 352 | // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` 353 | // See: 354 | struct mi_interpose_s { 355 | const void* replacement; 356 | const void* target; 357 | }; 358 | #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } 359 | #define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) 360 | #define MI_INTERPOSE_ZONE(fun) MI_INTERPOSE_FUN(malloc_##fun,fun) 361 | __attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] __attribute__((section("__DATA, __interpose"))) = 362 | { 363 | 364 | MI_INTERPOSE_MI(malloc_create_zone), 365 | MI_INTERPOSE_MI(malloc_default_purgeable_zone), 366 | MI_INTERPOSE_MI(malloc_default_zone), 367 | MI_INTERPOSE_MI(malloc_destroy_zone), 368 | MI_INTERPOSE_MI(malloc_get_all_zones), 369 | MI_INTERPOSE_MI(malloc_get_zone_name), 370 | MI_INTERPOSE_MI(malloc_jumpstart), 371 | MI_INTERPOSE_MI(malloc_printf), 372 | MI_INTERPOSE_MI(malloc_set_zone_name), 373 | MI_INTERPOSE_MI(_malloc_fork_child), 374 | MI_INTERPOSE_MI(_malloc_fork_parent), 375 | MI_INTERPOSE_MI(_malloc_fork_prepare), 376 | 377 | MI_INTERPOSE_ZONE(zone_batch_free), 378 | MI_INTERPOSE_ZONE(zone_batch_malloc), 379 | MI_INTERPOSE_ZONE(zone_calloc), 380 | MI_INTERPOSE_ZONE(zone_check), 381 | MI_INTERPOSE_ZONE(zone_free), 382 | MI_INTERPOSE_ZONE(zone_from_ptr), 383 | MI_INTERPOSE_ZONE(zone_log), 384 | MI_INTERPOSE_ZONE(zone_malloc), 385 | MI_INTERPOSE_ZONE(zone_memalign), 386 | MI_INTERPOSE_ZONE(zone_print), 387 | MI_INTERPOSE_ZONE(zone_print_ptr_info), 388 | MI_INTERPOSE_ZONE(zone_realloc), 389 | MI_INTERPOSE_ZONE(zone_register), 390 | MI_INTERPOSE_ZONE(zone_unregister), 391 | MI_INTERPOSE_ZONE(zone_valloc) 392 | }; 393 | 394 | 395 | #else 396 | 397 | // ------------------------------------------------------ 398 | // hook into the zone api's without interposing 399 | // This is the official way of adding an allocator but 400 | // it seems less robust than using interpose. 401 | // ------------------------------------------------------ 402 | 403 | static inline malloc_zone_t* mi_get_default_zone(void) 404 | { 405 | // The first returned zone is the real default 406 | malloc_zone_t** zones = NULL; 407 | unsigned count = 0; 408 | kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count); 409 | if (ret == KERN_SUCCESS && count > 0) { 410 | return zones[0]; 411 | } 412 | else { 413 | // fallback 414 | return malloc_default_zone(); 415 | } 416 | } 417 | 418 | #if defined(__clang__) 419 | __attribute__((constructor(0))) 420 | #else 421 | __attribute__((constructor)) // seems not supported by g++-11 on the M1 422 | #endif 423 | static void _mi_macos_override_malloc(void) { 424 | malloc_zone_t* purgeable_zone = NULL; 425 | 426 | #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) 427 | // force the purgeable zone to exist to avoid strange bugs 428 | if (malloc_default_purgeable_zone) { 429 | purgeable_zone = malloc_default_purgeable_zone(); 430 | } 431 | #endif 432 | 433 | // Register our zone. 434 | // thomcc: I think this is still needed to put us in the zone list. 435 | malloc_zone_register(&mi_malloc_zone); 436 | // Unregister the default zone, this makes our zone the new default 437 | // as that was the last registered. 438 | malloc_zone_t *default_zone = mi_get_default_zone(); 439 | // thomcc: Unsure if the next test is *always* false or just false in the 440 | // cases I've tried. I'm also unsure if the code inside is needed. at all 441 | if (default_zone != &mi_malloc_zone) { 442 | malloc_zone_unregister(default_zone); 443 | 444 | // Reregister the default zone so free and realloc in that zone keep working. 445 | malloc_zone_register(default_zone); 446 | } 447 | 448 | // Unregister, and re-register the purgeable_zone to avoid bugs if it occurs 449 | // earlier than the default zone. 450 | if (purgeable_zone != NULL) { 451 | malloc_zone_unregister(purgeable_zone); 452 | malloc_zone_register(purgeable_zone); 453 | } 454 | 455 | } 456 | #endif // MI_OSX_INTERPOSE 457 | 458 | #endif // MI_MALLOC_OVERRIDE 459 | -------------------------------------------------------------------------------- /mimalloc/c/src/prim/osx/prim.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2023, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | // We use the unix/prim.c with the mmap API on macOSX 9 | #include "../unix/prim.c" 10 | -------------------------------------------------------------------------------- /mimalloc/c/src/prim/prim.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2023, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | // Select the implementation of the primitives 9 | // depending on the OS. 10 | 11 | #if defined(_WIN32) 12 | #include "windows/prim.c" // VirtualAlloc (Windows) 13 | 14 | #elif defined(__APPLE__) 15 | #include "osx/prim.c" // macOSX (actually defers to mmap in unix/prim.c) 16 | 17 | #elif defined(__wasi__) 18 | #define MI_USE_SBRK 19 | #include "wasi/prim.c" // memory-grow or sbrk (Wasm) 20 | 21 | #else 22 | #include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /mimalloc/c/src/prim/readme.md: -------------------------------------------------------------------------------- 1 | ## Portability Primitives 2 | 3 | This is the portability layer where all primitives needed from the OS are defined. 4 | 5 | - `include/mimalloc/prim.h`: primitive portability API definition. 6 | - `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform 7 | (and on macOS, `osx/prim.c` defers to `unix/prim.c`). 8 | 9 | Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's. -------------------------------------------------------------------------------- /mimalloc/c/src/prim/wasi/prim.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2023, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | // This file is included in `src/prim/prim.c` 9 | 10 | #include "mimalloc.h" 11 | #include "mimalloc/internal.h" 12 | #include "mimalloc/atomic.h" 13 | #include "mimalloc/prim.h" 14 | 15 | //--------------------------------------------- 16 | // Initialize 17 | //--------------------------------------------- 18 | 19 | void _mi_prim_mem_init( mi_os_mem_config_t* config ) { 20 | config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB 21 | config->alloc_granularity = 16; 22 | config->has_overcommit = false; 23 | config->must_free_whole = true; 24 | config->has_virtual_reserve = false; 25 | } 26 | 27 | //--------------------------------------------- 28 | // Free 29 | //--------------------------------------------- 30 | 31 | int _mi_prim_free(void* addr, size_t size ) { 32 | MI_UNUSED(addr); MI_UNUSED(size); 33 | // wasi heap cannot be shrunk 34 | return 0; 35 | } 36 | 37 | 38 | //--------------------------------------------- 39 | // Allocation: sbrk or memory_grow 40 | //--------------------------------------------- 41 | 42 | #if defined(MI_USE_SBRK) 43 | static void* mi_memory_grow( size_t size ) { 44 | void* p = sbrk(size); 45 | if (p == (void*)(-1)) return NULL; 46 | #if !defined(__wasi__) // on wasi this is always zero initialized already (?) 47 | memset(p,0,size); 48 | #endif 49 | return p; 50 | } 51 | #elif defined(__wasi__) 52 | static void* mi_memory_grow( size_t size ) { 53 | size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size())) 54 | : __builtin_wasm_memory_size(0)); 55 | if (base == SIZE_MAX) return NULL; 56 | return (void*)(base * _mi_os_page_size()); 57 | } 58 | #endif 59 | 60 | #if defined(MI_USE_PTHREADS) 61 | static pthread_mutex_t mi_heap_grow_mutex = PTHREAD_MUTEX_INITIALIZER; 62 | #endif 63 | 64 | static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { 65 | void* p = NULL; 66 | if (try_alignment <= 1) { 67 | // `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now) 68 | #if defined(MI_USE_PTHREADS) 69 | pthread_mutex_lock(&mi_heap_grow_mutex); 70 | #endif 71 | p = mi_memory_grow(size); 72 | #if defined(MI_USE_PTHREADS) 73 | pthread_mutex_unlock(&mi_heap_grow_mutex); 74 | #endif 75 | } 76 | else { 77 | void* base = NULL; 78 | size_t alloc_size = 0; 79 | // to allocate aligned use a lock to try to avoid thread interaction 80 | // between getting the current size and actual allocation 81 | // (also, `sbrk` is not thread safe in general) 82 | #if defined(MI_USE_PTHREADS) 83 | pthread_mutex_lock(&mi_heap_grow_mutex); 84 | #endif 85 | { 86 | void* current = mi_memory_grow(0); // get current size 87 | if (current != NULL) { 88 | void* aligned_current = mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space 89 | alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size()); 90 | base = mi_memory_grow(alloc_size); 91 | } 92 | } 93 | #if defined(MI_USE_PTHREADS) 94 | pthread_mutex_unlock(&mi_heap_grow_mutex); 95 | #endif 96 | if (base != NULL) { 97 | p = mi_align_up_ptr(base, try_alignment); 98 | if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) { 99 | // another thread used wasm_memory_grow/sbrk in-between and we do not have enough 100 | // space after alignment. Give up (and waste the space as we cannot shrink :-( ) 101 | // (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align) 102 | p = NULL; 103 | } 104 | } 105 | } 106 | /* 107 | if (p == NULL) { 108 | _mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment); 109 | errno = ENOMEM; 110 | return NULL; 111 | } 112 | */ 113 | mi_assert_internal( p == NULL || try_alignment == 0 || (uintptr_t)p % try_alignment == 0 ); 114 | return p; 115 | } 116 | 117 | // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. 118 | int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { 119 | MI_UNUSED(allow_large); MI_UNUSED(commit); 120 | *is_large = false; 121 | *is_zero = false; 122 | *addr = mi_prim_mem_grow(size, try_alignment); 123 | return (*addr != NULL ? 0 : ENOMEM); 124 | } 125 | 126 | 127 | //--------------------------------------------- 128 | // Commit/Reset/Protect 129 | //--------------------------------------------- 130 | 131 | int _mi_prim_commit(void* addr, size_t size, bool* is_zero) { 132 | MI_UNUSED(addr); MI_UNUSED(size); 133 | *is_zero = false; 134 | return 0; 135 | } 136 | 137 | int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) { 138 | MI_UNUSED(addr); MI_UNUSED(size); 139 | *needs_recommit = false; 140 | return 0; 141 | } 142 | 143 | int _mi_prim_reset(void* addr, size_t size) { 144 | MI_UNUSED(addr); MI_UNUSED(size); 145 | return 0; 146 | } 147 | 148 | int _mi_prim_protect(void* addr, size_t size, bool protect) { 149 | MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect); 150 | return 0; 151 | } 152 | 153 | 154 | //--------------------------------------------- 155 | // Huge pages and NUMA nodes 156 | //--------------------------------------------- 157 | 158 | int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) { 159 | MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node); 160 | *is_zero = true; 161 | *addr = NULL; 162 | return ENOSYS; 163 | } 164 | 165 | size_t _mi_prim_numa_node(void) { 166 | return 0; 167 | } 168 | 169 | size_t _mi_prim_numa_node_count(void) { 170 | return 1; 171 | } 172 | 173 | 174 | //---------------------------------------------------------------- 175 | // Clock 176 | //---------------------------------------------------------------- 177 | 178 | #include 179 | 180 | #if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) 181 | 182 | mi_msecs_t _mi_prim_clock_now(void) { 183 | struct timespec t; 184 | #ifdef CLOCK_MONOTONIC 185 | clock_gettime(CLOCK_MONOTONIC, &t); 186 | #else 187 | clock_gettime(CLOCK_REALTIME, &t); 188 | #endif 189 | return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000); 190 | } 191 | 192 | #else 193 | 194 | // low resolution timer 195 | mi_msecs_t _mi_prim_clock_now(void) { 196 | #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) 197 | return (mi_msecs_t)clock(); 198 | #elif (CLOCKS_PER_SEC < 1000) 199 | return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); 200 | #else 201 | return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); 202 | #endif 203 | } 204 | 205 | #endif 206 | 207 | 208 | //---------------------------------------------------------------- 209 | // Process info 210 | //---------------------------------------------------------------- 211 | 212 | void _mi_prim_process_info(mi_process_info_t* pinfo) 213 | { 214 | // use defaults 215 | MI_UNUSED(pinfo); 216 | } 217 | 218 | 219 | //---------------------------------------------------------------- 220 | // Output 221 | //---------------------------------------------------------------- 222 | 223 | void _mi_prim_out_stderr( const char* msg ) { 224 | fputs(msg,stderr); 225 | } 226 | 227 | 228 | //---------------------------------------------------------------- 229 | // Environment 230 | //---------------------------------------------------------------- 231 | 232 | bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { 233 | // cannot call getenv() when still initializing the C runtime. 234 | if (_mi_preloading()) return false; 235 | const char* s = getenv(name); 236 | if (s == NULL) { 237 | // we check the upper case name too. 238 | char buf[64+1]; 239 | size_t len = _mi_strnlen(name,sizeof(buf)-1); 240 | for (size_t i = 0; i < len; i++) { 241 | buf[i] = _mi_toupper(name[i]); 242 | } 243 | buf[len] = 0; 244 | s = getenv(buf); 245 | } 246 | if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false; 247 | _mi_strlcpy(result, s, result_size); 248 | return true; 249 | } 250 | 251 | 252 | //---------------------------------------------------------------- 253 | // Random 254 | //---------------------------------------------------------------- 255 | 256 | bool _mi_prim_random_buf(void* buf, size_t buf_len) { 257 | return false; 258 | } 259 | 260 | 261 | //---------------------------------------------------------------- 262 | // Thread init/done 263 | //---------------------------------------------------------------- 264 | 265 | void _mi_prim_thread_init_auto_done(void) { 266 | // nothing 267 | } 268 | 269 | void _mi_prim_thread_done_auto_done(void) { 270 | // nothing 271 | } 272 | 273 | void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { 274 | MI_UNUSED(heap); 275 | } 276 | -------------------------------------------------------------------------------- /mimalloc/c/src/prim/windows/etw-mimalloc.wprp: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /mimalloc/c/src/prim/windows/etw.man: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jakubtomsu/odin-mimalloc/46ccf1291634fe3dda762327dce06e99e3eb0fe7/mimalloc/c/src/prim/windows/etw.man -------------------------------------------------------------------------------- /mimalloc/c/src/prim/windows/readme.md: -------------------------------------------------------------------------------- 1 | ## Primitives: 2 | 3 | - `prim.c` contains Windows primitives for OS allocation. 4 | 5 | ## Event Tracing for Windows (ETW) 6 | 7 | - `etw.h` is generated from `etw.man` which contains the manifest for mimalloc events. 8 | (100 is an allocation, 101 is for a free) 9 | 10 | - `etw-mimalloc.wprp` is a profile for the Windows Performance Recorder (WPR). 11 | In an admin prompt, you can use: 12 | ``` 13 | > wpr -start src\prim\windows\etw-mimalloc.wprp -filemode 14 | > 15 | > wpr -stop test.etl 16 | ``` 17 | and then open `test.etl` in the Windows Performance Analyzer (WPA). -------------------------------------------------------------------------------- /mimalloc/c/src/random.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2019-2021, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #include "mimalloc.h" 8 | #include "mimalloc/internal.h" 9 | #include "mimalloc/prim.h" // _mi_prim_random_buf 10 | #include // memset 11 | 12 | /* ---------------------------------------------------------------------------- 13 | We use our own PRNG to keep predictable performance of random number generation 14 | and to avoid implementations that use a lock. We only use the OS provided 15 | random source to initialize the initial seeds. Since we do not need ultimate 16 | performance but we do rely on the security (for secret cookies in secure mode) 17 | we use a cryptographically secure generator (chacha20). 18 | -----------------------------------------------------------------------------*/ 19 | 20 | #define MI_CHACHA_ROUNDS (20) // perhaps use 12 for better performance? 21 | 22 | 23 | /* ---------------------------------------------------------------------------- 24 | Chacha20 implementation as the original algorithm with a 64-bit nonce 25 | and counter: https://en.wikipedia.org/wiki/Salsa20 26 | The input matrix has sixteen 32-bit values: 27 | Position 0 to 3: constant key 28 | Position 4 to 11: the key 29 | Position 12 to 13: the counter. 30 | Position 14 to 15: the nonce. 31 | 32 | The implementation uses regular C code which compiles very well on modern compilers. 33 | (gcc x64 has no register spills, and clang 6+ uses SSE instructions) 34 | -----------------------------------------------------------------------------*/ 35 | 36 | static inline uint32_t rotl(uint32_t x, uint32_t shift) { 37 | return (x << shift) | (x >> (32 - shift)); 38 | } 39 | 40 | static inline void qround(uint32_t x[16], size_t a, size_t b, size_t c, size_t d) { 41 | x[a] += x[b]; x[d] = rotl(x[d] ^ x[a], 16); 42 | x[c] += x[d]; x[b] = rotl(x[b] ^ x[c], 12); 43 | x[a] += x[b]; x[d] = rotl(x[d] ^ x[a], 8); 44 | x[c] += x[d]; x[b] = rotl(x[b] ^ x[c], 7); 45 | } 46 | 47 | static void chacha_block(mi_random_ctx_t* ctx) 48 | { 49 | // scramble into `x` 50 | uint32_t x[16]; 51 | for (size_t i = 0; i < 16; i++) { 52 | x[i] = ctx->input[i]; 53 | } 54 | for (size_t i = 0; i < MI_CHACHA_ROUNDS; i += 2) { 55 | qround(x, 0, 4, 8, 12); 56 | qround(x, 1, 5, 9, 13); 57 | qround(x, 2, 6, 10, 14); 58 | qround(x, 3, 7, 11, 15); 59 | qround(x, 0, 5, 10, 15); 60 | qround(x, 1, 6, 11, 12); 61 | qround(x, 2, 7, 8, 13); 62 | qround(x, 3, 4, 9, 14); 63 | } 64 | 65 | // add scrambled data to the initial state 66 | for (size_t i = 0; i < 16; i++) { 67 | ctx->output[i] = x[i] + ctx->input[i]; 68 | } 69 | ctx->output_available = 16; 70 | 71 | // increment the counter for the next round 72 | ctx->input[12] += 1; 73 | if (ctx->input[12] == 0) { 74 | ctx->input[13] += 1; 75 | if (ctx->input[13] == 0) { // and keep increasing into the nonce 76 | ctx->input[14] += 1; 77 | } 78 | } 79 | } 80 | 81 | static uint32_t chacha_next32(mi_random_ctx_t* ctx) { 82 | if (ctx->output_available <= 0) { 83 | chacha_block(ctx); 84 | ctx->output_available = 16; // (assign again to suppress static analysis warning) 85 | } 86 | const uint32_t x = ctx->output[16 - ctx->output_available]; 87 | ctx->output[16 - ctx->output_available] = 0; // reset once the data is handed out 88 | ctx->output_available--; 89 | return x; 90 | } 91 | 92 | static inline uint32_t read32(const uint8_t* p, size_t idx32) { 93 | const size_t i = 4*idx32; 94 | return ((uint32_t)p[i+0] | (uint32_t)p[i+1] << 8 | (uint32_t)p[i+2] << 16 | (uint32_t)p[i+3] << 24); 95 | } 96 | 97 | static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t nonce) 98 | { 99 | // since we only use chacha for randomness (and not encryption) we 100 | // do not _need_ to read 32-bit values as little endian but we do anyways 101 | // just for being compatible :-) 102 | memset(ctx, 0, sizeof(*ctx)); 103 | for (size_t i = 0; i < 4; i++) { 104 | const uint8_t* sigma = (uint8_t*)"expand 32-byte k"; 105 | ctx->input[i] = read32(sigma,i); 106 | } 107 | for (size_t i = 0; i < 8; i++) { 108 | ctx->input[i + 4] = read32(key,i); 109 | } 110 | ctx->input[12] = 0; 111 | ctx->input[13] = 0; 112 | ctx->input[14] = (uint32_t)nonce; 113 | ctx->input[15] = (uint32_t)(nonce >> 32); 114 | } 115 | 116 | static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) { 117 | memset(ctx_new, 0, sizeof(*ctx_new)); 118 | _mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input)); 119 | ctx_new->input[12] = 0; 120 | ctx_new->input[13] = 0; 121 | ctx_new->input[14] = (uint32_t)nonce; 122 | ctx_new->input[15] = (uint32_t)(nonce >> 32); 123 | mi_assert_internal(ctx->input[14] != ctx_new->input[14] || ctx->input[15] != ctx_new->input[15]); // do not reuse nonces! 124 | chacha_block(ctx_new); 125 | } 126 | 127 | 128 | /* ---------------------------------------------------------------------------- 129 | Random interface 130 | -----------------------------------------------------------------------------*/ 131 | 132 | #if MI_DEBUG>1 133 | static bool mi_random_is_initialized(mi_random_ctx_t* ctx) { 134 | return (ctx != NULL && ctx->input[0] != 0); 135 | } 136 | #endif 137 | 138 | void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) { 139 | mi_assert_internal(mi_random_is_initialized(ctx)); 140 | mi_assert_internal(ctx != ctx_new); 141 | chacha_split(ctx, (uintptr_t)ctx_new /*nonce*/, ctx_new); 142 | } 143 | 144 | uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { 145 | mi_assert_internal(mi_random_is_initialized(ctx)); 146 | #if MI_INTPTR_SIZE <= 4 147 | return chacha_next32(ctx); 148 | #elif MI_INTPTR_SIZE == 8 149 | return (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx)); 150 | #else 151 | # error "define mi_random_next for this platform" 152 | #endif 153 | } 154 | 155 | 156 | /* ---------------------------------------------------------------------------- 157 | To initialize a fresh random context. 158 | If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR. 159 | -----------------------------------------------------------------------------*/ 160 | 161 | uintptr_t _mi_os_random_weak(uintptr_t extra_seed) { 162 | uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random 163 | x ^= _mi_prim_clock_now(); 164 | // and do a few randomization steps 165 | uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1; 166 | for (uintptr_t i = 0; i < max; i++) { 167 | x = _mi_random_shuffle(x); 168 | } 169 | mi_assert_internal(x != 0); 170 | return x; 171 | } 172 | 173 | static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { 174 | uint8_t key[32]; 175 | if (use_weak || !_mi_prim_random_buf(key, sizeof(key))) { 176 | // if we fail to get random data from the OS, we fall back to a 177 | // weak random source based on the current time 178 | #if !defined(__wasi__) 179 | if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); } 180 | #endif 181 | uintptr_t x = _mi_os_random_weak(0); 182 | for (size_t i = 0; i < 8; i++) { // key is eight 32-bit words. 183 | x = _mi_random_shuffle(x); 184 | ((uint32_t*)key)[i] = (uint32_t)x; 185 | } 186 | ctx->weak = true; 187 | } 188 | else { 189 | ctx->weak = false; 190 | } 191 | chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ ); 192 | } 193 | 194 | void _mi_random_init(mi_random_ctx_t* ctx) { 195 | mi_random_init_ex(ctx, false); 196 | } 197 | 198 | void _mi_random_init_weak(mi_random_ctx_t * ctx) { 199 | mi_random_init_ex(ctx, true); 200 | } 201 | 202 | void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx) { 203 | if (ctx->weak) { 204 | _mi_random_init(ctx); 205 | } 206 | } 207 | 208 | /* -------------------------------------------------------- 209 | test vectors from 210 | ----------------------------------------------------------- */ 211 | /* 212 | static bool array_equals(uint32_t* x, uint32_t* y, size_t n) { 213 | for (size_t i = 0; i < n; i++) { 214 | if (x[i] != y[i]) return false; 215 | } 216 | return true; 217 | } 218 | static void chacha_test(void) 219 | { 220 | uint32_t x[4] = { 0x11111111, 0x01020304, 0x9b8d6f43, 0x01234567 }; 221 | uint32_t x_out[4] = { 0xea2a92f4, 0xcb1cf8ce, 0x4581472e, 0x5881c4bb }; 222 | qround(x, 0, 1, 2, 3); 223 | mi_assert_internal(array_equals(x, x_out, 4)); 224 | 225 | uint32_t y[16] = { 226 | 0x879531e0, 0xc5ecf37d, 0x516461b1, 0xc9a62f8a, 227 | 0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0x2a5f714c, 228 | 0x53372767, 0xb00a5631, 0x974c541a, 0x359e9963, 229 | 0x5c971061, 0x3d631689, 0x2098d9d6, 0x91dbd320 }; 230 | uint32_t y_out[16] = { 231 | 0x879531e0, 0xc5ecf37d, 0xbdb886dc, 0xc9a62f8a, 232 | 0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0xcfacafd2, 233 | 0xe46bea80, 0xb00a5631, 0x974c541a, 0x359e9963, 234 | 0x5c971061, 0xccc07c79, 0x2098d9d6, 0x91dbd320 }; 235 | qround(y, 2, 7, 8, 13); 236 | mi_assert_internal(array_equals(y, y_out, 16)); 237 | 238 | mi_random_ctx_t r = { 239 | { 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574, 240 | 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c, 241 | 0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c, 242 | 0x00000001, 0x09000000, 0x4a000000, 0x00000000 }, 243 | {0}, 244 | 0 245 | }; 246 | uint32_t r_out[16] = { 247 | 0xe4e7f110, 0x15593bd1, 0x1fdd0f50, 0xc47120a3, 248 | 0xc7f4d1c7, 0x0368c033, 0x9aaa2204, 0x4e6cd4c3, 249 | 0x466482d2, 0x09aa9f07, 0x05d7c214, 0xa2028bd9, 250 | 0xd19c12b5, 0xb94e16de, 0xe883d0cb, 0x4e3c50a2 }; 251 | chacha_block(&r); 252 | mi_assert_internal(array_equals(r.output, r_out, 16)); 253 | } 254 | */ 255 | -------------------------------------------------------------------------------- /mimalloc/c/src/segment-map.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2019-2023, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | /* ----------------------------------------------------------- 9 | The following functions are to reliably find the segment or 10 | block that encompasses any pointer p (or NULL if it is not 11 | in any of our segments). 12 | We maintain a bitmap of all memory with 1 bit per MI_SEGMENT_SIZE (64MiB) 13 | set to 1 if it contains the segment meta data. 14 | ----------------------------------------------------------- */ 15 | #include "mimalloc.h" 16 | #include "mimalloc/internal.h" 17 | #include "mimalloc/atomic.h" 18 | 19 | #if (MI_INTPTR_SIZE==8) 20 | #define MI_MAX_ADDRESS ((size_t)40 << 40) // 40TB (to include huge page areas) 21 | #else 22 | #define MI_MAX_ADDRESS ((size_t)2 << 30) // 2Gb 23 | #endif 24 | 25 | #define MI_SEGMENT_MAP_BITS (MI_MAX_ADDRESS / MI_SEGMENT_SIZE) 26 | #define MI_SEGMENT_MAP_SIZE (MI_SEGMENT_MAP_BITS / 8) 27 | #define MI_SEGMENT_MAP_WSIZE (MI_SEGMENT_MAP_SIZE / MI_INTPTR_SIZE) 28 | 29 | static _Atomic(uintptr_t) mi_segment_map[MI_SEGMENT_MAP_WSIZE + 1]; // 2KiB per TB with 64MiB segments 30 | 31 | static size_t mi_segment_map_index_of(const mi_segment_t* segment, size_t* bitidx) { 32 | mi_assert_internal(_mi_ptr_segment(segment + 1) == segment); // is it aligned on MI_SEGMENT_SIZE? 33 | if ((uintptr_t)segment >= MI_MAX_ADDRESS) { 34 | *bitidx = 0; 35 | return MI_SEGMENT_MAP_WSIZE; 36 | } 37 | else { 38 | const uintptr_t segindex = ((uintptr_t)segment) / MI_SEGMENT_SIZE; 39 | *bitidx = segindex % MI_INTPTR_BITS; 40 | const size_t mapindex = segindex / MI_INTPTR_BITS; 41 | mi_assert_internal(mapindex < MI_SEGMENT_MAP_WSIZE); 42 | return mapindex; 43 | } 44 | } 45 | 46 | void _mi_segment_map_allocated_at(const mi_segment_t* segment) { 47 | size_t bitidx; 48 | size_t index = mi_segment_map_index_of(segment, &bitidx); 49 | mi_assert_internal(index <= MI_SEGMENT_MAP_WSIZE); 50 | if (index==MI_SEGMENT_MAP_WSIZE) return; 51 | uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); 52 | uintptr_t newmask; 53 | do { 54 | newmask = (mask | ((uintptr_t)1 << bitidx)); 55 | } while (!mi_atomic_cas_weak_release(&mi_segment_map[index], &mask, newmask)); 56 | } 57 | 58 | void _mi_segment_map_freed_at(const mi_segment_t* segment) { 59 | size_t bitidx; 60 | size_t index = mi_segment_map_index_of(segment, &bitidx); 61 | mi_assert_internal(index <= MI_SEGMENT_MAP_WSIZE); 62 | if (index == MI_SEGMENT_MAP_WSIZE) return; 63 | uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); 64 | uintptr_t newmask; 65 | do { 66 | newmask = (mask & ~((uintptr_t)1 << bitidx)); 67 | } while (!mi_atomic_cas_weak_release(&mi_segment_map[index], &mask, newmask)); 68 | } 69 | 70 | // Determine the segment belonging to a pointer or NULL if it is not in a valid segment. 71 | static mi_segment_t* _mi_segment_of(const void* p) { 72 | if (p == NULL) return NULL; 73 | mi_segment_t* segment = _mi_ptr_segment(p); 74 | mi_assert_internal(segment != NULL); 75 | size_t bitidx; 76 | size_t index = mi_segment_map_index_of(segment, &bitidx); 77 | // fast path: for any pointer to valid small/medium/large object or first MI_SEGMENT_SIZE in huge 78 | const uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); 79 | if mi_likely((mask & ((uintptr_t)1 << bitidx)) != 0) { 80 | return segment; // yes, allocated by us 81 | } 82 | if (index==MI_SEGMENT_MAP_WSIZE) return NULL; 83 | 84 | // TODO: maintain max/min allocated range for efficiency for more efficient rejection of invalid pointers? 85 | 86 | // search downwards for the first segment in case it is an interior pointer 87 | // could be slow but searches in MI_INTPTR_SIZE * MI_SEGMENT_SIZE (512MiB) steps trough 88 | // valid huge objects 89 | // note: we could maintain a lowest index to speed up the path for invalid pointers? 90 | size_t lobitidx; 91 | size_t loindex; 92 | uintptr_t lobits = mask & (((uintptr_t)1 << bitidx) - 1); 93 | if (lobits != 0) { 94 | loindex = index; 95 | lobitidx = mi_bsr(lobits); // lobits != 0 96 | } 97 | else if (index == 0) { 98 | return NULL; 99 | } 100 | else { 101 | mi_assert_internal(index > 0); 102 | uintptr_t lomask = mask; 103 | loindex = index; 104 | do { 105 | loindex--; 106 | lomask = mi_atomic_load_relaxed(&mi_segment_map[loindex]); 107 | } while (lomask != 0 && loindex > 0); 108 | if (lomask == 0) return NULL; 109 | lobitidx = mi_bsr(lomask); // lomask != 0 110 | } 111 | mi_assert_internal(loindex < MI_SEGMENT_MAP_WSIZE); 112 | // take difference as the addresses could be larger than the MAX_ADDRESS space. 113 | size_t diff = (((index - loindex) * (8*MI_INTPTR_SIZE)) + bitidx - lobitidx) * MI_SEGMENT_SIZE; 114 | segment = (mi_segment_t*)((uint8_t*)segment - diff); 115 | 116 | if (segment == NULL) return NULL; 117 | mi_assert_internal((void*)segment < p); 118 | bool cookie_ok = (_mi_ptr_cookie(segment) == segment->cookie); 119 | mi_assert_internal(cookie_ok); 120 | if mi_unlikely(!cookie_ok) return NULL; 121 | if (((uint8_t*)segment + mi_segment_size(segment)) <= (uint8_t*)p) return NULL; // outside the range 122 | mi_assert_internal(p >= (void*)segment && (uint8_t*)p < (uint8_t*)segment + mi_segment_size(segment)); 123 | return segment; 124 | } 125 | 126 | // Is this a valid pointer in our heap? 127 | static bool mi_is_valid_pointer(const void* p) { 128 | return ((_mi_segment_of(p) != NULL) || (_mi_arena_contains(p))); 129 | } 130 | 131 | mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { 132 | return mi_is_valid_pointer(p); 133 | } 134 | 135 | /* 136 | // Return the full segment range belonging to a pointer 137 | static void* mi_segment_range_of(const void* p, size_t* size) { 138 | mi_segment_t* segment = _mi_segment_of(p); 139 | if (segment == NULL) { 140 | if (size != NULL) *size = 0; 141 | return NULL; 142 | } 143 | else { 144 | if (size != NULL) *size = segment->segment_size; 145 | return segment; 146 | } 147 | mi_assert_expensive(page == NULL || mi_segment_is_valid(_mi_page_segment(page),tld)); 148 | mi_assert_internal(page == NULL || (mi_segment_page_size(_mi_page_segment(page)) - (MI_SECURE == 0 ? 0 : _mi_os_page_size())) >= block_size); 149 | mi_reset_delayed(tld); 150 | mi_assert_internal(page == NULL || mi_page_not_in_queue(page, tld)); 151 | return page; 152 | } 153 | */ 154 | -------------------------------------------------------------------------------- /mimalloc/c/src/static.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2020, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #ifndef _DEFAULT_SOURCE 8 | #define _DEFAULT_SOURCE 9 | #endif 10 | #if defined(__sun) 11 | // same remarks as os.c for the static's context. 12 | #undef _XOPEN_SOURCE 13 | #undef _POSIX_C_SOURCE 14 | #endif 15 | 16 | #include "mimalloc.h" 17 | #include "mimalloc/internal.h" 18 | 19 | // For a static override we create a single object file 20 | // containing the whole library. If it is linked first 21 | // it will override all the standard library allocation 22 | // functions (on Unix's). 23 | #include "alloc.c" // includes alloc-override.c 24 | #include "alloc-aligned.c" 25 | #include "alloc-posix.c" 26 | #include "arena.c" 27 | #include "bitmap.c" 28 | #include "heap.c" 29 | #include "init.c" 30 | #include "options.c" 31 | #include "os.c" 32 | #include "page.c" // includes page-queue.c 33 | #include "random.c" 34 | #include "segment.c" 35 | #include "segment-map.c" 36 | #include "stats.c" 37 | #include "prim/prim.c" 38 | #if MI_OSX_ZONE 39 | #include "prim/osx/alloc-override-zone.c" 40 | #endif 41 | -------------------------------------------------------------------------------- /mimalloc/c/src/stats.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018-2021, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #include "mimalloc.h" 8 | #include "mimalloc/internal.h" 9 | #include "mimalloc/atomic.h" 10 | #include "mimalloc/prim.h" 11 | 12 | #include // snprintf 13 | #include // memset 14 | 15 | #if defined(_MSC_VER) && (_MSC_VER < 1920) 16 | #pragma warning(disable:4204) // non-constant aggregate initializer 17 | #endif 18 | 19 | /* ----------------------------------------------------------- 20 | Statistics operations 21 | ----------------------------------------------------------- */ 22 | 23 | static bool mi_is_in_main(void* stat) { 24 | return ((uint8_t*)stat >= (uint8_t*)&_mi_stats_main 25 | && (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t))); 26 | } 27 | 28 | static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { 29 | if (amount == 0) return; 30 | if (mi_is_in_main(stat)) 31 | { 32 | // add atomically (for abandoned pages) 33 | int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount); 34 | mi_atomic_maxi64_relaxed(&stat->peak, current + amount); 35 | if (amount > 0) { 36 | mi_atomic_addi64_relaxed(&stat->allocated,amount); 37 | } 38 | else { 39 | mi_atomic_addi64_relaxed(&stat->freed, -amount); 40 | } 41 | } 42 | else { 43 | // add thread local 44 | stat->current += amount; 45 | if (stat->current > stat->peak) stat->peak = stat->current; 46 | if (amount > 0) { 47 | stat->allocated += amount; 48 | } 49 | else { 50 | stat->freed += -amount; 51 | } 52 | } 53 | } 54 | 55 | void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { 56 | if (mi_is_in_main(stat)) { 57 | mi_atomic_addi64_relaxed( &stat->count, 1 ); 58 | mi_atomic_addi64_relaxed( &stat->total, (int64_t)amount ); 59 | } 60 | else { 61 | stat->count++; 62 | stat->total += amount; 63 | } 64 | } 65 | 66 | void _mi_stat_increase(mi_stat_count_t* stat, size_t amount) { 67 | mi_stat_update(stat, (int64_t)amount); 68 | } 69 | 70 | void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { 71 | mi_stat_update(stat, -((int64_t)amount)); 72 | } 73 | 74 | // must be thread safe as it is called from stats_merge 75 | static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src, int64_t unit) { 76 | if (stat==src) return; 77 | if (src->allocated==0 && src->freed==0) return; 78 | mi_atomic_addi64_relaxed( &stat->allocated, src->allocated * unit); 79 | mi_atomic_addi64_relaxed( &stat->current, src->current * unit); 80 | mi_atomic_addi64_relaxed( &stat->freed, src->freed * unit); 81 | // peak scores do not work across threads.. 82 | mi_atomic_addi64_relaxed( &stat->peak, src->peak * unit); 83 | } 84 | 85 | static void mi_stat_counter_add(mi_stat_counter_t* stat, const mi_stat_counter_t* src, int64_t unit) { 86 | if (stat==src) return; 87 | mi_atomic_addi64_relaxed( &stat->total, src->total * unit); 88 | mi_atomic_addi64_relaxed( &stat->count, src->count * unit); 89 | } 90 | 91 | // must be thread safe as it is called from stats_merge 92 | static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { 93 | if (stats==src) return; 94 | mi_stat_add(&stats->segments, &src->segments,1); 95 | mi_stat_add(&stats->pages, &src->pages,1); 96 | mi_stat_add(&stats->reserved, &src->reserved, 1); 97 | mi_stat_add(&stats->committed, &src->committed, 1); 98 | mi_stat_add(&stats->reset, &src->reset, 1); 99 | mi_stat_add(&stats->purged, &src->purged, 1); 100 | mi_stat_add(&stats->page_committed, &src->page_committed, 1); 101 | 102 | mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned, 1); 103 | mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned, 1); 104 | mi_stat_add(&stats->threads, &src->threads, 1); 105 | 106 | mi_stat_add(&stats->malloc, &src->malloc, 1); 107 | mi_stat_add(&stats->segments_cache, &src->segments_cache, 1); 108 | mi_stat_add(&stats->normal, &src->normal, 1); 109 | mi_stat_add(&stats->huge, &src->huge, 1); 110 | mi_stat_add(&stats->large, &src->large, 1); 111 | 112 | mi_stat_counter_add(&stats->pages_extended, &src->pages_extended, 1); 113 | mi_stat_counter_add(&stats->mmap_calls, &src->mmap_calls, 1); 114 | mi_stat_counter_add(&stats->commit_calls, &src->commit_calls, 1); 115 | mi_stat_counter_add(&stats->reset_calls, &src->reset_calls, 1); 116 | mi_stat_counter_add(&stats->purge_calls, &src->purge_calls, 1); 117 | 118 | mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire, 1); 119 | mi_stat_counter_add(&stats->searches, &src->searches, 1); 120 | mi_stat_counter_add(&stats->normal_count, &src->normal_count, 1); 121 | mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1); 122 | mi_stat_counter_add(&stats->large_count, &src->large_count, 1); 123 | #if MI_STAT>1 124 | for (size_t i = 0; i <= MI_BIN_HUGE; i++) { 125 | if (src->normal_bins[i].allocated > 0 || src->normal_bins[i].freed > 0) { 126 | mi_stat_add(&stats->normal_bins[i], &src->normal_bins[i], 1); 127 | } 128 | } 129 | #endif 130 | } 131 | 132 | /* ----------------------------------------------------------- 133 | Display statistics 134 | ----------------------------------------------------------- */ 135 | 136 | // unit > 0 : size in binary bytes 137 | // unit == 0: count as decimal 138 | // unit < 0 : count in binary 139 | static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { 140 | char buf[32]; buf[0] = 0; 141 | int len = 32; 142 | const char* suffix = (unit <= 0 ? " " : "B"); 143 | const int64_t base = (unit == 0 ? 1000 : 1024); 144 | if (unit>0) n *= unit; 145 | 146 | const int64_t pos = (n < 0 ? -n : n); 147 | if (pos < base) { 148 | if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column 149 | snprintf(buf, len, "%d %-3s", (int)n, (n==0 ? "" : suffix)); 150 | } 151 | } 152 | else { 153 | int64_t divider = base; 154 | const char* magnitude = "K"; 155 | if (pos >= divider*base) { divider *= base; magnitude = "M"; } 156 | if (pos >= divider*base) { divider *= base; magnitude = "G"; } 157 | const int64_t tens = (n / (divider/10)); 158 | const long whole = (long)(tens/10); 159 | const long frac1 = (long)(tens%10); 160 | char unitdesc[8]; 161 | snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); 162 | snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); 163 | } 164 | _mi_fprintf(out, arg, (fmt==NULL ? "%12s" : fmt), buf); 165 | } 166 | 167 | 168 | static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { 169 | mi_printf_amount(n,unit,out,arg,NULL); 170 | } 171 | 172 | static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { 173 | if (unit==1) _mi_fprintf(out, arg, "%12s"," "); 174 | else mi_print_amount(n,0,out,arg); 175 | } 176 | 177 | static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) { 178 | _mi_fprintf(out, arg,"%10s:", msg); 179 | if (unit > 0) { 180 | mi_print_amount(stat->peak, unit, out, arg); 181 | mi_print_amount(stat->allocated, unit, out, arg); 182 | mi_print_amount(stat->freed, unit, out, arg); 183 | mi_print_amount(stat->current, unit, out, arg); 184 | mi_print_amount(unit, 1, out, arg); 185 | mi_print_count(stat->allocated, unit, out, arg); 186 | if (stat->allocated > stat->freed) { 187 | _mi_fprintf(out, arg, " "); 188 | _mi_fprintf(out, arg, (notok == NULL ? "not all freed" : notok)); 189 | _mi_fprintf(out, arg, "\n"); 190 | } 191 | else { 192 | _mi_fprintf(out, arg, " ok\n"); 193 | } 194 | } 195 | else if (unit<0) { 196 | mi_print_amount(stat->peak, -1, out, arg); 197 | mi_print_amount(stat->allocated, -1, out, arg); 198 | mi_print_amount(stat->freed, -1, out, arg); 199 | mi_print_amount(stat->current, -1, out, arg); 200 | if (unit==-1) { 201 | _mi_fprintf(out, arg, "%24s", ""); 202 | } 203 | else { 204 | mi_print_amount(-unit, 1, out, arg); 205 | mi_print_count((stat->allocated / -unit), 0, out, arg); 206 | } 207 | if (stat->allocated > stat->freed) 208 | _mi_fprintf(out, arg, " not all freed!\n"); 209 | else 210 | _mi_fprintf(out, arg, " ok\n"); 211 | } 212 | else { 213 | mi_print_amount(stat->peak, 1, out, arg); 214 | mi_print_amount(stat->allocated, 1, out, arg); 215 | _mi_fprintf(out, arg, "%11s", " "); // no freed 216 | mi_print_amount(stat->current, 1, out, arg); 217 | _mi_fprintf(out, arg, "\n"); 218 | } 219 | } 220 | 221 | static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { 222 | mi_stat_print_ex(stat, msg, unit, out, arg, NULL); 223 | } 224 | 225 | static void mi_stat_peak_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { 226 | _mi_fprintf(out, arg, "%10s:", msg); 227 | mi_print_amount(stat->peak, unit, out, arg); 228 | _mi_fprintf(out, arg, "\n"); 229 | } 230 | 231 | static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { 232 | _mi_fprintf(out, arg, "%10s:", msg); 233 | mi_print_amount(stat->total, -1, out, arg); 234 | _mi_fprintf(out, arg, "\n"); 235 | } 236 | 237 | 238 | static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) { 239 | const int64_t avg_tens = (stat->count == 0 ? 0 : (stat->total*10 / stat->count)); 240 | const long avg_whole = (long)(avg_tens/10); 241 | const long avg_frac1 = (long)(avg_tens%10); 242 | _mi_fprintf(out, arg, "%10s: %5ld.%ld avg\n", msg, avg_whole, avg_frac1); 243 | } 244 | 245 | 246 | static void mi_print_header(mi_output_fun* out, void* arg ) { 247 | _mi_fprintf(out, arg, "%10s: %11s %11s %11s %11s %11s %11s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count "); 248 | } 249 | 250 | #if MI_STAT>1 251 | static void mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out, void* arg) { 252 | bool found = false; 253 | char buf[64]; 254 | for (size_t i = 0; i <= max; i++) { 255 | if (bins[i].allocated > 0) { 256 | found = true; 257 | int64_t unit = _mi_bin_size((uint8_t)i); 258 | snprintf(buf, 64, "%s %3lu", fmt, (long)i); 259 | mi_stat_print(&bins[i], buf, unit, out, arg); 260 | } 261 | } 262 | if (found) { 263 | _mi_fprintf(out, arg, "\n"); 264 | mi_print_header(out, arg); 265 | } 266 | } 267 | #endif 268 | 269 | 270 | 271 | //------------------------------------------------------------ 272 | // Use an output wrapper for line-buffered output 273 | // (which is nice when using loggers etc.) 274 | //------------------------------------------------------------ 275 | typedef struct buffered_s { 276 | mi_output_fun* out; // original output function 277 | void* arg; // and state 278 | char* buf; // local buffer of at least size `count+1` 279 | size_t used; // currently used chars `used <= count` 280 | size_t count; // total chars available for output 281 | } buffered_t; 282 | 283 | static void mi_buffered_flush(buffered_t* buf) { 284 | buf->buf[buf->used] = 0; 285 | _mi_fputs(buf->out, buf->arg, NULL, buf->buf); 286 | buf->used = 0; 287 | } 288 | 289 | static void mi_cdecl mi_buffered_out(const char* msg, void* arg) { 290 | buffered_t* buf = (buffered_t*)arg; 291 | if (msg==NULL || buf==NULL) return; 292 | for (const char* src = msg; *src != 0; src++) { 293 | char c = *src; 294 | if (buf->used >= buf->count) mi_buffered_flush(buf); 295 | mi_assert_internal(buf->used < buf->count); 296 | buf->buf[buf->used++] = c; 297 | if (c == '\n') mi_buffered_flush(buf); 298 | } 299 | } 300 | 301 | //------------------------------------------------------------ 302 | // Print statistics 303 | //------------------------------------------------------------ 304 | 305 | static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept { 306 | // wrap the output function to be line buffered 307 | char buf[256]; 308 | buffered_t buffer = { out0, arg0, NULL, 0, 255 }; 309 | buffer.buf = buf; 310 | mi_output_fun* out = &mi_buffered_out; 311 | void* arg = &buffer; 312 | 313 | // and print using that 314 | mi_print_header(out,arg); 315 | #if MI_STAT>1 316 | mi_stats_print_bins(stats->normal_bins, MI_BIN_HUGE, "normal",out,arg); 317 | #endif 318 | #if MI_STAT 319 | mi_stat_print(&stats->normal, "normal", (stats->normal_count.count == 0 ? 1 : -(stats->normal.allocated / stats->normal_count.count)), out, arg); 320 | mi_stat_print(&stats->large, "large", (stats->large_count.count == 0 ? 1 : -(stats->large.allocated / stats->large_count.count)), out, arg); 321 | mi_stat_print(&stats->huge, "huge", (stats->huge_count.count == 0 ? 1 : -(stats->huge.allocated / stats->huge_count.count)), out, arg); 322 | mi_stat_count_t total = { 0,0,0,0 }; 323 | mi_stat_add(&total, &stats->normal, 1); 324 | mi_stat_add(&total, &stats->large, 1); 325 | mi_stat_add(&total, &stats->huge, 1); 326 | mi_stat_print(&total, "total", 1, out, arg); 327 | #endif 328 | #if MI_STAT>1 329 | mi_stat_print(&stats->malloc, "malloc req", 1, out, arg); 330 | _mi_fprintf(out, arg, "\n"); 331 | #endif 332 | mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, ""); 333 | mi_stat_print_ex(&stats->committed, "committed", 1, out, arg, ""); 334 | mi_stat_peak_print(&stats->reset, "reset", 1, out, arg ); 335 | mi_stat_peak_print(&stats->purged, "purged", 1, out, arg ); 336 | mi_stat_print(&stats->page_committed, "touched", 1, out, arg); 337 | mi_stat_print(&stats->segments, "segments", -1, out, arg); 338 | mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg); 339 | mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg); 340 | mi_stat_print(&stats->pages, "pages", -1, out, arg); 341 | mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out, arg); 342 | mi_stat_counter_print(&stats->pages_extended, "-extended", out, arg); 343 | mi_stat_counter_print(&stats->page_no_retire, "-noretire", out, arg); 344 | mi_stat_counter_print(&stats->mmap_calls, "mmaps", out, arg); 345 | mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); 346 | mi_stat_counter_print(&stats->reset_calls, "resets", out, arg); 347 | mi_stat_counter_print(&stats->purge_calls, "purges", out, arg); 348 | mi_stat_print(&stats->threads, "threads", -1, out, arg); 349 | mi_stat_counter_print_avg(&stats->searches, "searches", out, arg); 350 | _mi_fprintf(out, arg, "%10s: %5zu\n", "numa nodes", _mi_os_numa_node_count()); 351 | 352 | size_t elapsed; 353 | size_t user_time; 354 | size_t sys_time; 355 | size_t current_rss; 356 | size_t peak_rss; 357 | size_t current_commit; 358 | size_t peak_commit; 359 | size_t page_faults; 360 | mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults); 361 | _mi_fprintf(out, arg, "%10s: %5ld.%03ld s\n", "elapsed", elapsed/1000, elapsed%1000); 362 | _mi_fprintf(out, arg, "%10s: user: %ld.%03ld s, system: %ld.%03ld s, faults: %lu, rss: ", "process", 363 | user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, (unsigned long)page_faults ); 364 | mi_printf_amount((int64_t)peak_rss, 1, out, arg, "%s"); 365 | if (peak_commit > 0) { 366 | _mi_fprintf(out, arg, ", commit: "); 367 | mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s"); 368 | } 369 | _mi_fprintf(out, arg, "\n"); 370 | } 371 | 372 | static mi_msecs_t mi_process_start; // = 0 373 | 374 | static mi_stats_t* mi_stats_get_default(void) { 375 | mi_heap_t* heap = mi_heap_get_default(); 376 | return &heap->tld->stats; 377 | } 378 | 379 | static void mi_stats_merge_from(mi_stats_t* stats) { 380 | if (stats != &_mi_stats_main) { 381 | mi_stats_add(&_mi_stats_main, stats); 382 | memset(stats, 0, sizeof(mi_stats_t)); 383 | } 384 | } 385 | 386 | void mi_stats_reset(void) mi_attr_noexcept { 387 | mi_stats_t* stats = mi_stats_get_default(); 388 | if (stats != &_mi_stats_main) { memset(stats, 0, sizeof(mi_stats_t)); } 389 | memset(&_mi_stats_main, 0, sizeof(mi_stats_t)); 390 | if (mi_process_start == 0) { mi_process_start = _mi_clock_start(); }; 391 | } 392 | 393 | void mi_stats_merge(void) mi_attr_noexcept { 394 | mi_stats_merge_from( mi_stats_get_default() ); 395 | } 396 | 397 | void _mi_stats_done(mi_stats_t* stats) { // called from `mi_thread_done` 398 | mi_stats_merge_from(stats); 399 | } 400 | 401 | void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { 402 | mi_stats_merge_from(mi_stats_get_default()); 403 | _mi_stats_print(&_mi_stats_main, out, arg); 404 | } 405 | 406 | void mi_stats_print(void* out) mi_attr_noexcept { 407 | // for compatibility there is an `out` parameter (which can be `stdout` or `stderr`) 408 | mi_stats_print_out((mi_output_fun*)out, NULL); 409 | } 410 | 411 | void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { 412 | _mi_stats_print(mi_stats_get_default(), out, arg); 413 | } 414 | 415 | 416 | // ---------------------------------------------------------------- 417 | // Basic timer for convenience; use milli-seconds to avoid doubles 418 | // ---------------------------------------------------------------- 419 | 420 | static mi_msecs_t mi_clock_diff; 421 | 422 | mi_msecs_t _mi_clock_now(void) { 423 | return _mi_prim_clock_now(); 424 | } 425 | 426 | mi_msecs_t _mi_clock_start(void) { 427 | if (mi_clock_diff == 0.0) { 428 | mi_msecs_t t0 = _mi_clock_now(); 429 | mi_clock_diff = _mi_clock_now() - t0; 430 | } 431 | return _mi_clock_now(); 432 | } 433 | 434 | mi_msecs_t _mi_clock_end(mi_msecs_t start) { 435 | mi_msecs_t end = _mi_clock_now(); 436 | return (end - start - mi_clock_diff); 437 | } 438 | 439 | 440 | // -------------------------------------------------------- 441 | // Basic process statistics 442 | // -------------------------------------------------------- 443 | 444 | mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept 445 | { 446 | mi_process_info_t pinfo; 447 | _mi_memzero_var(pinfo); 448 | pinfo.elapsed = _mi_clock_end(mi_process_start); 449 | pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); 450 | pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); 451 | pinfo.current_rss = pinfo.current_commit; 452 | pinfo.peak_rss = pinfo.peak_commit; 453 | pinfo.utime = 0; 454 | pinfo.stime = 0; 455 | pinfo.page_faults = 0; 456 | 457 | _mi_prim_process_info(&pinfo); 458 | 459 | if (elapsed_msecs!=NULL) *elapsed_msecs = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX)); 460 | if (user_msecs!=NULL) *user_msecs = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX)); 461 | if (system_msecs!=NULL) *system_msecs = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX)); 462 | if (current_rss!=NULL) *current_rss = pinfo.current_rss; 463 | if (peak_rss!=NULL) *peak_rss = pinfo.peak_rss; 464 | if (current_commit!=NULL) *current_commit = pinfo.current_commit; 465 | if (peak_commit!=NULL) *peak_commit = pinfo.peak_commit; 466 | if (page_faults!=NULL) *page_faults = pinfo.page_faults; 467 | } 468 | -------------------------------------------------------------------------------- /mimalloc/mimalloc.odin: -------------------------------------------------------------------------------- 1 | package mimalloc 2 | 3 | /* ---------------------------------------------------------------------------- 4 | Copyright (c) 2018-2023, Microsoft Research, Daan Leijen 5 | This is free software you can redistribute it and/or modify it under the 6 | terms of the MIT license. A copy of the license can be found in the file 7 | "LICENSE" at the root of this distribution. 8 | -----------------------------------------------------------------------------*/ 9 | 10 | import "core:c" 11 | 12 | #assert(size_of(uint) == size_of(c.size_t)) 13 | 14 | VERSION :: 212 // major + 2 digits minor 15 | 16 | SMALL_WSIZE_MAX :: 128 17 | SMALL_SIZE_MAX :: SMALL_WSIZE_MAX * size_of(rawptr) 18 | 19 | // heap_t pointer 20 | Heap :: distinct rawptr 21 | // arena_id_t 22 | Arena_Id :: distinct c.int 23 | 24 | // block_visit_fun 25 | Block_Visit_Proc :: #type proc "c" ( 26 | heap: Heap, 27 | area: ^Heap_Area, 28 | block: rawptr, 29 | block_size: uint, 30 | arg: rawptr, 31 | ) -> bool 32 | 33 | // deferred_free_fun 34 | Deferred_Free_Proc :: #type proc "c" (force: bool, heartbeat: uint, arg: rawptr) 35 | // output_fun 36 | Output_Proc :: #type proc "c" (msg: cstring, arg: rawptr) 37 | // error_fun 38 | Error_Proc :: #type proc "c" (err: int, arg: rawptr) 39 | 40 | // heap_area_t 41 | // An area of heap space contains blocks of a single size. 42 | Heap_Area :: struct { 43 | blocks: rawptr, // start of the area containing heap blocks 44 | reserved: uint, // bytes reserved for this area (virtual) 45 | committed: uint, // current available bytes for this area 46 | used: uint, // number of allocated blocks 47 | block_size: uint, // size in bytes of each block 48 | full_block_size: uint, // size in bytes of a full block including padding and metadata. 49 | } 50 | 51 | when ODIN_OS == .Windows { 52 | when ODIN_DEBUG { 53 | foreign import lib {"mimalloc_windows_x64_debug.lib", "system:Advapi32.lib"} 54 | } else { 55 | foreign import lib {"mimalloc_windows_x64_release.lib", "system:Advapi32.lib"} 56 | } 57 | } else when ODIN_OS == .Darwin { 58 | #panic("TODO") 59 | } else when ODIN_OS == .Linux { 60 | #panic("TODO") 61 | } else { 62 | #panic("OS currently not supported.") 63 | } 64 | 65 | @(default_calling_convention = "c", link_prefix = "mi_") 66 | foreign lib { 67 | // ------------------------------------------------------ 68 | // Standard malloc interface 69 | // ------------------------------------------------------ 70 | 71 | malloc :: proc(size: uint) -> rawptr --- 72 | calloc :: proc(count: uint, size: uint) -> rawptr --- 73 | realloc :: proc(p: rawptr, newsize: uint) -> rawptr --- 74 | expand :: proc(p: rawptr, newsize: uint) -> rawptr --- 75 | 76 | free :: proc(p: rawptr) --- 77 | strdup :: proc(s: cstring) -> cstring --- 78 | strndup :: proc(s: cstring, n: uint) -> cstring --- 79 | realpath :: proc(fname: cstring, resolved_name: cstring) -> cstring --- 80 | 81 | 82 | // ------------------------------------------------------ 83 | // Extended functionality 84 | // ------------------------------------------------------ 85 | 86 | malloc_small :: proc(size: uint) -> rawptr --- 87 | zalloc_small :: proc(size: uint) -> rawptr --- 88 | zalloc :: proc(size: uint) -> rawptr --- 89 | 90 | mallocn :: proc(count: uint, size: uint) -> rawptr --- 91 | reallocn :: proc(p: rawptr, count: uint, size: uint) -> rawptr --- 92 | reallocf :: proc(p: rawptr, newsize: uint) -> rawptr --- 93 | 94 | usable_size :: proc(p: rawptr) -> uint --- 95 | good_size :: proc(size: uint) -> uint --- 96 | 97 | 98 | // ------------------------------------------------------ 99 | // Internals 100 | // ------------------------------------------------------ 101 | 102 | register_deferred_free :: proc(deferred_free: Deferred_Free_Proc, arg: rawptr) --- 103 | register_output :: proc(out: Output_Proc, arg: rawptr) --- 104 | register_error :: proc(fun: Error_Proc, arg: rawptr) --- 105 | 106 | collect :: proc(force: bool) --- 107 | version :: proc() -> int --- 108 | stats_reset :: proc() --- 109 | stats_merge :: proc() --- 110 | stats_print :: proc(out: rawptr = nil) --- // backward compatibility: `out` is ignored and should be NULL 111 | stats_print_out :: proc(out: Output_Proc, arg: rawptr) --- 112 | 113 | process_init :: proc() --- 114 | thread_init :: proc() --- 115 | thread_done :: proc() --- 116 | thread_stats_print_out :: proc(out: Output_Proc, arg: rawptr) --- 117 | 118 | process_info :: proc(elapsed_msecs: ^uint, user_msecs: ^uint, system_msecs: ^uint, current_rss: ^uint, peak_rss: ^uint, current_commit: ^uint, peak_commit: ^uint, page_faults: ^uint) --- 119 | 120 | 121 | // ------------------------------------------------------------------------------------- 122 | // Aligned allocation 123 | // Note that `alignment` always follows `size` for consistency with unaligned 124 | // allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. 125 | // ------------------------------------------------------------------------------------- 126 | 127 | malloc_aligned :: proc(size: uint, alignment: uint) -> rawptr --- 128 | malloc_aligned_at :: proc(size: uint, alignment: uint, offset: uint) -> rawptr --- 129 | zalloc_aligned :: proc(size: uint, alignment: uint) -> rawptr --- 130 | zalloc_aligned_at :: proc(size: uint, alignment: uint, offset: uint) -> rawptr --- 131 | calloc_aligned :: proc(count: uint, size: uint, alignment: uint) -> rawptr --- 132 | calloc_aligned_at :: proc(count: uint, size: uint, alignment: uint, offset: uint) -> rawptr --- 133 | realloc_aligned :: proc(p: rawptr, newsize: uint, alignment: uint) -> rawptr --- 134 | realloc_aligned_at :: proc(p: rawptr, newsize: uint, alignment: uint, offset: uint) -> rawptr --- 135 | 136 | 137 | // ------------------------------------------------------------------------------------- 138 | // Heaps: first-class, but can only allocate from the same thread that created it. 139 | // ------------------------------------------------------------------------------------- 140 | 141 | heap_new :: proc() -> Heap --- 142 | // Safe delete a heap without freeing any still allocated blocks in that heap 143 | heap_delete :: proc(heap: Heap) --- 144 | heap_destroy :: proc(heap: Heap) --- 145 | heap_set_default :: proc(heap: Heap) -> Heap --- 146 | heap_get_default :: proc() -> Heap --- 147 | heap_get_backing :: proc() -> Heap --- 148 | heap_collect :: proc(heap: Heap, force: bool) --- 149 | 150 | heap_malloc :: proc(heap: Heap, size: uint) -> rawptr --- 151 | heap_zalloc :: proc(heap: Heap, size: uint) -> rawptr --- 152 | heap_calloc :: proc(heap: Heap, count: uint, size: uint) -> rawptr --- 153 | heap_mallocn :: proc(heap: Heap, count: uint, size: uint) -> rawptr --- 154 | heap_malloc_small :: proc(heap: Heap, size: uint) -> rawptr --- 155 | 156 | heap_realloc :: proc(heap: Heap, p: rawptr, newsize: uint) -> rawptr --- 157 | heap_reallocn :: proc(heap: Heap, p: rawptr, count: uint, size: uint) -> rawptr --- 158 | heap_reallocf :: proc(heap: Heap, p: rawptr, newsize: uint) -> rawptr --- 159 | 160 | heap_strdup :: proc(heap: Heap, s: cstring) -> cstring --- 161 | heap_strndup :: proc(heap: Heap, s: cstring, n: uint) -> cstring --- 162 | heap_realpath :: proc(heap: Heap, fname: cstring, resolved_name: cstring) -> cstring --- 163 | 164 | heap_malloc_aligned :: proc(heap: Heap, size: uint, alignment: uint) -> rawptr --- 165 | heap_malloc_aligned_at :: proc(heap: Heap, size: uint, alignment: uint, offset: uint) -> rawptr --- 166 | heap_zalloc_aligned :: proc(heap: Heap, size: uint, alignment: uint) -> rawptr --- 167 | heap_zalloc_aligned_at :: proc(heap: Heap, size: uint, alignment: uint, offset: uint) -> rawptr --- 168 | heap_calloc_aligned :: proc(heap: Heap, count: uint, size: uint, alignment: uint) -> rawptr --- 169 | heap_calloc_aligned_at :: proc(heap: Heap, count: uint, size: uint, alignment: uint, offset: uint) -> rawptr --- 170 | heap_realloc_aligned :: proc(heap: Heap, p: rawptr, newsize: uint, alignment: uint) -> rawptr --- 171 | heap_realloc_aligned_at :: proc(heap: Heap, p: rawptr, newsize: uint, alignment: uint, offset: uint) -> rawptr --- 172 | 173 | 174 | // -------------------------------------------------------------------------------- 175 | // Zero initialized re-allocation. 176 | // Only valid on memory that was originally allocated with zero initialization too. 177 | // e.g. `calloc`, `zalloc`, `zalloc_aligned` etc. 178 | // see 179 | // -------------------------------------------------------------------------------- 180 | 181 | rezalloc :: proc(p: rawptr, newsize: uint) -> rawptr --- 182 | recalloc :: proc(p: rawptr, newcount: uint, size: uint) -> rawptr --- 183 | 184 | rezalloc_aligned :: proc(p: rawptr, newsize: uint, alignment: uint) -> rawptr --- 185 | rezalloc_aligned_at :: proc(p: rawptr, newsize: uint, alignment: uint, offset: uint) -> rawptr --- 186 | recalloc_aligned :: proc(p: rawptr, newcount: uint, size: uint, alignment: uint) -> rawptr --- 187 | recalloc_aligned_at :: proc(p: rawptr, newcount: uint, size: uint, alignment: uint, offset: uint) -> rawptr --- 188 | 189 | heap_rezalloc :: proc(heap: Heap, p: rawptr, newsize: uint) -> rawptr --- 190 | heap_recalloc :: proc(heap: Heap, p: rawptr, newcount: uint, size: uint) -> rawptr --- 191 | 192 | heap_rezalloc_aligned :: proc(heap: Heap, p: rawptr, newsize: uint, alignment: uint) -> rawptr --- 193 | heap_rezalloc_aligned_at :: proc(heap: Heap, p: rawptr, newsize: uint, alignment: uint, offset: uint) -> rawptr --- 194 | heap_recalloc_aligned :: proc(heap: Heap, p: rawptr, newcount: uint, size: uint, alignment: uint) -> rawptr --- 195 | heap_recalloc_aligned_at :: proc(heap: Heap, p: rawptr, newcount: uint, size: uint, alignment: uint, offset: uint) -> rawptr --- 196 | 197 | 198 | // ------------------------------------------------------ 199 | // Analysis 200 | // ------------------------------------------------------ 201 | 202 | heap_contains_block :: proc(heap: Heap, p: rawptr) -> bool --- 203 | heap_check_owned :: proc(heap: Heap, p: rawptr) -> bool --- 204 | check_owned :: proc(p: rawptr) -> bool --- 205 | 206 | heap_visit_blocks :: proc(heap: Heap, visit_all_blocks: bool, visitor: Block_Visit_Proc, arg: rawptr) -> bool --- 207 | 208 | // Experimental 209 | is_in_heap_region :: proc(p: rawptr) -> bool --- 210 | is_redirected :: proc() -> bool --- 211 | 212 | reserve_huge_os_pages_interleave :: proc(pages: uint, numa_nodes: uint, timeout_msecs: uint) -> int --- 213 | reserve_huge_os_pages_at :: proc(pages: uint, numa_node: int, timeout_msecs: uint) -> int --- 214 | 215 | reserve_os_memory :: proc(size: uint, commit: bool, allow_large: bool) -> int --- 216 | manage_os_memory :: proc(start: rawptr, size: uint, is_committed: bool, is_large: bool, is_zero: bool, numa_node: int) -> bool --- 217 | 218 | debug_show_arenas :: proc() --- 219 | 220 | // Experimental: heaps associated with specific memory arena's 221 | arena_area :: proc(arena_id: Arena_Id, size: ^uint) -> rawptr --- 222 | reserve_huge_os_pages_at_ex :: proc(pages: uint, numa_node: int, timeout_msecs: uint, exclusive: bool, arena_id: ^Arena_Id) -> int --- 223 | reserve_os_memory_ex :: proc(size: uint, commit: bool, allow_large: bool, exclusive: bool, arena_id: ^Arena_Id) -> int --- 224 | manage_os_memory_ex :: proc(start: rawptr, size: uint, is_committed: bool, is_large: bool, is_zero: bool, numa_node: int, exclusive: bool, arena_id: ^Arena_Id) -> bool --- 225 | 226 | when VERSION >= 182 { 227 | // Create a heap that only allocates in the specified arena 228 | heap_new_in_arena :: proc(arena_id: Arena_Id) -> Heap --- 229 | } 230 | 231 | // deprecated 232 | @(deprecated = "reserve_huge_os_pages is deprecated.") 233 | reserve_huge_os_pages :: proc(pages: uint, max_secs: f64, pages_reserved: ^uint) -> int --- 234 | 235 | 236 | // ------------------------------------------------------ 237 | // Options 238 | // ------------------------------------------------------ 239 | 240 | option_is_enabled :: proc(option: Option) -> bool --- 241 | option_enable :: proc(option: Option) --- 242 | option_disable :: proc(option: Option) --- 243 | option_set_enabled :: proc(option: Option, enable: bool) --- 244 | option_set_enabled_default :: proc(option: Option, enable: bool) --- 245 | 246 | option_get :: proc(option: Option) -> int --- 247 | option_get_clamp :: proc(option: Option, min: int, max: int) -> int --- 248 | option_get_size :: proc(option: Option) -> uint --- 249 | option_set :: proc(option: Option, value: int) --- 250 | option_set_default :: proc(option: Option, value: int) --- 251 | } // foreign lib 252 | 253 | // option_t 254 | Option :: enum c.int { 255 | // stable options 256 | show_errors, // print error messages 257 | show_stats, // print statistics on termination 258 | verbose, // print verbose messages 259 | 260 | // the following options are experimental (see src/options.h) 261 | eager_commit, // eager commit segments? (after `eager_commit_delay` segments) (=1) 262 | arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2) 263 | purge_decommits, // should a memory purge decommit (or only reset) (=1) 264 | allow_large_os_pages, // allow large (2MiB) OS pages, implies eager commit 265 | reserve_huge_os_pages, // reserve N huge OS pages (1GiB/page) at startup 266 | reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node 267 | reserve_os_memory, // reserve specified amount of OS memory in an arena at startup 268 | deprecated_segment_cache, 269 | deprecated_page_reset, 270 | abandoned_page_purge, // immediately purge delayed purges on thread termination 271 | deprecated_segment_reset, 272 | eager_commit_delay, 273 | purge_delay, // memory purging is delayed by N milli seconds use 0 for immediate purging or -1 for no purging at all. 274 | use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes. 275 | limit_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas) 276 | os_tag, // tag used for OS logging (macOS only for now) 277 | max_errors, // issue at most N error messages 278 | max_warnings, // issue at most N warning messages 279 | max_segment_reclaim, 280 | destroy_on_exit, // if set, release all memory on exit sometimes used for dynamic unloading but can be unsafe. 281 | arena_reserve, // initial memory size in KiB for arena reservation (1GiB on 64-bit) 282 | arena_purge_mult, 283 | purge_extend_delay, 284 | _option_last, 285 | 286 | // legacy option names 287 | large_os_pages = allow_large_os_pages, 288 | eager_region_commit = arena_eager_commit, 289 | reset_decommits = purge_decommits, 290 | reset_delay = purge_delay, 291 | abandoned_page_reset = abandoned_page_purge, 292 | } 293 | -------------------------------------------------------------------------------- /mimalloc/wrapper.odin: -------------------------------------------------------------------------------- 1 | package mimalloc 2 | 3 | import "core:mem" 4 | 5 | global_allocator_proc :: proc( 6 | allocator_data: rawptr, 7 | mode: mem.Allocator_Mode, 8 | size, alignment: int, 9 | old_memory: rawptr, 10 | old_size: int, 11 | location := #caller_location, 12 | ) -> ( 13 | []byte, 14 | mem.Allocator_Error, 15 | ) { 16 | 17 | switch mode { 18 | case .Alloc: 19 | ptr := zalloc_aligned(uint(size), uint(alignment)) 20 | return mem.byte_slice(ptr, size), nil 21 | 22 | case .Free: 23 | free(old_memory) 24 | return nil, nil 25 | 26 | case .Free_All: 27 | return nil, .Mode_Not_Implemented 28 | 29 | case .Resize: 30 | ptr := realloc_aligned(old_memory, uint(size), uint(alignment)) 31 | return mem.byte_slice(ptr, size), nil 32 | 33 | case .Query_Features: 34 | set := (^mem.Allocator_Mode_Set)(old_memory) 35 | if set != nil { 36 | set^ = {.Alloc, .Alloc_Non_Zeroed, .Free, .Resize, .Query_Features} 37 | } 38 | return nil, nil 39 | 40 | case .Query_Info: 41 | return nil, .Mode_Not_Implemented 42 | 43 | case .Alloc_Non_Zeroed: 44 | ptr := malloc_aligned(uint(size), uint(alignment)) 45 | return mem.byte_slice(ptr, size), nil 46 | } 47 | 48 | return nil, nil 49 | } 50 | 51 | global_allocator :: proc "contextless" () -> mem.Allocator { 52 | return {procedure = global_allocator_proc, data = nil} 53 | } 54 | 55 | heap_allocator_proc :: proc( 56 | allocator_data: rawptr, 57 | mode: mem.Allocator_Mode, 58 | size, alignment: int, 59 | old_memory: rawptr, 60 | old_size: int, 61 | location := #caller_location, 62 | ) -> ( 63 | []byte, 64 | mem.Allocator_Error, 65 | ) { 66 | heap := cast(Heap)allocator_data 67 | 68 | switch mode { 69 | case .Alloc: 70 | ptr := heap_zalloc_aligned(heap, uint(size), uint(alignment)) 71 | return mem.byte_slice(ptr, size), nil 72 | 73 | case .Free: 74 | free(old_memory) 75 | return nil, nil 76 | 77 | case .Free_All: 78 | return nil, .Mode_Not_Implemented 79 | 80 | case .Resize: 81 | ptr := heap_realloc_aligned(heap, old_memory, uint(size), uint(alignment)) 82 | return mem.byte_slice(ptr, size), nil 83 | 84 | case .Query_Features: 85 | set := (^mem.Allocator_Mode_Set)(old_memory) 86 | if set != nil { 87 | set^ = {.Alloc, .Alloc_Non_Zeroed, .Free, .Resize, .Query_Features} 88 | } 89 | return nil, nil 90 | 91 | case .Query_Info: 92 | return nil, .Mode_Not_Implemented 93 | 94 | case .Alloc_Non_Zeroed: 95 | ptr := heap_malloc_aligned(heap, uint(size), uint(alignment)) 96 | return mem.byte_slice(ptr, size), nil 97 | } 98 | 99 | return nil, nil 100 | } 101 | 102 | heap_allocator :: proc "contextless" (heap: Heap) -> mem.Allocator { 103 | return {procedure = heap_allocator_proc, data = heap} 104 | } 105 | -------------------------------------------------------------------------------- /odinfmt.json: -------------------------------------------------------------------------------- 1 | { 2 | "character_width": 110, 3 | "tabs": false, 4 | "spaces": 4, 5 | "newline_limit": 3, 6 | "tabs_width": 4 7 | } -------------------------------------------------------------------------------- /ols.json: -------------------------------------------------------------------------------- 1 | { 2 | "collections": [ 3 | {"name": "core", "path": "D:\\Odin\\core"}, 4 | {"name": "vendor", "path": "D:\\Odin\\vendor"}, 5 | ], 6 | "enable_document_symbols": true, 7 | "enable_semantic_tokens": true, 8 | "enable_hover": true, 9 | "enable_snippets": true, 10 | "checker_args": "-ignore-unknown-attributes -debug", 11 | "verbose": true 12 | } 13 | --------------------------------------------------------------------------------