├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── aligned_alloc.c ├── calloc.c ├── dump.c ├── free.c ├── glue.h ├── malloc.c ├── malloc_usable_size.c ├── memalign.c ├── meta.h ├── posix_memalign.c └── realloc.c /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.a 3 | *.so 4 | config.mak 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © 2019 Rich Felker 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 17 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 18 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 19 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 20 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | ALL = libmallocng.a libmallocng.so 3 | SRCS = malloc.c calloc.c free.c realloc.c aligned_alloc.c posix_memalign.c memalign.c malloc_usable_size.c dump.c 4 | OBJS = $(SRCS:.c=.o) 5 | CFLAGS = -fPIC -Wall -O2 -ffreestanding 6 | 7 | -include config.mak 8 | 9 | all: $(ALL) 10 | 11 | $(OBJS): meta.h glue.h 12 | 13 | clean: 14 | rm -f $(ALL) $(OBJS) 15 | 16 | libmallocng.a: $(OBJS) 17 | rm -f $@ 18 | ar rc $@ $(OBJS) 19 | ranlib $@ 20 | 21 | libmallocng.so: $(OBJS) 22 | $(CC) $(CFLAGS) $(LDFLAGS) -shared -o $@ $(OBJS) 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Next-gen malloc for musl libc - Working draft 2 | 3 | This is a draft of the upcoming next-gen `malloc` implementation for 4 | musl libc. It is essentially feature-complete, but is not yet 5 | integrated with musl and lacks some additional hardening and 6 | optimizations that the final version is expected to include. 7 | 8 | The included `Makefile` builds static and shared library files that 9 | programs can link with, or use with `LD_PRELOAD` (for the shared 10 | version). 11 | 12 | ## High-level design 13 | 14 | This allocator organizes memory dynamically into small slab-style 15 | groups of up to 32 identical-size allocation units with status 16 | controlled by bitmasks, and utilizes a mix of in-band and out-of-band 17 | metadata to isolate sensitive state from regions easily accessible 18 | through out-of-bounds writes, while avoiding the need for expensive 19 | global data structures. 20 | 21 | The closest analogue among other well-known allocators is probably 22 | OpenBSD's omalloc. 23 | 24 | Base allocation granularity and alignment is 16 bytes. Large 25 | allocations are made individually by mmap, but they also have 26 | out-of-band metadata records treating them as special one-member 27 | groups, allowing `realloc` and `free` to validate them. Smaller 28 | allocations come from groups of one of 48 size classes, spaced 29 | linearly up to 128 (the first 8 classes), then roughly geometrically 30 | with four steps per doubling, but adjusted to divide powers of two 31 | with minimal remainder (waste). 32 | -------------------------------------------------------------------------------- /aligned_alloc.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "meta.h" 4 | 5 | void *aligned_alloc(size_t align, size_t len) 6 | { 7 | if ((align & -align) != align) { 8 | errno = EINVAL; 9 | return 0; 10 | } 11 | 12 | if (len > SIZE_MAX - align || align >= (1ULL<<31)*UNIT) { 13 | errno = ENOMEM; 14 | return 0; 15 | } 16 | 17 | if (DISABLE_ALIGNED_ALLOC) { 18 | errno = ENOMEM; 19 | return 0; 20 | } 21 | 22 | if (align <= UNIT) align = UNIT; 23 | 24 | unsigned char *p = malloc(len + align - UNIT); 25 | struct meta *g = get_meta(p); 26 | int idx = get_slot_index(p); 27 | size_t stride = get_stride(g); 28 | unsigned char *start = g->mem->storage + stride*idx; 29 | unsigned char *end = g->mem->storage + stride*(idx+1) - IB; 30 | size_t adj = -(uintptr_t)p & (align-1); 31 | 32 | if (!adj) { 33 | set_size(p, end, len); 34 | return p; 35 | } 36 | p += adj; 37 | uint32_t offset = (size_t)(p-g->mem->storage)/UNIT; 38 | if (offset <= 0xffff) { 39 | *(uint16_t *)(p-2) = offset; 40 | p[-4] = 0; 41 | } else { 42 | // use a 32-bit offset if 16-bit doesn't fit. for this, 43 | // 16-bit field must be zero, [-4] byte nonzero. 44 | *(uint16_t *)(p-2) = 0; 45 | *(uint32_t *)(p-8) = offset; 46 | p[-4] = 1; 47 | } 48 | p[-3] = idx; 49 | set_size(p, end, len); 50 | // store offset to aligned enframing. this facilitates cycling 51 | // offset and also iteration of heap for debugging/measurement. 52 | // for extreme overalignment it won't fit but these are classless 53 | // allocations anyway. 54 | *(uint16_t *)(start - 2) = (size_t)(p-start)/UNIT; 55 | start[-3] = 7<<5; 56 | return p; 57 | } 58 | -------------------------------------------------------------------------------- /calloc.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "meta.h" 5 | 6 | void *calloc(size_t m, size_t n) 7 | { 8 | if (n && m > (size_t)-1/n) { 9 | errno = ENOMEM; 10 | return 0; 11 | } 12 | n *= m; 13 | void *p = malloc(n); 14 | if (!p || is_allzero(p)) return p; 15 | return memset(p, 0, n); 16 | } 17 | -------------------------------------------------------------------------------- /dump.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "meta.h" 3 | 4 | static void print_group(FILE *f, struct meta *g) 5 | { 6 | size_t size = g->sizeclass>48 7 | ? g->maplen*4096UL-UNIT 8 | : UNIT*size_classes[g->sizeclass]; 9 | fprintf(f, "%p: %p [%d slots] [class %d (%zu)]: ", g, g->mem, 10 | g->last_idx+1, g->sizeclass, size); 11 | for (int i=0; i<=g->last_idx; i++) { 12 | putc((g->avail_mask & (1u< g->mem->active_idx) ? 'i' 14 | : (g->freed_mask & (1u<next)!=h); 25 | } 26 | 27 | static void print_full_groups(FILE *f) 28 | { 29 | struct meta_area *p; 30 | struct meta *m; 31 | for (p=ctx.meta_area_head; p; p=p->next) { 32 | for (int i=0; inslots; i++) { 33 | m = &p->slots[i]; 34 | if (m->mem && !m->next) 35 | print_group(f, m); 36 | } 37 | } 38 | } 39 | 40 | static size_t count_list(struct meta *h) 41 | { 42 | size_t cnt = 0; 43 | struct meta *m = h; 44 | if (!m) return 0; 45 | do cnt++; 46 | while ((m=m->next)!=h); 47 | return cnt; 48 | } 49 | 50 | void dump_heap(FILE *f) 51 | { 52 | wrlock(); 53 | 54 | fprintf(f, "free meta records: %zu\n", count_list(ctx.free_meta_head)); 55 | fprintf(f, "available new meta records: %zu\n", ctx.avail_meta_count); 56 | fprintf(f, "available new meta areas: %zu\n", ctx.avail_meta_area_count); 57 | 58 | fprintf(f, "entirely filled, inactive groups:\n"); 59 | print_full_groups(f); 60 | 61 | fprintf(f, "free groups by size class:\n"); 62 | for (int i=0; i<48; i++) { 63 | if (!ctx.active[i]) continue; 64 | fprintf(f, "-- class %d (%d) (%zu used) --\n", i, size_classes[i]*UNIT, ctx.usage_by_class[i]); 65 | print_group_list(f, ctx.active[i]); 66 | } 67 | 68 | unlock(); 69 | } 70 | -------------------------------------------------------------------------------- /free.c: -------------------------------------------------------------------------------- 1 | #define _BSD_SOURCE 2 | #include 3 | #include 4 | 5 | #include "meta.h" 6 | 7 | struct mapinfo { 8 | void *base; 9 | size_t len; 10 | }; 11 | 12 | static struct mapinfo nontrivial_free(struct meta *, int); 13 | 14 | static struct mapinfo free_group(struct meta *g) 15 | { 16 | struct mapinfo mi = { 0 }; 17 | int sc = g->sizeclass; 18 | if (sc < 48) { 19 | ctx.usage_by_class[sc] -= g->last_idx+1; 20 | } 21 | if (g->maplen) { 22 | step_seq(); 23 | record_seq(sc); 24 | mi.base = g->mem; 25 | mi.len = g->maplen*4096UL; 26 | } else { 27 | void *p = g->mem; 28 | struct meta *m = get_meta(p); 29 | int idx = get_slot_index(p); 30 | g->mem->meta = 0; 31 | // not checking size/reserved here; it's intentionally invalid 32 | mi = nontrivial_free(m, idx); 33 | } 34 | free_meta(g); 35 | return mi; 36 | } 37 | 38 | static int okay_to_free(struct meta *g) 39 | { 40 | int sc = g->sizeclass; 41 | 42 | if (!g->freeable) return 0; 43 | 44 | // always free individual mmaps not suitable for reuse 45 | if (sc >= 48 || get_stride(g) < UNIT*size_classes[sc]) 46 | return 1; 47 | 48 | // always free groups allocated inside another group's slot 49 | // since recreating them should not be expensive and they 50 | // might be blocking freeing of a much larger group. 51 | if (!g->maplen) return 1; 52 | 53 | // if there is another non-full group, free this one to 54 | // consolidate future allocations, reduce fragmentation. 55 | if (g->next != g) return 1; 56 | 57 | // free any group in a size class that's not bouncing 58 | if (!is_bouncing(sc)) return 1; 59 | 60 | size_t cnt = g->last_idx+1; 61 | size_t usage = ctx.usage_by_class[sc]; 62 | 63 | // if usage is high enough that a larger count should be 64 | // used, free the low-count group so a new one will be made. 65 | if (9*cnt <= usage && cnt < 20) 66 | return 1; 67 | 68 | // otherwise, keep the last group in a bouncing class. 69 | return 0; 70 | } 71 | 72 | static struct mapinfo nontrivial_free(struct meta *g, int i) 73 | { 74 | uint32_t self = 1u<sizeclass; 76 | uint32_t mask = g->freed_mask | g->avail_mask; 77 | 78 | if (mask+self == (2u<last_idx)-1 && okay_to_free(g)) { 79 | // any multi-slot group is necessarily on an active list 80 | // here, but single-slot groups might or might not be. 81 | if (g->next) { 82 | assert(sc < 48); 83 | int activate_new = (ctx.active[sc]==g); 84 | dequeue(&ctx.active[sc], g); 85 | if (activate_new && ctx.active[sc]) 86 | activate_group(ctx.active[sc]); 87 | } 88 | return free_group(g); 89 | } else if (!mask) { 90 | assert(sc < 48); 91 | // might still be active if there were no allocations 92 | // after last available slot was taken. 93 | if (ctx.active[sc] != g) { 94 | queue(&ctx.active[sc], g); 95 | } 96 | } 97 | a_or(&g->freed_mask, self); 98 | return (struct mapinfo){ 0 }; 99 | } 100 | 101 | void free(void *p) 102 | { 103 | if (!p) return; 104 | 105 | struct meta *g = get_meta(p); 106 | int idx = get_slot_index(p); 107 | size_t stride = get_stride(g); 108 | unsigned char *start = g->mem->storage + stride*idx; 109 | unsigned char *end = start + stride - IB; 110 | get_nominal_size(p, end); 111 | uint32_t self = 1u<last_idx)-1; 112 | ((unsigned char *)p)[-3] = 255; 113 | // invalidate offset to group header, and cycle offset of 114 | // used region within slot if current offset is zero. 115 | *(uint16_t *)((char *)p-2) = 0; 116 | 117 | // release any whole pages contained in the slot to be freed 118 | // unless it's a single-slot group that will be unmapped. 119 | if (((uintptr_t)(start-1) ^ (uintptr_t)end) >= 2*PGSZ && g->last_idx) { 120 | unsigned char *base = start + (-(uintptr_t)start & (PGSZ-1)); 121 | size_t len = (end-base) & -PGSZ; 122 | if (len) madvise(base, len, MADV_FREE); 123 | } 124 | 125 | // atomic free without locking if this is neither first or last slot 126 | for (;;) { 127 | uint32_t freed = g->freed_mask; 128 | uint32_t avail = g->avail_mask; 129 | uint32_t mask = freed | avail; 130 | assert(!(mask&self)); 131 | if (!freed || mask+self==all) break; 132 | if (!MT) 133 | g->freed_mask = freed+self; 134 | else if (a_cas(&g->freed_mask, freed, freed+self)!=freed) 135 | continue; 136 | return; 137 | } 138 | 139 | wrlock(); 140 | struct mapinfo mi = nontrivial_free(g, idx); 141 | unlock(); 142 | if (mi.len) munmap(mi.base, mi.len); 143 | } 144 | -------------------------------------------------------------------------------- /glue.h: -------------------------------------------------------------------------------- 1 | #ifndef MALLOC_GLUE_H 2 | #define MALLOC_GLUE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | // use macros to appropriately namespace these. for libc, 10 | // the names would be changed to lie in __ namespace. 11 | #define size_classes malloc_size_classes 12 | #define ctx malloc_context 13 | #define alloc_meta malloc_alloc_meta 14 | #define is_allzero malloc_allzerop 15 | 16 | #if USE_REAL_ASSERT 17 | #include 18 | #else 19 | #undef assert 20 | #define assert(x) do { if (!(x)) __builtin_trap(); } while(0) 21 | #endif 22 | 23 | #undef brk 24 | #if USE_BRK 25 | #include 26 | #define brk(p) ((uintptr_t)syscall(SYS_brk, p)) 27 | #else 28 | #define brk(p) ((p)-1) 29 | #endif 30 | 31 | #ifndef MADV_FREE 32 | #undef madvise 33 | #define madvise(p,l,a) (-1) 34 | #define MADV_FREE 0 35 | #endif 36 | 37 | #ifndef MREMAP_MAYMOVE 38 | #undef mremap 39 | #define mremap(p,o,n,f) MAP_FAILED 40 | #endif 41 | 42 | #define DISABLE_ALIGNED_ALLOC 0 43 | 44 | static inline int a_ctz_32(uint32_t x) 45 | { 46 | return __builtin_ctz(x); 47 | } 48 | 49 | static inline int a_clz_32(uint32_t x) 50 | { 51 | return __builtin_clz(x); 52 | } 53 | 54 | static inline int a_cas(volatile int *p, int t, int s) 55 | { 56 | return __sync_val_compare_and_swap(p, t, s); 57 | } 58 | 59 | static inline int a_swap(volatile int *p, int v) 60 | { 61 | int x; 62 | do x = *p; 63 | while (a_cas(p, x, v)!=x); 64 | return x; 65 | } 66 | 67 | static inline void a_or(volatile int *p, int v) 68 | { 69 | __sync_fetch_and_or(p, v); 70 | } 71 | 72 | static inline uint64_t get_random_secret() 73 | { 74 | uint64_t secret; 75 | getentropy(&secret, sizeof secret); 76 | return secret; 77 | } 78 | 79 | static inline size_t get_page_size() 80 | { 81 | return sysconf(_SC_PAGESIZE); 82 | } 83 | 84 | // no portable "is multithreaded" predicate so assume true 85 | #define MT 1 86 | 87 | #define LOCK_TYPE_MUTEX 1 88 | #define LOCK_TYPE_RWLOCK 2 89 | 90 | #ifndef LOCK_TYPE 91 | #define LOCK_TYPE LOCK_TYPE_MUTEX 92 | #endif 93 | 94 | #if LOCK_TYPE == LOCK_TYPE_MUTEX 95 | 96 | #define RDLOCK_IS_EXCLUSIVE 1 97 | 98 | __attribute__((__visibility__("hidden"))) 99 | extern pthread_mutex_t malloc_lock; 100 | 101 | #define LOCK_OBJ_DEF \ 102 | pthread_mutex_t malloc_lock = PTHREAD_MUTEX_INITIALIZER 103 | 104 | static inline void rdlock() 105 | { 106 | if (MT) pthread_mutex_lock(&malloc_lock); 107 | } 108 | static inline void wrlock() 109 | { 110 | if (MT) pthread_mutex_lock(&malloc_lock); 111 | } 112 | static inline void unlock() 113 | { 114 | if (MT) pthread_mutex_unlock(&malloc_lock); 115 | } 116 | static inline void upgradelock() 117 | { 118 | } 119 | 120 | #elif LOCK_TYPE == LOCK_TYPE_RWLOCK 121 | 122 | #define RDLOCK_IS_EXCLUSIVE 0 123 | 124 | __attribute__((__visibility__("hidden"))) 125 | extern pthread_rwlock_t malloc_lock; 126 | 127 | #define LOCK_OBJ_DEF \ 128 | pthread_rwlock_t malloc_lock = PTHREAD_RWLOCK_INITIALIZER 129 | 130 | static inline void rdlock() 131 | { 132 | if (MT) pthread_rwlock_rdlock(&malloc_lock); 133 | } 134 | static inline void wrlock() 135 | { 136 | if (MT) pthread_rwlock_wrlock(&malloc_lock); 137 | } 138 | static inline void unlock() 139 | { 140 | if (MT) pthread_rwlock_unlock(&malloc_lock); 141 | } 142 | static inline void upgradelock() 143 | { 144 | unlock(); 145 | wrlock(); 146 | } 147 | 148 | #endif 149 | 150 | #endif 151 | -------------------------------------------------------------------------------- /malloc.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "meta.h" 9 | 10 | LOCK_OBJ_DEF; 11 | 12 | const uint16_t size_classes[] = { 13 | 1, 2, 3, 4, 5, 6, 7, 8, 14 | 9, 10, 12, 15, 15 | 18, 20, 25, 31, 16 | 36, 42, 50, 63, 17 | 72, 84, 102, 127, 18 | 146, 170, 204, 255, 19 | 292, 340, 409, 511, 20 | 584, 682, 818, 1023, 21 | 1169, 1364, 1637, 2047, 22 | 2340, 2730, 3276, 4095, 23 | 4680, 5460, 6552, 8191, 24 | }; 25 | 26 | static const uint8_t small_cnt_tab[][3] = { 27 | { 30, 30, 30 }, 28 | { 31, 15, 15 }, 29 | { 20, 10, 10 }, 30 | { 31, 15, 7 }, 31 | { 25, 12, 6 }, 32 | { 21, 10, 5 }, 33 | { 18, 8, 4 }, 34 | { 31, 15, 7 }, 35 | { 28, 14, 6 }, 36 | }; 37 | 38 | static const uint8_t med_cnt_tab[4] = { 28, 24, 20, 32 }; 39 | 40 | struct malloc_context ctx = { 0 }; 41 | 42 | struct meta *alloc_meta(void) 43 | { 44 | struct meta *m; 45 | unsigned char *p; 46 | if (!ctx.init_done) { 47 | #ifndef PAGESIZE 48 | ctx.pagesize = get_page_size(); 49 | #endif 50 | ctx.secret = get_random_secret(); 51 | ctx.init_done = 1; 52 | } 53 | size_t pagesize = PGSZ; 54 | if (pagesize < 4096) pagesize = 4096; 55 | if ((m = dequeue_head(&ctx.free_meta_head))) return m; 56 | if (!ctx.avail_meta_count) { 57 | int need_unprotect = 1; 58 | if (!ctx.avail_meta_area_count && ctx.brk!=-1) { 59 | uintptr_t new = ctx.brk + pagesize; 60 | int need_guard = 0; 61 | if (!ctx.brk) { 62 | need_guard = 1; 63 | ctx.brk = brk(0); 64 | // some ancient kernels returned _ebss 65 | // instead of next page as initial brk. 66 | ctx.brk += -ctx.brk & (pagesize-1); 67 | new = ctx.brk + 2*pagesize; 68 | } 69 | if (brk(new) != new) { 70 | ctx.brk = -1; 71 | } else { 72 | if (need_guard) mmap((void *)ctx.brk, pagesize, 73 | PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); 74 | ctx.brk = new; 75 | ctx.avail_meta_areas = (void *)(new - pagesize); 76 | ctx.avail_meta_area_count = pagesize>>12; 77 | need_unprotect = 0; 78 | } 79 | } 80 | if (!ctx.avail_meta_area_count) { 81 | size_t n = 2UL << ctx.meta_alloc_shift; 82 | p = mmap(0, n*pagesize, PROT_NONE, 83 | MAP_PRIVATE|MAP_ANON, -1, 0); 84 | if (p==MAP_FAILED) return 0; 85 | ctx.avail_meta_areas = p + pagesize; 86 | ctx.avail_meta_area_count = (n-1)*(pagesize>>12); 87 | ctx.meta_alloc_shift++; 88 | } 89 | p = ctx.avail_meta_areas; 90 | if ((uintptr_t)p & (pagesize-1)) need_unprotect = 0; 91 | if (need_unprotect) 92 | if (mprotect(p, pagesize, PROT_READ|PROT_WRITE) 93 | && errno != ENOSYS) 94 | return 0; 95 | ctx.avail_meta_area_count--; 96 | ctx.avail_meta_areas = p + 4096; 97 | if (ctx.meta_area_tail) { 98 | ctx.meta_area_tail->next = (void *)p; 99 | } else { 100 | ctx.meta_area_head = (void *)p; 101 | } 102 | ctx.meta_area_tail = (void *)p; 103 | ctx.meta_area_tail->check = ctx.secret; 104 | ctx.avail_meta_count = ctx.meta_area_tail->nslots 105 | = (4096-sizeof(struct meta_area))/sizeof *m; 106 | ctx.avail_meta = ctx.meta_area_tail->slots; 107 | } 108 | ctx.avail_meta_count--; 109 | m = ctx.avail_meta++; 110 | m->prev = m->next = 0; 111 | return m; 112 | } 113 | 114 | static uint32_t try_avail(struct meta **pm) 115 | { 116 | struct meta *m = *pm; 117 | uint32_t first; 118 | if (!m) return 0; 119 | uint32_t mask = m->avail_mask; 120 | if (!mask) { 121 | if (!m) return 0; 122 | if (!m->freed_mask) { 123 | dequeue(pm, m); 124 | m = *pm; 125 | if (!m) return 0; 126 | } else { 127 | m = m->next; 128 | *pm = m; 129 | } 130 | 131 | mask = m->freed_mask; 132 | 133 | // skip fully-free group unless it's the only one 134 | // or it's a permanently non-freeable group 135 | if (mask == (2u<last_idx)-1 && m->freeable) { 136 | m = m->next; 137 | *pm = m; 138 | mask = m->freed_mask; 139 | } 140 | 141 | // activate more slots in a not-fully-active group 142 | // if needed, but only as a last resort. prefer using 143 | // any other group with free slots. this avoids 144 | // touching & dirtying as-yet-unused pages. 145 | if (!(mask & ((2u<mem->active_idx)-1))) { 146 | if (m->next != m) { 147 | m = m->next; 148 | *pm = m; 149 | } else { 150 | int cnt = m->mem->active_idx + 2; 151 | int size = size_classes[m->sizeclass]*UNIT; 152 | int span = UNIT + size*cnt; 153 | // activate up to next 4k boundary 154 | while ((span^(span+size-1)) < 4096) { 155 | cnt++; 156 | span += size; 157 | } 158 | if (cnt > m->last_idx+1) 159 | cnt = m->last_idx+1; 160 | m->mem->active_idx = cnt-1; 161 | } 162 | } 163 | mask = activate_group(m); 164 | assert(mask); 165 | decay_bounces(m->sizeclass); 166 | } 167 | first = mask&-mask; 168 | m->avail_mask = mask-first; 169 | return first; 170 | } 171 | 172 | static int alloc_slot(int, size_t); 173 | 174 | static struct meta *alloc_group(int sc, size_t req) 175 | { 176 | size_t size = UNIT*size_classes[sc]; 177 | int i = 0, cnt; 178 | unsigned char *p; 179 | struct meta *m = alloc_meta(); 180 | if (!m) return 0; 181 | size_t usage = ctx.usage_by_class[sc]; 182 | size_t pagesize = PGSZ; 183 | int active_idx; 184 | if (sc < 9) { 185 | while (i<2 && 4*small_cnt_tab[sc][i] > usage) 186 | i++; 187 | cnt = small_cnt_tab[sc][i]; 188 | } else { 189 | // lookup max number of slots fitting in power-of-two size 190 | // from a table, along with number of factors of two we 191 | // can divide out without a remainder or reaching 1. 192 | cnt = med_cnt_tab[sc&3]; 193 | 194 | // reduce cnt to avoid excessive eagar allocation. 195 | while (!(cnt&1) && 4*cnt > usage) 196 | cnt >>= 1; 197 | 198 | // data structures don't support groups whose slot offsets 199 | // in units don't fit in 16 bits. 200 | while (size*cnt >= 65536*UNIT) 201 | cnt >>= 1; 202 | } 203 | 204 | // If we selected a count of 1 above but it's not sufficient to use 205 | // mmap, increase to 2. Then it might be; if not it will nest. 206 | if (cnt==1 && size*cnt+UNIT <= pagesize/2) cnt = 2; 207 | 208 | // All choices of size*cnt are "just below" a power of two, so anything 209 | // larger than half the page size should be allocated as whole pages. 210 | if (size*cnt+UNIT > pagesize/2) { 211 | // check/update bounce counter to start/increase retention 212 | // of freed maps, and inhibit use of low-count, odd-size 213 | // small mappings and single-slot groups if activated. 214 | int nosmall = is_bouncing(sc); 215 | account_bounce(sc); 216 | step_seq(); 217 | 218 | // since the following count reduction opportunities have 219 | // an absolute memory usage cost, don't overdo them. count 220 | // coarse usage as part of usage. 221 | if (!(sc&1) && sc<32) usage += ctx.usage_by_class[sc+1]; 222 | 223 | // try to drop to a lower count if the one found above 224 | // increases usage by more than 25%. these reduced counts 225 | // roughly fill an integral number of pages, just not a 226 | // power of two, limiting amount of unusable space. 227 | if (4*cnt > usage && !nosmall) { 228 | if (0); 229 | else if ((sc&3)==1 && size*cnt>8*pagesize) cnt = 2; 230 | else if ((sc&3)==2 && size*cnt>4*pagesize) cnt = 3; 231 | else if ((sc&3)==0 && size*cnt>8*pagesize) cnt = 3; 232 | else if ((sc&3)==0 && size*cnt>2*pagesize) cnt = 5; 233 | } 234 | size_t needed = size*cnt + UNIT; 235 | needed += -needed & (pagesize-1); 236 | 237 | // produce an individually-mmapped allocation if usage is low, 238 | // bounce counter hasn't triggered, and either it saves memory 239 | // or it avoids eagar slot allocation without wasting too much. 240 | if (!nosmall && cnt<=7) { 241 | req += IB + UNIT; 242 | req += -req & (pagesize-1); 243 | if (req=4*pagesize && 2*cnt>usage)) { 244 | cnt = 1; 245 | needed = req; 246 | } 247 | } 248 | 249 | p = mmap(0, needed, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0); 250 | if (p==MAP_FAILED) { 251 | free_meta(m); 252 | return 0; 253 | } 254 | m->maplen = needed>>12; 255 | ctx.mmap_counter++; 256 | active_idx = (4096-UNIT)/size-1; 257 | if (active_idx > cnt-1) active_idx = cnt-1; 258 | if (active_idx < 0) active_idx = 0; 259 | } else { 260 | int j = size_to_class(UNIT+cnt*size-IB); 261 | int idx = alloc_slot(j, UNIT+cnt*size-IB); 262 | if (idx < 0) { 263 | free_meta(m); 264 | return 0; 265 | } 266 | struct meta *g = ctx.active[j]; 267 | p = enframe(g, idx, UNIT*size_classes[j]-IB, ctx.mmap_counter); 268 | m->maplen = 0; 269 | p[-3] = (p[-3]&31) | (6<<5); 270 | for (int i=0; i<=cnt; i++) 271 | p[UNIT+i*size-4] = 0; 272 | active_idx = cnt-1; 273 | } 274 | ctx.usage_by_class[sc] += cnt; 275 | m->avail_mask = (2u<freed_mask = (2u<<(cnt-1))-1 - m->avail_mask; 277 | m->mem = (void *)p; 278 | m->mem->meta = m; 279 | m->mem->active_idx = active_idx; 280 | m->last_idx = cnt-1; 281 | m->freeable = 1; 282 | m->sizeclass = sc; 283 | return m; 284 | } 285 | 286 | static int alloc_slot(int sc, size_t req) 287 | { 288 | uint32_t first = try_avail(&ctx.active[sc]); 289 | if (first) return a_ctz_32(first); 290 | 291 | struct meta *g = alloc_group(sc, req); 292 | if (!g) return -1; 293 | 294 | g->avail_mask--; 295 | queue(&ctx.active[sc], g); 296 | return 0; 297 | } 298 | 299 | void *malloc(size_t n) 300 | { 301 | if (size_overflows(n)) return 0; 302 | struct meta *g; 303 | uint32_t mask, first; 304 | int sc; 305 | int idx; 306 | int ctr; 307 | 308 | if (n >= MMAP_THRESHOLD) { 309 | size_t needed = n + IB + UNIT; 310 | void *p = mmap(0, needed, PROT_READ|PROT_WRITE, 311 | MAP_PRIVATE|MAP_ANON, -1, 0); 312 | if (p==MAP_FAILED) return 0; 313 | wrlock(); 314 | step_seq(); 315 | g = alloc_meta(); 316 | if (!g) { 317 | unlock(); 318 | munmap(p, needed); 319 | return 0; 320 | } 321 | g->mem = p; 322 | g->mem->meta = g; 323 | g->last_idx = 0; 324 | g->freeable = 1; 325 | g->sizeclass = 63; 326 | g->maplen = (needed+4095)/4096; 327 | g->avail_mask = g->freed_mask = 0; 328 | // use a global counter to cycle offset in 329 | // individually-mmapped allocations. 330 | ctx.mmap_counter++; 331 | idx = 0; 332 | goto success; 333 | } 334 | 335 | sc = size_to_class(n); 336 | 337 | rdlock(); 338 | g = ctx.active[sc]; 339 | 340 | // use coarse size classes initially when there are not yet 341 | // any groups of desired size. this allows counts of 2 or 3 342 | // to be allocated at first rather than having to start with 343 | // 7 or 5, the min counts for even size classes. 344 | if (!g && sc>=4 && sc<32 && sc!=6 && !(sc&1) && !ctx.usage_by_class[sc]) { 345 | size_t usage = ctx.usage_by_class[sc|1]; 346 | // if a new group may be allocated, count it toward 347 | // usage in deciding if we can use coarse class. 348 | if (!ctx.active[sc|1] || (!ctx.active[sc|1]->avail_mask 349 | && !ctx.active[sc|1]->freed_mask)) 350 | usage += 3; 351 | if (usage <= 12) 352 | sc |= 1; 353 | g = ctx.active[sc]; 354 | } 355 | 356 | for (;;) { 357 | mask = g ? g->avail_mask : 0; 358 | first = mask&-mask; 359 | if (!first) break; 360 | if (RDLOCK_IS_EXCLUSIVE || !MT) 361 | g->avail_mask = mask-first; 362 | else if (a_cas(&g->avail_mask, mask, mask-first)!=mask) 363 | continue; 364 | idx = a_ctz_32(first); 365 | goto success; 366 | } 367 | upgradelock(); 368 | 369 | idx = alloc_slot(sc, n); 370 | if (idx < 0) { 371 | unlock(); 372 | return 0; 373 | } 374 | g = ctx.active[sc]; 375 | 376 | success: 377 | ctr = ctx.mmap_counter; 378 | unlock(); 379 | return enframe(g, idx, n, ctr); 380 | } 381 | 382 | int is_allzero(void *p) 383 | { 384 | struct meta *g = get_meta(p); 385 | return g->sizeclass >= 48 || 386 | get_stride(g) < UNIT*size_classes[g->sizeclass]; 387 | } 388 | -------------------------------------------------------------------------------- /malloc_usable_size.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "meta.h" 3 | 4 | size_t malloc_usable_size(void *p) 5 | { 6 | struct meta *g = get_meta(p); 7 | int idx = get_slot_index(p); 8 | size_t stride = get_stride(g); 9 | unsigned char *start = g->mem->storage + stride*idx; 10 | unsigned char *end = start + stride - IB; 11 | return get_nominal_size(p, end); 12 | } 13 | -------------------------------------------------------------------------------- /memalign.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void *memalign(size_t align, size_t len) 4 | { 5 | return aligned_alloc(align, len); 6 | } 7 | -------------------------------------------------------------------------------- /meta.h: -------------------------------------------------------------------------------- 1 | #ifndef MALLOC_META_H 2 | #define MALLOC_META_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include "glue.h" 8 | 9 | __attribute__((__visibility__("hidden"))) 10 | extern const uint16_t size_classes[]; 11 | 12 | #define MMAP_THRESHOLD 131052 13 | 14 | #define UNIT 16 15 | #define IB 4 16 | 17 | struct group { 18 | struct meta *meta; 19 | unsigned char active_idx:5; 20 | char pad[UNIT - sizeof(struct meta *) - 1]; 21 | unsigned char storage[]; 22 | }; 23 | 24 | struct meta { 25 | struct meta *prev, *next; 26 | struct group *mem; 27 | volatile int avail_mask, freed_mask; 28 | uintptr_t last_idx:5; 29 | uintptr_t freeable:1; 30 | uintptr_t sizeclass:6; 31 | uintptr_t maplen:8*sizeof(uintptr_t)-12; 32 | }; 33 | 34 | struct meta_area { 35 | uint64_t check; 36 | struct meta_area *next; 37 | int nslots; 38 | struct meta slots[]; 39 | }; 40 | 41 | struct malloc_context { 42 | uint64_t secret; 43 | #ifndef PAGESIZE 44 | size_t pagesize; 45 | #endif 46 | int init_done; 47 | unsigned mmap_counter; 48 | struct meta *free_meta_head; 49 | struct meta *avail_meta; 50 | size_t avail_meta_count, avail_meta_area_count, meta_alloc_shift; 51 | struct meta_area *meta_area_head, *meta_area_tail; 52 | unsigned char *avail_meta_areas; 53 | struct meta *active[48]; 54 | size_t usage_by_class[48]; 55 | uint8_t unmap_seq[32], bounces[32]; 56 | uint8_t seq; 57 | uintptr_t brk; 58 | }; 59 | 60 | __attribute__((__visibility__("hidden"))) 61 | extern struct malloc_context ctx; 62 | 63 | #ifdef PAGESIZE 64 | #define PGSZ PAGESIZE 65 | #else 66 | #define PGSZ ctx.pagesize 67 | #endif 68 | 69 | __attribute__((__visibility__("hidden"))) 70 | struct meta *alloc_meta(void); 71 | 72 | __attribute__((__visibility__("hidden"))) 73 | int is_allzero(void *); 74 | 75 | static inline void queue(struct meta **phead, struct meta *m) 76 | { 77 | assert(!m->next); 78 | assert(!m->prev); 79 | if (*phead) { 80 | struct meta *head = *phead; 81 | m->next = head; 82 | m->prev = head->prev; 83 | m->next->prev = m->prev->next = m; 84 | } else { 85 | m->prev = m->next = m; 86 | *phead = m; 87 | } 88 | } 89 | 90 | static inline void dequeue(struct meta **phead, struct meta *m) 91 | { 92 | if (m->next != m) { 93 | m->prev->next = m->next; 94 | m->next->prev = m->prev; 95 | if (*phead == m) *phead = m->next; 96 | } else { 97 | *phead = 0; 98 | } 99 | m->prev = m->next = 0; 100 | } 101 | 102 | static inline struct meta *dequeue_head(struct meta **phead) 103 | { 104 | struct meta *m = *phead; 105 | if (m) dequeue(phead, m); 106 | return m; 107 | } 108 | 109 | static inline void free_meta(struct meta *m) 110 | { 111 | *m = (struct meta){0}; 112 | queue(&ctx.free_meta_head, m); 113 | } 114 | 115 | static inline uint32_t activate_group(struct meta *m) 116 | { 117 | assert(!m->avail_mask); 118 | uint32_t mask, act = (2u<mem->active_idx)-1; 119 | do mask = m->freed_mask; 120 | while (a_cas(&m->freed_mask, mask, mask&~act)!=mask); 121 | return m->avail_mask = mask & act; 122 | } 123 | 124 | static inline int get_slot_index(const unsigned char *p) 125 | { 126 | return p[-3] & 31; 127 | } 128 | 129 | static inline struct meta *get_meta(const unsigned char *p) 130 | { 131 | assert(!((uintptr_t)p & 15)); 132 | int offset = *(const uint16_t *)(p - 2); 133 | int index = get_slot_index(p); 134 | if (p[-4]) { 135 | assert(!offset); 136 | offset = *(uint32_t *)(p - 8); 137 | assert(offset > 0xffff); 138 | } 139 | const struct group *base = (const void *)(p - UNIT*offset - UNIT); 140 | const struct meta *meta = base->meta; 141 | assert(meta->mem == base); 142 | assert(index <= meta->last_idx); 143 | assert(!(meta->avail_mask & (1u<freed_mask & (1u<check == ctx.secret); 147 | if (meta->sizeclass < 48) { 148 | assert(offset >= size_classes[meta->sizeclass]*index); 149 | assert(offset < size_classes[meta->sizeclass]*(index+1)); 150 | } else { 151 | assert(meta->sizeclass == 63); 152 | } 153 | if (meta->maplen) { 154 | assert(offset <= meta->maplen*4096UL/UNIT - 1); 155 | } 156 | return (struct meta *)meta; 157 | } 158 | 159 | static inline size_t get_nominal_size(const unsigned char *p, const unsigned char *end) 160 | { 161 | size_t reserved = p[-3] >> 5; 162 | if (reserved >= 5) { 163 | assert(reserved == 5); 164 | reserved = *(const uint32_t *)(end-4); 165 | assert(reserved >= 5); 166 | assert(!end[-5]); 167 | } 168 | assert(reserved <= end-p); 169 | assert(!*(end-reserved)); 170 | // also check the slot's overflow byte 171 | assert(!*end); 172 | return end-reserved-p; 173 | } 174 | 175 | static inline size_t get_stride(const struct meta *g) 176 | { 177 | if (!g->last_idx && g->maplen) { 178 | return g->maplen*4096UL - UNIT; 179 | } else { 180 | return UNIT*size_classes[g->sizeclass]; 181 | } 182 | } 183 | 184 | static inline void set_size(unsigned char *p, unsigned char *end, size_t n) 185 | { 186 | int reserved = end-p-n; 187 | if (reserved) end[-reserved] = 0; 188 | if (reserved >= 5) { 189 | *(uint32_t *)(end-4) = reserved; 190 | end[-5] = 0; 191 | reserved = 5; 192 | } 193 | p[-3] = (p[-3]&31) + (reserved<<5); 194 | } 195 | 196 | static inline void *enframe(struct meta *g, int idx, size_t n, int ctr) 197 | { 198 | size_t stride = get_stride(g); 199 | size_t slack = (stride-IB-n)/UNIT; 200 | unsigned char *p = g->mem->storage + stride*idx; 201 | unsigned char *end = p+stride-IB; 202 | // cycle offset within slot to increase interval to address 203 | // reuse, facilitate trapping double-free. 204 | int off = (p[-3] ? *(uint16_t *)(p-2) + 1 : ctr) & 255; 205 | assert(!p[-4]); 206 | if (off > slack) { 207 | size_t m = slack; 208 | m |= m>>1; m |= m>>2; m |= m>>4; 209 | off &= m; 210 | if (off > slack) off -= slack+1; 211 | assert(off <= slack); 212 | } 213 | if (off) { 214 | // store offset in unused header at offset zero 215 | // if enframing at non-zero offset. 216 | *(uint16_t *)(p-2) = off; 217 | p[-3] = 7<<5; 218 | p += UNIT*off; 219 | // for nonzero offset there is no permanent check 220 | // byte, so make one. 221 | p[-4] = 0; 222 | } 223 | *(uint16_t *)(p-2) = (size_t)(p-g->mem->storage)/UNIT; 224 | p[-3] = idx; 225 | set_size(p, end, n); 226 | return p; 227 | } 228 | 229 | static inline int size_to_class(size_t n) 230 | { 231 | n = (n+IB-1)>>4; 232 | if (n<10) return n; 233 | n++; 234 | int i = (28-a_clz_32(n))*4 + 8; 235 | if (n>size_classes[i+1]) i+=2; 236 | if (n>size_classes[i]) i++; 237 | return i; 238 | } 239 | 240 | static inline int size_overflows(size_t n) 241 | { 242 | if (n >= SIZE_MAX/2 - 4096) { 243 | errno = ENOMEM; 244 | return 1; 245 | } 246 | return 0; 247 | } 248 | 249 | static inline void step_seq(void) 250 | { 251 | if (ctx.seq==255) { 252 | for (int i=0; i<32; i++) ctx.unmap_seq[i] = 0; 253 | ctx.seq = 1; 254 | } else { 255 | ctx.seq++; 256 | } 257 | } 258 | 259 | static inline void record_seq(int sc) 260 | { 261 | if (sc-7U < 32) ctx.unmap_seq[sc-7] = ctx.seq; 262 | } 263 | 264 | static inline void account_bounce(int sc) 265 | { 266 | if (sc-7U < 32) { 267 | int seq = ctx.unmap_seq[sc-7]; 268 | if (seq && ctx.seq-seq < 10) { 269 | if (ctx.bounces[sc-7]+1 < 100) 270 | ctx.bounces[sc-7]++; 271 | else 272 | ctx.bounces[sc-7] = 150; 273 | } 274 | } 275 | } 276 | 277 | static inline void decay_bounces(int sc) 278 | { 279 | if (sc-7U < 32 && ctx.bounces[sc-7]) 280 | ctx.bounces[sc-7]--; 281 | } 282 | 283 | static inline int is_bouncing(int sc) 284 | { 285 | return (sc-7U < 32 && ctx.bounces[sc-7] >= 100); 286 | } 287 | 288 | #endif 289 | -------------------------------------------------------------------------------- /posix_memalign.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | int posix_memalign(void **res, size_t align, size_t len) 5 | { 6 | if (align < sizeof(void *)) return EINVAL; 7 | void *mem = aligned_alloc(align, len); 8 | if (!mem) return errno; 9 | *res = mem; 10 | return 0; 11 | } 12 | -------------------------------------------------------------------------------- /realloc.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include "meta.h" 6 | 7 | void *realloc(void *p, size_t n) 8 | { 9 | if (!p) return malloc(n); 10 | if (size_overflows(n)) return 0; 11 | 12 | struct meta *g = get_meta(p); 13 | int idx = get_slot_index(p); 14 | size_t stride = get_stride(g); 15 | unsigned char *start = g->mem->storage + stride*idx; 16 | unsigned char *end = start + stride - IB; 17 | size_t old_size = get_nominal_size(p, end); 18 | size_t avail_size = end-(unsigned char *)p; 19 | void *new; 20 | 21 | // only resize in-place if size class matches 22 | if (n <= avail_size && n= g->sizeclass) { 24 | set_size(p, end, n); 25 | return p; 26 | } 27 | 28 | // use mremap if old and new size are both mmap-worthy 29 | if (g->sizeclass>=48 && n>=MMAP_THRESHOLD) { 30 | assert(g->sizeclass==63); 31 | size_t base = (unsigned char *)p-start; 32 | size_t needed = (n + base + UNIT + IB + 4095) & -4096; 33 | new = g->maplen*4096UL == needed ? g->mem : 34 | mremap(g->mem, g->maplen*4096UL, needed, MREMAP_MAYMOVE); 35 | if (new!=MAP_FAILED) { 36 | g->mem = new; 37 | g->maplen = needed/4096; 38 | p = g->mem->storage + base; 39 | end = g->mem->storage + (needed - UNIT) - IB; 40 | *end = 0; 41 | set_size(p, end, n); 42 | return p; 43 | } 44 | } 45 | 46 | new = malloc(n); 47 | if (!new) return 0; 48 | memcpy(new, p, n < old_size ? n : old_size); 49 | free(p); 50 | return new; 51 | } 52 | --------------------------------------------------------------------------------