├── .github └── workflows │ └── ci.yml ├── .gitignore ├── Cargo.toml ├── LICENSE.md ├── README.md ├── benchmarks ├── Cargo.toml ├── microbench.rs └── random_actions.rs ├── buckets.py ├── check.sh ├── fuzz ├── .gitignore ├── Cargo.toml └── fuzz_targets │ └── fuzz_talc.rs ├── graph_microbench.py ├── graph_random_actions.py ├── rustfmt.toml ├── stable_examples ├── Cargo.toml └── examples │ ├── stable_allocator_api.rs │ └── std_global_allocator.rs ├── talc ├── Cargo.toml ├── LICENSE.md ├── README.md ├── README_WASM.md ├── benchmark_graphs │ ├── microbench.png │ ├── random_actions.png │ └── random_actions_multi.png └── src │ ├── lib.rs │ ├── locking.rs │ ├── oom_handler.rs │ ├── ptr_utils.rs │ ├── span.rs │ ├── talc.rs │ ├── talc │ ├── counters.rs │ ├── llist.rs │ └── tag.rs │ └── talck.rs ├── wasm-perf.sh ├── wasm-perf ├── Cargo.toml ├── bench.js └── src │ └── lib.rs ├── wasm-size.sh └── wasm-size ├── Cargo.toml └── src └── lib.rs /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | 12 | env: 13 | CARGO_TERM_COLOR: always 14 | 15 | jobs: 16 | job1: 17 | name: Nightly + MIRI tests 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@v4 21 | - uses: dtolnay/rust-toolchain@nightly 22 | with: 23 | targets: x86_64-unknown-linux-gnu, i686-unknown-linux-gnu 24 | components: miri 25 | - uses: Swatinem/rust-cache@v2 26 | - run: | 27 | rustup run nightly cargo check -p talc --verbose 28 | 29 | rustup run nightly cargo test -p talc --features=counters --verbose 30 | rustup run nightly cargo test -p talc --tests --no-default-features --verbose 31 | rustup run nightly cargo test -p talc --tests --no-default-features --features=lock_api,allocator-api2,counters --verbose 32 | 33 | rustup run nightly cargo miri test -p talc --tests --verbose 34 | rustup run nightly cargo miri test -p talc --tests --target i686-unknown-linux-gnu --verbose 35 | 36 | rustup run nightly cargo check -p benchmarks --bin microbench --verbose 37 | rustup run nightly cargo check -p benchmarks --bin random_actions --verbose 38 | 39 | job2: 40 | name: Check Stable 41 | runs-on: ubuntu-latest 42 | steps: 43 | - uses: actions/checkout@v4 44 | - uses: dtolnay/rust-toolchain@stable 45 | with: 46 | targets: x86_64-unknown-linux-gnu, wasm32-unknown-unknown 47 | - uses: Swatinem/rust-cache@v2 48 | - run: | 49 | rustup run stable cargo check --no-default-features -p talc --verbose 50 | rustup run stable cargo check --no-default-features -p talc --features=lock_api --verbose 51 | rustup run stable cargo check --no-default-features -p talc --features=lock_api,allocator-api2,counters --verbose 52 | 53 | rustup run stable cargo check -p talc --no-default-features --target wasm32-unknown-unknown --verbose 54 | rustup run stable cargo check -p talc --no-default-features --features=lock_api,counters --target wasm32-unknown-unknown --verbose 55 | 56 | rustup run stable cargo check -p stable_examples --example stable_allocator_api --verbose 57 | rustup run stable cargo check -p stable_examples --example std_global_allocator --verbose 58 | 59 | job3: 60 | name: Check MSRV 61 | runs-on: ubuntu-latest 62 | steps: 63 | - uses: actions/checkout@v4 64 | - uses: dtolnay/rust-toolchain@stable 65 | - uses: Swatinem/rust-cache@v2 66 | - run: | 67 | rustup toolchain add 1.67.1 --profile minimal 68 | rustup run 1.67.1 cargo check -p talc --no-default-features --features lock_api,allocator-api2,counters --verbose 69 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | 3 | target 4 | Cargo.lock 5 | 6 | pkg 7 | benchmark_results 8 | 9 | *.asm 10 | perf* 11 | flamegraph.svg 12 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | 4 | members = [ 5 | "talc", 6 | "benchmarks", 7 | "fuzz", 8 | "stable_examples", 9 | "wasm-perf", 10 | "wasm-size", 11 | ] 12 | 13 | [profile.release] 14 | panic = "abort" 15 | 16 | [profile.release.package.wasm-size] 17 | opt-level = "z" 18 | codegen-units = 1 19 | 20 | # be realistic about the optimization configuration, even if it's a benchmark 21 | [profile.release.package.wasm-perf] 22 | opt-level = "z" 23 | codegen-units = 1 24 | 25 | # the fuzzer needs debuginfo 26 | [profile.release.package.talc-fuzz] 27 | debug = 1 28 | 29 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2023 Shaun Beautement 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a 6 | copy of this software and associated documentation files (the “Software”), 7 | to deal in the Software without restriction, including without limitation 8 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 | and/or sell copies of the Software, and to permit persons to whom the 10 | Software is furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included 13 | in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 | OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | talc/README.md -------------------------------------------------------------------------------- /benchmarks/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "benchmarks" 3 | version = "0.0.0" 4 | edition = "2021" 5 | publish = false 6 | 7 | [[bin]] 8 | name = "microbench" 9 | path = "microbench.rs" 10 | test = false 11 | doc = false 12 | 13 | [[bin]] 14 | name = "random_actions" 15 | path = "random_actions.rs" 16 | test = false 17 | doc = false 18 | 19 | [dependencies] 20 | fastrand = "1.9" 21 | spin = { version = "0.9.8", default-features = false, features = ["lock_api", "spin_mutex"] } 22 | talc = { path = "../talc" } 23 | linked_list_allocator = { version = "0.10", features = ["use_spin_nightly", "const_mut_refs", "alloc_ref"] } 24 | good_memory_allocator = { version = "0.1", features = ["spin", "allocator"] } 25 | buddy-alloc = "0.5" 26 | dlmalloc = { version = "0.2", default-features = false, features = ["global"] } 27 | frusa = "0.1" 28 | rlsf = "0.2" 29 | -------------------------------------------------------------------------------- /benchmarks/microbench.rs: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright (c) 2022 Philipp Schuster 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | // Heavily modified by Shaun Beautement. 26 | 27 | #![feature(iter_intersperse)] 28 | 29 | use buddy_alloc::{BuddyAllocParam, FastAllocParam}; 30 | use good_memory_allocator::DEFAULT_SMALLBINS_AMOUNT; 31 | use talc::{ErrOnOom, Talc}; 32 | 33 | use std::alloc::{GlobalAlloc, Layout}; 34 | use std::fs::File; 35 | use std::ptr::{addr_of_mut, NonNull}; 36 | use std::time::Instant; 37 | 38 | const BENCH_DURATION: f64 = 1.0; 39 | 40 | const HEAP_SIZE: usize = 0x10000000; 41 | static mut HEAP_MEMORY: [u8; HEAP_SIZE] = [0u8; HEAP_SIZE]; 42 | 43 | 44 | // Turn DlMalloc into an arena allocator 45 | struct DlmallocArena(std::sync::atomic::AtomicBool); 46 | 47 | unsafe impl dlmalloc::Allocator for DlmallocArena { 48 | fn alloc(&self, _size: usize) -> (*mut u8, usize, u32) { 49 | let has_data = self.0.fetch_and(false, core::sync::atomic::Ordering::SeqCst); 50 | 51 | if has_data { 52 | let align = std::mem::align_of::(); 53 | let heap_align_offset = addr_of_mut!(HEAP_MEMORY).align_offset(align); 54 | unsafe { (addr_of_mut!(HEAP_MEMORY).cast::().add(heap_align_offset), (HEAP_SIZE - heap_align_offset) / align * align, 1) } 55 | } else { 56 | (core::ptr::null_mut(), 0, 0) 57 | } 58 | } 59 | 60 | fn remap(&self, _ptr: *mut u8, _oldsize: usize, _newsize: usize, _can_move: bool) -> *mut u8 { 61 | unimplemented!() 62 | } 63 | 64 | fn free_part(&self, _ptr: *mut u8, _oldsize: usize, _newsize: usize) -> bool { 65 | unimplemented!() 66 | } 67 | 68 | fn free(&self, _ptr: *mut u8, _size: usize) -> bool { 69 | unimplemented!() 70 | } 71 | 72 | fn can_release_part(&self, _flags: u32) -> bool { 73 | false 74 | } 75 | 76 | fn allocates_zeros(&self) -> bool { 77 | false 78 | } 79 | 80 | fn page_size(&self) -> usize { 81 | 4 * 1024 82 | } 83 | } 84 | 85 | struct DlMallocator(spin::Mutex>); 86 | 87 | unsafe impl GlobalAlloc for DlMallocator { 88 | unsafe fn alloc(&self, layout: Layout) -> *mut u8 { 89 | self.0.lock().malloc(layout.size(), layout.align()) 90 | } 91 | 92 | unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { 93 | self.0.lock().free(ptr, layout.size(), layout.align()); 94 | } 95 | 96 | unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { 97 | self.0.lock().realloc(ptr, layout.size(), layout.align(), new_size) 98 | } 99 | 100 | unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { 101 | self.0.lock().calloc(layout.size(), layout.align()) 102 | } 103 | } 104 | 105 | struct GlobalRLSF<'p>(spin::Mutex>); 106 | unsafe impl<'a> GlobalAlloc for GlobalRLSF<'a> { 107 | unsafe fn alloc(&self, layout: Layout) -> *mut u8 { 108 | self.0.lock().allocate(layout).map_or(std::ptr::null_mut(), |nn| nn.as_ptr()) 109 | } 110 | 111 | unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { 112 | self.0.lock().deallocate(NonNull::new_unchecked(ptr), layout.align()); 113 | } 114 | 115 | unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { 116 | self.0.lock().reallocate(NonNull::new_unchecked(ptr), Layout::from_size_align_unchecked(new_size, layout.align())) 117 | .map_or(std::ptr::null_mut(), |nn| nn.as_ptr()) 118 | } 119 | } 120 | 121 | 122 | 123 | fn main() { 124 | const BENCHMARK_RESULTS_DIR: &str = "./benchmark_results/micro/"; 125 | // create a directory for the benchmark results. 126 | let _ = std::fs::create_dir_all(BENCHMARK_RESULTS_DIR).unwrap(); 127 | 128 | let sum_file = File::create(BENCHMARK_RESULTS_DIR.to_owned() + "Alloc Plus Dealloc.csv").unwrap(); 129 | let mut csvs = Csvs { sum_file }; 130 | 131 | // warm up the memory caches, avoid demand paging issues, etc. 132 | for i in 0..HEAP_SIZE { 133 | unsafe { 134 | addr_of_mut!(HEAP_MEMORY).cast::().add(i).write(0xAE); 135 | } 136 | } 137 | 138 | /* let linked_list_allocator = 139 | unsafe { linked_list_allocator::LockedHeap::new(HEAP_MEMORY.as_mut_ptr() as _, HEAP_SIZE) }; 140 | 141 | benchmark_allocator(&linked_list_allocator, "Linked List Allocator", &mut csvs); */ 142 | 143 | let mut galloc_allocator = 144 | good_memory_allocator::SpinLockedAllocator::::empty(); 145 | unsafe { 146 | galloc_allocator.init(addr_of_mut!(HEAP_MEMORY) as usize, HEAP_SIZE); 147 | } 148 | 149 | benchmark_allocator(&mut galloc_allocator, "Galloc", &mut csvs); 150 | 151 | let buddy_alloc = unsafe { 152 | buddy_alloc::NonThreadsafeAlloc::new( 153 | FastAllocParam::new(addr_of_mut!(HEAP_MEMORY).cast(), HEAP_SIZE / 8), 154 | BuddyAllocParam::new(addr_of_mut!(HEAP_MEMORY).cast::().add(HEAP_SIZE / 8), HEAP_SIZE / 8 * 7, 64), 155 | ) 156 | }; 157 | benchmark_allocator(&buddy_alloc, "Buddy Allocator", &mut csvs); 158 | 159 | let dlmalloc = DlMallocator(spin::Mutex::new( 160 | dlmalloc::Dlmalloc::new_with_allocator(DlmallocArena(true.into())) 161 | )); 162 | benchmark_allocator(&dlmalloc, "Dlmalloc", &mut csvs); 163 | 164 | let talc = Talc::new(ErrOnOom).lock::(); 165 | unsafe { talc.lock().claim(addr_of_mut!(HEAP_MEMORY).into()) }.unwrap(); 166 | 167 | benchmark_allocator(&talc, "Talc", &mut csvs); 168 | 169 | let tlsf = GlobalRLSF(spin::Mutex::new(rlsf::Tlsf::new())); 170 | tlsf.0.lock().insert_free_block(unsafe { std::mem::transmute(&mut HEAP_MEMORY[..]) }); 171 | benchmark_allocator(&tlsf, "RLSF", &mut csvs); 172 | 173 | // benchmark_allocator(&std::alloc::System, "System", &mut csvs); 174 | // benchmark_allocator(&frusa::Frusa2M::new(&std::alloc::System), "Frusa", &mut csvs); 175 | } 176 | 177 | fn now() -> u64 { 178 | #[cfg(target_arch = "x86_64")] 179 | { 180 | let mut x = 0u32; 181 | unsafe { std::arch::x86_64::__rdtscp(&mut x) } 182 | } 183 | 184 | #[cfg(target_arch = "aarch64")] 185 | { 186 | let mut timer: u64; 187 | unsafe { std::arch::asm!("mrs {0}, cntvct_el0", out(reg) timer, options(nomem, nostack)); } 188 | return timer; 189 | } 190 | 191 | #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] 192 | compile_error!( 193 | "Hardware-based counter is not implemented for this architecture. Supported: x86_64, aarch64" 194 | ); 195 | } 196 | 197 | struct Csvs { 198 | pub sum_file: File, 199 | } 200 | 201 | fn benchmark_allocator(allocator: &dyn GlobalAlloc, name: &str, csvs: &mut Csvs) { 202 | eprintln!("Benchmarking: {name}..."); 203 | 204 | let mut active_allocations = Vec::new(); 205 | 206 | let mut alloc_ticks_vec = Vec::new(); 207 | let mut dealloc_ticks_vec = Vec::new(); 208 | 209 | // warm up 210 | for i in 1..10000 { 211 | let layout = Layout::from_size_align(i * 8, 8).unwrap(); 212 | let ptr = unsafe { allocator.alloc(layout) }; 213 | assert!(!ptr.is_null()); 214 | unsafe { let _ = ptr.read_volatile(); } 215 | unsafe { allocator.dealloc(ptr, layout); } 216 | } 217 | 218 | let bench_timer = Instant::now(); 219 | for i in 0.. { 220 | if i % 0x10000 == 0 && (Instant::now() - bench_timer).as_secs_f64() > BENCH_DURATION { break; } 221 | 222 | let size = fastrand::usize(1..0x10000); 223 | let align = 8 << fastrand::u16(..).trailing_zeros() / 2; 224 | let layout = Layout::from_size_align(size, align).unwrap(); 225 | 226 | let alloc_begin = now(); 227 | let alloc = unsafe { allocator.alloc(layout) }; 228 | let alloc_ticks = now().wrapping_sub(alloc_begin); 229 | 230 | if std::ptr::null_mut() != alloc { 231 | alloc_ticks_vec.push(alloc_ticks); 232 | active_allocations.push((alloc, layout)); 233 | } else { 234 | for (ptr, layout) in active_allocations.drain(..) { 235 | let dealloc_begin = now(); 236 | unsafe { allocator.dealloc(ptr, layout); } 237 | let dealloc_ticks = now().wrapping_sub(dealloc_begin); 238 | dealloc_ticks_vec.push(dealloc_ticks); 239 | } 240 | continue; 241 | } 242 | 243 | if active_allocations.len() > 10 && fastrand::usize(..10) == 0 { 244 | for _ in 0..8 { 245 | let index = fastrand::usize(..active_allocations.len()); 246 | let allocation = active_allocations.swap_remove(index); 247 | 248 | let dealloc_begin = now(); 249 | unsafe { 250 | allocator.dealloc(allocation.0, allocation.1); 251 | } 252 | let dealloc_ticks = now().wrapping_sub(dealloc_begin); 253 | dealloc_ticks_vec.push(dealloc_ticks); 254 | } 255 | } 256 | } 257 | 258 | alloc_ticks_vec.sort_unstable(); 259 | dealloc_ticks_vec.sort_unstable(); 260 | let alloc_ticks = alloc_ticks_vec.into_iter().map(|x| x as f64).collect::>(); 261 | let dealloc_ticks = dealloc_ticks_vec.into_iter().map(|x| x as f64).collect::>(); 262 | let filtered_alloc_ticks = filter_sorted_outliers(&alloc_ticks); 263 | let filtered_dealloc_ticks = filter_sorted_outliers(&dealloc_ticks); 264 | 265 | let alloc_quartiles = quartiles(filtered_alloc_ticks); 266 | let dealloc_quartiles = quartiles(filtered_dealloc_ticks); 267 | let mut sum_quartiles = [0.0; 5]; 268 | for i in 0..sum_quartiles.len() { sum_quartiles[i] = alloc_quartiles[i] + dealloc_quartiles[i] } 269 | 270 | let data_to_string = |data: &[f64]| 271 | String::from_iter(data.into_iter().map(|x| x.to_string()).intersperse(",".to_owned())); 272 | 273 | use std::io::Write; 274 | writeln!(csvs.sum_file, "{name},{}", data_to_string(&sum_quartiles[..])).unwrap(); 275 | 276 | } 277 | 278 | fn filter_sorted_outliers(samples: &[f64]) -> &[f64] { 279 | // filter extreme outliers 280 | // these might occur due to system interrupts or rescheduling 281 | 282 | let mean = samples.iter().sum::() / samples.len() as f64; 283 | let var = samples.iter().map(|&x| (x - mean) * (x - mean)).sum::() / samples.len() as f64; 284 | let stddev = var.sqrt(); 285 | let filter_limit = mean + stddev * 50.0; 286 | 287 | let mut i = samples.len(); 288 | while i > 0 { 289 | i -= 1; 290 | 291 | if samples[i] < filter_limit { 292 | return &samples[..=i]; 293 | } 294 | } 295 | 296 | unreachable!() 297 | } 298 | 299 | fn quartiles(data: &[f64]) -> [f64; 5] { 300 | let len = data.len(); 301 | [data[0], data[len/4], data[len/2], data[3*len/4], data[len-1]] 302 | } 303 | -------------------------------------------------------------------------------- /benchmarks/random_actions.rs: -------------------------------------------------------------------------------- 1 | /* The MIT License (MIT) 2 | 3 | Copyright © 2023 Roee Shoshani, Guy Nir 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the “Software”), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included 13 | in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | // Modified by Shaun Beautement 24 | 25 | #![feature(iter_intersperse)] 26 | 27 | use std::{ 28 | alloc::{GlobalAlloc, Layout}, fmt::Write, ptr::{addr_of_mut, NonNull}, sync::{Arc, Barrier}, time::{Duration, Instant} 29 | }; 30 | 31 | use buddy_alloc::{BuddyAllocParam, FastAllocParam, NonThreadsafeAlloc}; 32 | 33 | const THREAD_COUNT: usize = 12; 34 | 35 | const RA_TRIALS_AMOUNT: usize = 7; 36 | const RA_TIME: Duration = Duration::from_millis(200); 37 | const RA_MAX_ALLOC_SIZES: &[usize] = &[1000, 5000, 10000, 50000, 100000]; 38 | const RA_MAX_REALLOC_SIZE_MULTI: usize = 10; 39 | const RA_TARGET_MIN_ALLOCATIONS: usize = 300; 40 | 41 | const HE_MAX_ALLOC_SIZE: usize = 100000; 42 | const HE_MAX_REALLOC_SIZE_MULTI: usize = 1000000; 43 | 44 | const HEAP_SIZE: usize = 1 << 29; 45 | static mut HEAP: [u8; HEAP_SIZE] = [0u8; HEAP_SIZE]; 46 | 47 | const BENCHMARK_RESULTS_DIR: &str = "./benchmark_results"; 48 | 49 | struct NamedAllocator { 50 | name: &'static str, 51 | init_fn: unsafe fn() -> Box, 52 | } 53 | 54 | fn main() { 55 | // create a directory for the benchmark results. 56 | let _ = std::fs::create_dir(BENCHMARK_RESULTS_DIR); 57 | 58 | let allocators = &[ 59 | NamedAllocator { name: "Talc", init_fn: init_talc }, 60 | NamedAllocator { name: "RLSF", init_fn: init_rlsf }, 61 | NamedAllocator { name: "Frusa", init_fn: init_frusa }, 62 | NamedAllocator { name: "Dlmalloc", init_fn: init_dlmalloc }, 63 | NamedAllocator { name: "System", init_fn: init_system }, 64 | NamedAllocator { name: "Buddy Alloc", init_fn: init_buddy_alloc }, 65 | NamedAllocator { name: "Linked List", init_fn: init_linked_list_allocator }, 66 | ]; 67 | 68 | print!("Run heap efficiency benchmarks? y/N: "); 69 | std::io::Write::flush(&mut std::io::stdout()).unwrap(); 70 | let mut input = String::new(); 71 | std::io::stdin().read_line(&mut input).unwrap(); 72 | if input.trim() == "y" { 73 | // heap efficiency benchmark 74 | 75 | println!("| Allocator | Average Random Actions Heap Efficiency |"); 76 | println!("| --------------------- | -------------------------------------- |"); 77 | 78 | for allocator in allocators { 79 | // these request memory from the OS on-demand, instead of being arena-allocated 80 | if matches!(allocator.name, "Frusa" | "System") { continue; } 81 | 82 | let efficiency = heap_efficiency(unsafe {(allocator.init_fn)() }.as_ref()); 83 | 84 | println!("|{:>22} | {:>38} |", allocator.name, format!("{:2.2}%", efficiency)); 85 | } 86 | } 87 | 88 | let mut csv = String::new(); 89 | 90 | write!(csv, ",").unwrap(); 91 | csv.extend(RA_MAX_ALLOC_SIZES.iter().map(|i| i.to_string()).intersperse(",".to_owned())); 92 | writeln!(csv).unwrap(); 93 | 94 | for &NamedAllocator { name, init_fn } in allocators { 95 | write!(csv, "{}", name).unwrap(); 96 | 97 | for &max_alloc_size in RA_MAX_ALLOC_SIZES.iter() { 98 | eprintln!("benchmarking {} - max alloc size {}B ...", name, max_alloc_size); 99 | 100 | let score = (0..RA_TRIALS_AMOUNT) 101 | .map(|_| { 102 | let allocator = unsafe { (init_fn)() }; 103 | let allocator_ref = allocator.as_ref(); 104 | 105 | std::thread::scope(|scope| { 106 | let barrier = Arc::new(Barrier::new(THREAD_COUNT)); 107 | let mut handles = vec![]; 108 | 109 | for _ in 0..THREAD_COUNT { 110 | let bi = barrier.clone(); 111 | handles.push(scope.spawn(move || random_actions( allocator_ref, max_alloc_size, bi))); 112 | } 113 | 114 | handles.into_iter().map(|h| h.join().unwrap()).sum::() 115 | }) 116 | }).sum::() / RA_TRIALS_AMOUNT; 117 | 118 | write!(csv, ",{}", score).unwrap(); 119 | } 120 | 121 | writeln!(csv).unwrap(); 122 | } 123 | // remove the last newline. 124 | csv.pop(); 125 | 126 | std::fs::write(format!("{}/Random Actions Benchmark.csv", BENCHMARK_RESULTS_DIR), csv).unwrap(); 127 | } 128 | 129 | 130 | pub fn random_actions(allocator: &dyn GlobalAlloc, max_alloc_size: usize, barrier: Arc) -> usize { 131 | let mut score = 0; 132 | let mut v: Vec> = Vec::with_capacity(100000); 133 | let rng = fastrand::Rng::new(); 134 | 135 | barrier.wait(); 136 | let start = Instant::now(); 137 | while start.elapsed() < RA_TIME { 138 | for _ in 0..100 { 139 | let action = rng.usize(0..5); 140 | 141 | // 20% reallocate 142 | // 40% if there are enough allocations, deallocate 143 | // 40% if enough allocations else 80%, allocate 144 | 145 | // this avoids staying close to zero allocations 146 | // while also avoiding growing the heap unboundedly 147 | // as benchmarking high heap contention isn't usually relavent 148 | // but having a very low number of allocations isn't realistic either 149 | 150 | if action == 4 { 151 | if !v.is_empty() { 152 | let index = rng.usize(0..v.len()); 153 | if let Some(random_allocation) = v.get_mut(index) { 154 | let size = rng.usize(1..(max_alloc_size * RA_MAX_REALLOC_SIZE_MULTI)); 155 | random_allocation.realloc(size); 156 | } else { 157 | eprintln!("Reallocation failure!"); 158 | } 159 | score += 1; 160 | } 161 | } else if action < 2 || v.len() < RA_TARGET_MIN_ALLOCATIONS { 162 | let size = rng.usize(1..max_alloc_size); 163 | let alignment = std::mem::align_of::() << rng.u16(..).trailing_zeros() / 2; 164 | if let Some(allocation) = AllocationWrapper::new(size, alignment, allocator) { 165 | v.push(allocation); 166 | score += 1; 167 | } else { 168 | eprintln!("Allocation failure!"); 169 | } 170 | } else { 171 | let index = rng.usize(0..v.len()); 172 | v.swap_remove(index); 173 | score += 1; 174 | } 175 | } 176 | } 177 | 178 | score 179 | } 180 | 181 | pub fn heap_efficiency(allocator: &dyn GlobalAlloc) -> f64 { 182 | let mut v = Vec::with_capacity(100000); 183 | let mut used = 0; 184 | let mut total = 0; 185 | 186 | for _ in 0..300 { 187 | loop { 188 | let action = fastrand::usize(0..10); 189 | 190 | match action { 191 | 0..=4 => { 192 | let size = fastrand::usize(1..HE_MAX_ALLOC_SIZE); 193 | let align = std::mem::align_of::() << fastrand::u16(..).trailing_zeros() / 2; 194 | 195 | if let Some(allocation) = AllocationWrapper::new(size, align, allocator) { 196 | v.push(allocation); 197 | } else { 198 | used += v.iter().map(|a| a.layout.size()).sum::(); 199 | total += HEAP_SIZE; 200 | v.clear(); 201 | break; 202 | } 203 | } 204 | 5 => { 205 | if !v.is_empty() { 206 | let index = fastrand::usize(0..v.len()); 207 | v.swap_remove(index); 208 | } 209 | } 210 | 6..=9 => { 211 | if !v.is_empty() { 212 | let index = fastrand::usize(0..v.len()); 213 | 214 | if let Some(random_allocation) = v.get_mut(index) { 215 | let new_size = fastrand::usize(1..(HE_MAX_ALLOC_SIZE*HE_MAX_REALLOC_SIZE_MULTI)); 216 | random_allocation.realloc(new_size); 217 | } else { 218 | used += v.iter().map(|a| a.layout.size()).sum::(); 219 | total += HEAP_SIZE; 220 | v.clear(); 221 | break; 222 | } 223 | } 224 | } 225 | _ => unreachable!(), 226 | } 227 | } 228 | } 229 | 230 | used as f64 / total as f64 * 100.0 231 | } 232 | 233 | struct AllocationWrapper<'a> { 234 | ptr: *mut u8, 235 | layout: Layout, 236 | allocator: &'a dyn GlobalAlloc, 237 | } 238 | impl<'a> AllocationWrapper<'a> { 239 | fn new(size: usize, align: usize, allocator: &'a dyn GlobalAlloc) -> Option { 240 | let layout = Layout::from_size_align(size, align).unwrap(); 241 | 242 | let ptr = unsafe { (*allocator).alloc(layout) }; 243 | 244 | if ptr.is_null() { 245 | return None; 246 | } 247 | 248 | Some(Self { ptr, layout, allocator }) 249 | } 250 | 251 | fn realloc(&mut self, new_size: usize) { 252 | let new_ptr = unsafe { (*self.allocator).realloc(self.ptr, self.layout, new_size) }; 253 | if new_ptr.is_null() { 254 | return; 255 | } 256 | self.ptr = new_ptr; 257 | self.layout = Layout::from_size_align(new_size, self.layout.align()).unwrap(); 258 | } 259 | } 260 | 261 | impl<'a> Drop for AllocationWrapper<'a> { 262 | fn drop(&mut self) { 263 | unsafe { (*self.allocator).dealloc(self.ptr, self.layout) } 264 | } 265 | } 266 | 267 | 268 | 269 | /// Memory must be available. 270 | unsafe fn init_talc() -> Box { 271 | unsafe { 272 | let talck: _ = talc::Talc::new(talc::ErrOnOom).lock::>(); 273 | talck.lock().claim(addr_of_mut!(HEAP).into()).unwrap(); 274 | Box::new(talck) 275 | } 276 | } 277 | 278 | unsafe fn init_linked_list_allocator() -> Box { 279 | let lla = linked_list_allocator::LockedHeap::new(addr_of_mut!(HEAP).cast(), HEAP_SIZE); 280 | lla.lock().init(addr_of_mut!(HEAP).cast(), HEAP_SIZE); 281 | Box::new(lla) 282 | } 283 | 284 | unsafe fn init_system() -> Box { 285 | Box::new(std::alloc::System) 286 | } 287 | 288 | unsafe fn init_frusa() -> Box { 289 | Box::new(frusa::Frusa2M::new(&std::alloc::System)) 290 | } 291 | 292 | #[allow(unused)] 293 | unsafe fn init_galloc() -> Box { 294 | let galloc = good_memory_allocator::SpinLockedAllocator 295 | ::<{good_memory_allocator::DEFAULT_SMALLBINS_AMOUNT}, {good_memory_allocator::DEFAULT_ALIGNMENT_SUB_BINS_AMOUNT}> 296 | ::empty(); 297 | galloc.init(addr_of_mut!(HEAP) as usize, HEAP_SIZE); 298 | Box::new(galloc) 299 | } 300 | 301 | unsafe fn init_rlsf() -> Box { 302 | let tlsf = GlobalRLSF(spin::Mutex::new(rlsf::Tlsf::new())); 303 | tlsf.0.lock().insert_free_block(unsafe { std::mem::transmute(&mut HEAP[..]) }); 304 | Box::new(tlsf) 305 | } 306 | 307 | unsafe fn init_buddy_alloc() -> Box { 308 | let ba = BuddyAllocWrapper(spin::Mutex::new(NonThreadsafeAlloc::new( 309 | FastAllocParam::new(addr_of_mut!(HEAP).cast(), HEAP_SIZE / 8), 310 | BuddyAllocParam::new( 311 | addr_of_mut!(HEAP).cast::().add(HEAP_SIZE / 8), 312 | HEAP_SIZE / 8 * 7, 313 | 64, 314 | ), 315 | ))); 316 | 317 | Box::new(ba) 318 | } 319 | 320 | unsafe fn init_dlmalloc() -> Box { 321 | let dl = DlMallocator(spin::Mutex::new( 322 | dlmalloc::Dlmalloc::new_with_allocator(DlmallocArena(spin::Mutex::new(false))), 323 | )); 324 | Box::new(dl) 325 | } 326 | 327 | struct BuddyAllocWrapper(pub spin::Mutex); 328 | 329 | unsafe impl Send for BuddyAllocWrapper {} 330 | unsafe impl Sync for BuddyAllocWrapper {} 331 | 332 | unsafe impl GlobalAlloc for BuddyAllocWrapper { 333 | unsafe fn alloc(&self, layout: Layout) -> *mut u8 { self.0.lock().alloc(layout) } 334 | 335 | unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { self.0.lock().dealloc(ptr, layout) } 336 | 337 | unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { self.0.lock().alloc_zeroed(layout) } 338 | 339 | unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { 340 | self.0.lock().realloc(ptr, layout, new_size) 341 | } 342 | } 343 | 344 | struct DlMallocator(spin::Mutex>); 345 | 346 | unsafe impl GlobalAlloc for DlMallocator { 347 | unsafe fn alloc(&self, layout: Layout) -> *mut u8 { 348 | self.0.lock().malloc(layout.size(), layout.align()) 349 | } 350 | 351 | unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { 352 | self.0.lock().free(ptr, layout.size(), layout.align()); 353 | } 354 | 355 | unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { 356 | self.0.lock().realloc(ptr, layout.size(), layout.align(), new_size) 357 | } 358 | 359 | unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { 360 | self.0.lock().calloc(layout.size(), layout.align()) 361 | } 362 | } 363 | 364 | // Turn DlMalloc into an arena allocator 365 | struct DlmallocArena(spin::Mutex); 366 | 367 | unsafe impl dlmalloc::Allocator for DlmallocArena { 368 | fn alloc(&self, _: usize) -> (*mut u8, usize, u32) { 369 | let mut lock = self.0.lock(); 370 | 371 | if *lock { 372 | (core::ptr::null_mut(), 0, 0) 373 | } else { 374 | *lock = true; 375 | let align = std::mem::align_of::(); 376 | let heap_align_offset = addr_of_mut!(HEAP).align_offset(align); 377 | (unsafe { addr_of_mut!(HEAP).cast::().add(heap_align_offset) }, (HEAP_SIZE - heap_align_offset) / align * align, 1) 378 | } 379 | } 380 | 381 | fn remap(&self, _ptr: *mut u8, _oldsize: usize, _newsize: usize, _can_move: bool) -> *mut u8 { 382 | unimplemented!() 383 | } 384 | 385 | fn free_part(&self, _ptr: *mut u8, _oldsize: usize, _newsize: usize) -> bool { 386 | unimplemented!() 387 | } 388 | 389 | fn free(&self, _ptr: *mut u8, _size: usize) -> bool { 390 | true 391 | } 392 | 393 | fn can_release_part(&self, _flags: u32) -> bool { 394 | false 395 | } 396 | 397 | fn allocates_zeros(&self) -> bool { 398 | false 399 | } 400 | 401 | fn page_size(&self) -> usize { 402 | 4 * 1024 403 | } 404 | } 405 | 406 | struct GlobalRLSF<'p>(spin::Mutex>); 407 | unsafe impl<'a> GlobalAlloc for GlobalRLSF<'a> { 408 | unsafe fn alloc(&self, layout: Layout) -> *mut u8 { 409 | self.0.lock().allocate(layout).map_or(std::ptr::null_mut(), |nn| nn.as_ptr()) 410 | } 411 | 412 | unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { 413 | self.0.lock().deallocate(NonNull::new_unchecked(ptr), layout.align()); 414 | } 415 | 416 | unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { 417 | self.0.lock().reallocate(NonNull::new_unchecked(ptr), Layout::from_size_align_unchecked(new_size, layout.align())) 418 | .map_or(std::ptr::null_mut(), |nn| nn.as_ptr()) 419 | } 420 | } 421 | -------------------------------------------------------------------------------- /buckets.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | # modify these parameters to determine the bucketing strategy 4 | # the main things we want are 5 | # - coverage up to the 100MiB-1GiB area 6 | # - minimize number of allocation sizes per bucket 7 | # - facilitate particularly quick allocation of small sizes 8 | # - don't sacrifice the speed of large allocations much 9 | 10 | ## for 32-bit machines: 11 | #word_size = 4 12 | #word_buckets_limit = 64 13 | #double_buckets_limit = 128 14 | #exp_fractions = 2 15 | 16 | # for 64-bit machines: 17 | word_size = 8 18 | word_buckets_limit = 256 19 | double_buckets_limit = 512 20 | exp_fractions = 4 21 | 22 | 23 | # the rest of this 24 | 25 | min_chunk_size = word_size * 3 26 | 27 | word_bins_count = (word_buckets_limit - min_chunk_size) // word_size 28 | print("word bins count:", word_bins_count) 29 | 30 | for i, sb in enumerate(range(min_chunk_size, word_buckets_limit, word_size)): 31 | print("{1:>3}: {0:>8} {0:>20b} | ".format(sb, i), end='\n') 32 | 33 | double_bins_count = (double_buckets_limit - word_buckets_limit) // (2*word_size) 34 | print("double bins count:", double_bins_count) 35 | 36 | for i, bsb in enumerate(range(word_buckets_limit, double_buckets_limit, 2*word_size)): 37 | print("{1:>3}: {0:>8} {0:>20b} | ".format(bsb, i), end='\n') 38 | 39 | print("pseudo log-spaced bins") 40 | 41 | b_ofst = int(math.log2(double_buckets_limit)) # log2_start_pow | 16 42 | b_p2dv = int(math.log2(exp_fractions)) # log2_div_count | 4 43 | 44 | for b in range(0, (word_size * 8 * 2) - word_bins_count - double_bins_count): 45 | # calculation for size from b 46 | size = ((1 << b_p2dv) + (b & ((1<> b_p2dv) + (b_ofst-b_p2dv)) 47 | 48 | # calculation of b from size 49 | size_log2 = math.floor(math.log2(size)) 50 | b_calc = ((size >> size_log2 - b_p2dv) ^ (1<3}: {0:>8} {0:>20b} | ".format(size, b + word_bins_count + double_bins_count), end='\n') 56 | -------------------------------------------------------------------------------- /check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euxo pipefail 4 | 5 | # This is the whole kitchen sink to help ensure builds are ready to be published. 6 | 7 | # STABLE CONFIGURATIONS 8 | 9 | rustup run stable cargo check --no-default-features -p talc 10 | rustup run stable cargo check --no-default-features -p talc --features=lock_api 11 | rustup run stable cargo check --no-default-features -p talc --features=lock_api,allocator-api2,counters 12 | 13 | rustup run stable cargo check -p talc --no-default-features --target wasm32-unknown-unknown 14 | rustup run stable cargo check -p talc --no-default-features --features=lock_api,counters --target wasm32-unknown-unknown 15 | 16 | # check that the examples work 17 | rustup run stable cargo check -p stable_examples --example stable_allocator_api 18 | rustup run stable cargo check -p stable_examples --example std_global_allocator 19 | 20 | # check whether MSRV has been broken 21 | rustup run 1.67.1 cargo check -p talc --no-default-features --features lock_api,allocator-api2,counters 22 | 23 | 24 | # NIGHTLY CONFIGURATIONS 25 | 26 | rustup run nightly cargo check -p talc 27 | 28 | rustup run nightly cargo test -p talc --features=counters 29 | rustup run nightly cargo test -p talc --tests --no-default-features 30 | rustup run nightly cargo test -p talc --tests --no-default-features --features=lock_api,allocator-api2,counters 31 | 32 | rustup run nightly cargo miri test -p talc --tests 33 | rustup run nightly cargo miri test -p talc --tests --target i686-unknown-linux-gnu 34 | 35 | # check the benchmarks 36 | rustup run nightly cargo check -p benchmarks --bin microbench 37 | rustup run nightly cargo check -p benchmarks --bin random_actions 38 | 39 | 40 | # WASM BENCHMARKS CHECK 41 | 42 | # check wasm size benches 43 | ./wasm-size.sh check 44 | 45 | # check wasm perf benches 46 | ./wasm-perf.sh check 47 | -------------------------------------------------------------------------------- /fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | corpus 3 | artifacts 4 | coverage 5 | -------------------------------------------------------------------------------- /fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "talc-fuzz" 3 | version = "0.0.0" 4 | publish = false 5 | edition = "2021" 6 | 7 | [package.metadata] 8 | cargo-fuzz = true 9 | 10 | [dependencies] 11 | libfuzzer-sys = "0.4" 12 | arbitrary = { version = "1", features = ["derive"] } 13 | spin = "0.9.8" 14 | rand = "0.8.5" 15 | 16 | [dependencies.talc] 17 | path = "../talc" 18 | features = ["fuzzing", "counters"] 19 | 20 | [[bin]] 21 | name = "fuzz_talc" 22 | path = "fuzz_targets/fuzz_talc.rs" 23 | test = false 24 | doc = false 25 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/fuzz_talc.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | #![feature(allocator_api)] 4 | #![feature(slice_ptr_get)] 5 | 6 | use std::alloc::{alloc, dealloc, GlobalAlloc, Layout}; 7 | use std::ptr; 8 | 9 | use talc::*; 10 | 11 | use libfuzzer_sys::fuzz_target; 12 | 13 | use libfuzzer_sys::arbitrary::Arbitrary; 14 | 15 | #[derive(Arbitrary, Debug)] 16 | enum Actions { 17 | /// Allocate memory with the given size and align of 1 << (align % 12) 18 | Alloc { size: u16, align_bit: u8 }, 19 | /// Dealloc the ith allocation 20 | Dealloc { index: u8 }, 21 | /// Realloc the ith allocation 22 | Realloc { index: u8, new_size: u16 }, 23 | /// Claim a new segment of memory 24 | Claim { offset: u16, size: u16, capacity: u16 }, 25 | // Extend the ith heap by the additional amount specified on the low and high side 26 | Extend { index: u8, low: u16, high: u16 }, 27 | // Truncate the ith heap by the additional amount specified on the low and high side 28 | Truncate { index: u8, low: u16, high: u16 }, 29 | } 30 | use Actions::*; 31 | 32 | fuzz_target!(|actions: Vec| { 33 | let allocator = Talc::new(ErrOnOom).lock::>(); 34 | 35 | let mut allocations: Vec<(*mut u8, Layout)> = vec![]; 36 | let mut heaps: Vec<(*mut u8, Layout, Span)> = vec![]; 37 | 38 | for action in actions { 39 | match action { 40 | Alloc { size, align_bit } => { 41 | if size == 0 || align_bit > 12 { continue; } 42 | //eprintln!("ALLOC | size: {:x} align: {:x}", size as usize, 1 << align_bit % 12); 43 | 44 | let layout = Layout::from_size_align(size as usize, 1 << align_bit).unwrap(); 45 | let ptr = unsafe { allocator.alloc(layout) }; 46 | 47 | if ptr::null_mut() != ptr { 48 | allocations.push((ptr, layout)); 49 | unsafe { ptr.write_bytes(0xab, layout.size()); } 50 | } 51 | } 52 | Dealloc { index } => { 53 | if index as usize >= allocations.len() { continue; } 54 | 55 | let (ptr, layout) = allocations[index as usize]; 56 | 57 | //eprintln!("DEALLOC | ptr: {:p} size: {:x} align: {:x}", ptr, layout.size(), layout.align()); 58 | unsafe { allocator.dealloc(ptr, layout); } 59 | allocations.swap_remove(index as usize); 60 | } 61 | Realloc { index, new_size } => { 62 | if index as usize >= allocations.len() { continue; } 63 | if new_size == 0 { continue; } 64 | 65 | let (ptr, old_layout) = allocations[index as usize]; 66 | 67 | //eprintln!("REALLOC | ptr: {:p} old size: {:x} old align: {:x} new_size: {:x}", ptr, old_layout.size(), old_layout.align(), new_size as usize); 68 | 69 | let new_layout = Layout::from_size_align(new_size as usize, old_layout.align()).unwrap(); 70 | 71 | let ptr = unsafe { allocator.realloc(ptr, old_layout, new_size as usize) }; 72 | 73 | if !ptr.is_null() { 74 | allocations[index as usize] = (ptr, new_layout); 75 | if old_layout.size() < new_size as usize { 76 | unsafe { ptr.add(old_layout.size()).write_bytes(0xcd, new_size as usize - old_layout.size()); } 77 | } 78 | } 79 | }, 80 | Claim { offset, size, capacity } => { 81 | if capacity == 0 { continue; } 82 | 83 | let capacity = capacity as usize; 84 | 85 | let mem_layout = Layout::from_size_align(capacity, 1).unwrap(); 86 | let mem = unsafe { alloc(mem_layout) }; 87 | assert!(!mem.is_null()); 88 | 89 | let size = size as usize % capacity; 90 | let offset = if size == capacity { 0 } else { offset as usize % (capacity - size) }; 91 | 92 | let heap = Span::from_base_size(mem, mem_layout.size()) 93 | .truncate(offset, capacity - size + offset); 94 | let heap = unsafe { allocator.lock().claim(heap) }; 95 | 96 | if let Ok(heap) = heap { 97 | heaps.push((mem, mem_layout, heap)); 98 | } else { 99 | unsafe { dealloc(mem, mem_layout); } 100 | } 101 | }, 102 | Extend { index, low, high } => { 103 | //eprintln!("EXTEND | low: {} high: {} old arena {}", low, high, allocator.talc().get_arena()); 104 | 105 | let index = index as usize; 106 | if index >= heaps.len() { continue; } 107 | 108 | let (mem, mem_layout, old_heap) = heaps[index]; 109 | 110 | let new_heap = old_heap.extend(low as usize, high as usize) 111 | .fit_within(Span::from_base_size(mem, mem_layout.size())); 112 | let new_heap = unsafe { allocator.lock().extend(old_heap, new_heap) }; 113 | 114 | heaps[index].2 = new_heap; 115 | }, 116 | Truncate { index, low, high } => { 117 | //eprintln!("TRUNCATE | low: {} high: {} old arena {}", low, high, allocator.talc().get_arena()); 118 | 119 | let index = index as usize; 120 | if index >= heaps.len() { continue; } 121 | 122 | let old_heap = heaps[index].2; 123 | 124 | let mut talc = allocator.lock(); 125 | 126 | let new_heap = old_heap 127 | .truncate(low as usize, high as usize) 128 | .fit_over(unsafe { talc.get_allocated_span(old_heap) }); 129 | let new_heap = unsafe { talc.truncate(old_heap, new_heap) }; 130 | 131 | if new_heap.is_empty() { 132 | let (mem, mem_layout, _) = heaps.swap_remove(index); 133 | unsafe { dealloc(mem, mem_layout); } 134 | } else { 135 | heaps[index].2 = new_heap; 136 | } 137 | } 138 | } 139 | } 140 | 141 | // Free any remaining allocations. 142 | for (ptr, layout) in allocations { 143 | //eprintln!("DEALLOC FINAL | ptr: {:p} size: {:x} align: {:x}", ptr, layout.size(), layout.align()); 144 | unsafe { allocator.dealloc(ptr, layout); } 145 | } 146 | 147 | // drop the remaining heaps 148 | for (mem, mem_layout, _) in heaps { 149 | unsafe { dealloc(mem, mem_layout); } 150 | } 151 | }); 152 | -------------------------------------------------------------------------------- /graph_microbench.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | from matplotlib.lines import Line2D 3 | import os 4 | 5 | BENCHMARK_RESULTS_DIR = 'benchmark_results/micro/' 6 | BENCHMARK_RESULT_GRAPHS_DIR = 'talc/benchmark_graphs/' 7 | 8 | def get_benchmark_data(filename): 9 | print("reading", filename) 10 | with open(filename, 'r') as f: 11 | rows = f.readlines() 12 | 13 | allocators = [] 14 | for row in rows: 15 | lst = row.strip().split(',') 16 | if lst[1] != "": 17 | allocators.append((lst[0], [float(i) for i in lst[1:]])) 18 | return allocators 19 | 20 | def plot_data(filename, filepath): 21 | allocators = get_benchmark_data(filepath) 22 | if len(allocators) == 0: 23 | return 24 | 25 | # sort by median 26 | allocators.sort(key=lambda a: -a[1][2]) 27 | 28 | print("plotting", filename) 29 | 30 | rightlim = max([x[1][3] for x in allocators])*1.2 31 | 32 | plt.boxplot([x[1] for x in allocators], sym="", vert=False, showmeans=False, meanline=False, whis=(0, 100)) 33 | 34 | i = 1 35 | for al in allocators: 36 | plt.annotate(str(int(al[1][4])), (rightlim - 50, i), ) 37 | i += 1 38 | 39 | plt.title(filename.split(".")[0]) 40 | plt.xlim(left=0, right=rightlim) 41 | plt.yticks(range(1, len(allocators) + 1), [x[0] for x in allocators]) 42 | plt.xlabel("Ticks") 43 | 44 | plt.tight_layout() 45 | plt.show() 46 | 47 | 48 | def main(): 49 | if not os.path.exists(BENCHMARK_RESULTS_DIR): 50 | print("No results dir. Has the benchmark been run?") 51 | return 52 | 53 | for filename in os.listdir(BENCHMARK_RESULTS_DIR): 54 | filepath = BENCHMARK_RESULTS_DIR + filename 55 | if not os.path.isfile(filepath): 56 | continue 57 | 58 | plot_data(filename, filepath) 59 | 60 | print("complete") 61 | 62 | if __name__ == '__main__': 63 | main() 64 | 65 | -------------------------------------------------------------------------------- /graph_random_actions.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import os 3 | 4 | BENCHMARK_RESULTS_DIR = 'benchmark_results/' 5 | BENCHMARK_RESULT_GRAPHS_DIR = 'talc/benchmark_graphs/' 6 | 7 | def get_benchmark_data(filename): 8 | with open(filename, 'r') as f: 9 | rows = f.readlines() 10 | 11 | max_sizes = [int(i) for i in rows[0].split(',')[1:]] 12 | 13 | allocators = {} 14 | for row in rows[1:]: 15 | lst = row.split(',') 16 | allocators[lst[0]] = [float(i) for i in lst[1:]] 17 | 18 | return max_sizes, allocators 19 | 20 | def main(): 21 | if not os.path.exists(BENCHMARK_RESULTS_DIR): 22 | os.mkdir(BENCHMARK_RESULTS_DIR) 23 | 24 | filename = "Random Actions Benchmark.csv" 25 | 26 | max_sizes, data = get_benchmark_data(BENCHMARK_RESULTS_DIR + filename) 27 | 28 | yvalues = [] 29 | for k, v in data.items(): 30 | plt.plot(max_sizes, v, label=k) 31 | yvalues.append(v) 32 | 33 | plt.xscale('log') 34 | plt.yscale('log') 35 | plt.legend() 36 | 37 | plt.title(filename[:filename.find('.csv')]) 38 | plt.xticks(ticks=max_sizes, labels=[str(x) + " / " + str(x*10) for x in max_sizes], rotation=15) 39 | plt.xlabel('Max Allocation Size (bytes) / Max Reallocation Size (bytes)') 40 | plt.ylabel('score') 41 | 42 | plt.tight_layout() 43 | plt.show() 44 | 45 | if __name__ == '__main__': 46 | main() 47 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | use_small_heuristics = "Max" 2 | version = "Two" 3 | -------------------------------------------------------------------------------- /stable_examples/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "stable_examples" 3 | version = "0.0.0" 4 | edition = "2021" 5 | publish = false 6 | 7 | [dev-dependencies] 8 | allocator-api2 = "0.2.18" 9 | spin = "0.9.8" 10 | 11 | # put talc into a stable-compatible configuration 12 | # TODO: make talc's features stable-by-default 13 | [dev-dependencies.talc] 14 | path = "../talc" 15 | default-features = false 16 | features = ["lock_api", "allocator-api2"] -------------------------------------------------------------------------------- /stable_examples/examples/stable_allocator_api.rs: -------------------------------------------------------------------------------- 1 | use talc::{ErrOnOom, Talc}; 2 | use allocator_api2::vec::Vec; 3 | 4 | // This uses the `allocator-api2` crate to compile successfully on stable Rust. 5 | 6 | // Run with: 7 | // `cargo +stable run -p stable_examples --example stable_allocator_api` 8 | // `cargo miri run -p stable_examples --example stable_allocator_api` 9 | 10 | // TODO: Change talc's configuration to be stable-by-default, which will 11 | // avoid `rust-analyzer` complaining here about `Talck` not implementing 12 | // `allocator_api2::alloc::Allocator` if `rust-analyzer` is using a nightly 13 | // toolchain. 14 | 15 | fn main() { 16 | // Establish some memory for the allocator. 17 | let mut arena = [0u8; 10000]; 18 | 19 | // Create the allocator and "claim" the memory. 20 | let talck = Talc::new(ErrOnOom).lock::>(); 21 | 22 | // We know the memory is fine for use (unsafe) and that it's big enough for the metadata (unwrap). 23 | let heap = unsafe { 24 | talck.lock().claim(arena.as_mut().into()).unwrap() 25 | }; 26 | 27 | // Allocate, grow, shrink 28 | let mut vec = Vec::with_capacity_in(100, &talck); 29 | vec.extend(0..300usize); 30 | vec.truncate(100); 31 | vec.shrink_to_fit(); 32 | 33 | // --- Resize the arena while allocations are active! --- // 34 | 35 | // Let's see how to shrink the arena, as this is more complicated than extending it, 36 | // as we need to respect the allocations that are currently present. 37 | 38 | // First, lock the allocator. We don't want a race condition between 39 | // getting the allocated span (see below) and truncating. 40 | // If the minimum heap span changes and we try to truncate to an invalid 41 | // heap, a panic will occur. 42 | let mut talc = talck.lock(); 43 | 44 | // Retrieve the shrink-wrapped span of memory in this heap. 45 | let allocated_span = unsafe { talc.get_allocated_span(heap) }; 46 | 47 | // Let's say we want to leave only a little bit of memory on either side, 48 | // and free the rest of the heap. 49 | // Additionally, make sure we don't "truncate" to beyond the original heap's boundary. 50 | let new_heap = allocated_span.extend(200, 200).fit_within(heap); 51 | 52 | // Finally, truncate the heap! 53 | let _heap2 = unsafe { 54 | talc.truncate(heap, new_heap) 55 | }; 56 | 57 | // and we're done! 58 | drop(talc); 59 | 60 | // deallocate vec 61 | drop(vec); 62 | } -------------------------------------------------------------------------------- /stable_examples/examples/std_global_allocator.rs: -------------------------------------------------------------------------------- 1 | use std::ptr::addr_of; 2 | 3 | use talc::*; 4 | 5 | // Run with: 6 | // `cargo +stable run -p stable_examples --example std_global_allocator` 7 | // `cargo miri run -p stable_examples --example std_global_allocator` 8 | 9 | // Notes: 10 | // 11 | // ## Using `spin::Mutex<()>` 12 | // The `spin` crate provides a simple mutex we can use on most platforms. 13 | // We'll use it for the sake of example. 14 | // 15 | // ## Using `ClaimOnOom` 16 | // An OOM handler with support for claiming memory on-demand is required, as allocations may 17 | // occur prior to the execution of `main`. 18 | // 19 | // ## Arena pointer conversion from `*const` to `*mut` 20 | // Without `const_mut_refs` being stable just yet, we need to get a mutable pointer 21 | // indirectly. It's not clear whether this is acceptable (see: https://github.com/SFBdragon/talc/issues/32) 22 | // but MIRI is fine with it and Rust's aliasing/provenance rules don't stipulate yet. 23 | // Once Rust 1.83 lands with `const_mut_refs`, this example will be changed 24 | // to just use `&raw mut`. 25 | 26 | static mut ARENA: [u8; 10000] = [0; 10000]; 27 | 28 | #[global_allocator] 29 | static ALLOCATOR: Talck, ClaimOnOom> = Talc::new(unsafe { 30 | ClaimOnOom::new(Span::from_array(addr_of!(ARENA) as *mut [u8; 10000])) 31 | }).lock(); 32 | 33 | fn main() { 34 | let mut vec = Vec::with_capacity(100); 35 | vec.extend(0..300usize); 36 | vec.truncate(100); 37 | vec.shrink_to_fit(); 38 | } 39 | -------------------------------------------------------------------------------- /talc/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "talc" 3 | version = "4.4.2" 4 | rust-version = "1.67.1" 5 | edition = "2021" 6 | readme = "README.md" 7 | authors = ["Shaun Beautement "] 8 | description = "A fast and flexible allocator for no_std and WebAssembly" 9 | repository = "https://github.com/SFBdragon/talc" 10 | keywords = ["allocator", "no_std", "memory", "heap", "wasm"] 11 | categories = ["memory-management", "no-std", "embedded", "wasm"] 12 | license = "MIT" 13 | 14 | [features] 15 | fuzzing = [] 16 | counters = [] 17 | nightly_api = [] 18 | allocator = ["lock_api"] 19 | default = ["lock_api", "allocator", "nightly_api"] 20 | 21 | [dependencies] 22 | allocator-api2 = { version = "0.2", optional = true, default-features = false } 23 | lock_api = { version = "0.4", optional = true, default-features = false } 24 | 25 | [dev-dependencies] 26 | spin = { version = "0.9.8", default-features = false, features = ["lock_api", "spin_mutex"] } 27 | -------------------------------------------------------------------------------- /talc/LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2023 Shaun Beautement 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a 6 | copy of this software and associated documentation files (the “Software”), 7 | to deal in the Software without restriction, including without limitation 8 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 | and/or sell copies of the Software, and to permit persons to whom the 10 | Software is furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included 13 | in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 | OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /talc/README.md: -------------------------------------------------------------------------------- 1 | # Talc Allocator [![Crates.io](https://img.shields.io/crates/v/talc?style=flat-square&color=orange)](https://crates.io/crates/talc) [![Downloads](https://img.shields.io/crates/d/talc?style=flat-square)](https://crates.io/crates/talc) [![docs.rs](https://img.shields.io/docsrs/talc?style=flat-square)](https://docs.rs/talc/latest/talc/) 2 | 3 | 4 | 5 | If you find Talc useful, please consider leaving tip via [Paypal](https://www.paypal.com/donate/?hosted_button_id=8CSQ92VV58VPQ) 6 | 7 | #### What is this for? 8 | - Embedded systems, OS kernels, and other `no_std` environments 9 | - WebAssembly apps, as a drop-in replacement for the default allocator 10 | - Subsystems in normal programs that need especially quick arena allocation 11 | 12 | #### Why Talc? 13 | - Performance is the primary focus, while retaining generality 14 | - Custom Out-Of-Memory handlers for just-in-time heap management and recovery 15 | - Supports creating and resizing arbitrarily many heaps 16 | - Optional allocation statistics 17 | - Partial validation with debug assertions enabled 18 | - Verified with MIRI 19 | 20 | #### Why not Talc? 21 | - Doesn't integrate with operating systems' dynamic memory facilities out-of-the-box yet 22 | - Doesn't scale well to allocation/deallocation-heavy concurrent processing 23 | - Though it's particularly good at concurrent reallocation. 24 | 25 | ## Table of Contents 26 | 27 | Targeting WebAssembly? You can find WASM-specific usage and benchmarks [here](https://github.com/SFBdragon/talc/blob/master/talc/README_WASM.md). 28 | 29 | - [Setup](#setup) 30 | - [Benchmarks](#benchmarks) 31 | - [General Usage](#general-usage) 32 | - [Advanced Usage](#advanced-usage) 33 | - [Conditional Features](#conditional-features) 34 | - [Stable Rust and MSRV](#stable-rust-and-msrv) 35 | - [Algorithm](#algorithm) 36 | - [Future Development](#future-development-v5) 37 | - [Changelog](#changelog) 38 | 39 | 40 | ## Setup 41 | 42 | As a global allocator: 43 | ```rust 44 | use talc::*; 45 | 46 | static mut ARENA: [u8; 10000] = [0; 10000]; 47 | 48 | #[global_allocator] 49 | static ALLOCATOR: Talck, ClaimOnOom> = Talc::new(unsafe { 50 | // if we're in a hosted environment, the Rust runtime may allocate before 51 | // main() is called, so we need to initialize the arena automatically 52 | ClaimOnOom::new(Span::from_array(core::ptr::addr_of!(ARENA).cast_mut())) 53 | }).lock(); 54 | 55 | fn main() { 56 | let mut vec = Vec::with_capacity(100); 57 | vec.extend(0..300usize); 58 | } 59 | ``` 60 | 61 | Or use it as an arena allocator via the `Allocator` API with `spin` as follows: 62 | ```rust 63 | #![feature(allocator_api)] 64 | use talc::*; 65 | use core::alloc::{Allocator, Layout}; 66 | 67 | static mut ARENA: [u8; 10000] = [0; 10000]; 68 | 69 | fn main () { 70 | let talck = Talc::new(ErrOnOom).lock::>(); 71 | unsafe { talck.lock().claim(ARENA.as_mut().into()); } 72 | 73 | talck.allocate(Layout::new::<[u32; 16]>()); 74 | } 75 | ``` 76 | 77 | Note that while the `spin` crate's mutexes are used here, any lock implementing `lock_api` works. 78 | 79 | See [General Usage](#general-usage) and [Advanced Usage](#advanced-usage) for more details. 80 | 81 | ## Benchmarks 82 | 83 | ### Heap Efficiency Benchmark Results 84 | 85 | The average occupied capacity upon first allocation failure when randomly allocating/deallocating/reallocating. 86 | 87 | | Allocator | Average Random Actions Heap Efficiency | 88 | | --------------------- | -------------------------------------- | 89 | | Dlmalloc | 99.14% | 90 | | Rlsf | 99.06% | 91 | | Talc | 98.97% | 92 | | Linked List | 98.36% | 93 | | Buddy Alloc | 63.14% | 94 | 95 | ### Random Actions Benchmark 96 | 97 | The number of successful allocations, deallocations, and reallocations within the allotted time. 98 | 99 | #### 1 Thread 100 | 101 | ![Random Actions Benchmark Results](https://github.com/SFBdragon/talc/blob/master/talc/benchmark_graphs/random_actions.png?raw=true) 102 | 103 | #### 4 Threads 104 | 105 | ![Random Actions Multi Benchmark Results](https://github.com/SFBdragon/talc/blob/master/talc/benchmark_graphs/random_actions_multi.png?raw=true) 106 | 107 | ## Allocations & Deallocations Microbenchmark 108 | 109 | ![Microbenchmark Results](https://github.com/SFBdragon/talc/blob/master/talc/benchmark_graphs/microbench.png?raw=true) 110 | 111 | Label indicates the maximum within 50 standard deviations from the median. Max allocation size is 0x10000. 112 | 113 | ## General Usage 114 | 115 | Here is the list of important `Talc` methods: 116 | * Constructors: 117 | * `new` 118 | * Information: 119 | * `get_allocated_span` - returns the minimum heap span containing all allocated memory in an established heap 120 | * `get_counters` - if feature `"counters"` is enabled, this returns a struct with allocation statistics 121 | * Management: 122 | * `claim` - claim memory to establishing a new heap 123 | * `extend` - extend an established heap 124 | * `truncate` - reduce the extent of an established heap 125 | * `lock` - wraps the `Talc` in a `Talck`, which supports the `GlobalAlloc` and `Allocator` APIs 126 | * Allocation: 127 | * `malloc` 128 | * `free` 129 | * `grow` 130 | * `grow_in_place` 131 | * `shrink` 132 | 133 | Read their [documentation](https://docs.rs/talc/latest/talc/struct.Talc.html) for more info. 134 | 135 | [`Span`](https://docs.rs/talc/latest/talc/struct.Span.html) is a handy little type for describing memory regions, as trying to manipulate `Range<*mut u8>` or `*mut [u8]` or `base_ptr`-`size` pairs tends to be inconvenient or annoying. 136 | 137 | ## Advanced Usage 138 | 139 | The most powerful feature of the allocator is that it has a modular OOM handling system, allowing you to fail out of or recover from allocation failure easily. 140 | 141 | Provided `OomHandler` implementations include: 142 | - `ErrOnOom`: allocations fail on OOM 143 | - `ClaimOnOom`: claims a heap upon first OOM, useful for initialization 144 | - `WasmHandler`: itegrate with WebAssembly's `memory` module for automatic memory heap management 145 | 146 | As an example of a custom implementation, recovering by extending the heap is implemented below. 147 | 148 | ```rust 149 | use talc::*; 150 | 151 | struct MyOomHandler { 152 | heap: Span, 153 | } 154 | 155 | impl OomHandler for MyOomHandler { 156 | fn handle_oom(talc: &mut Talc, layout: core::alloc::Layout) -> Result<(), ()> { 157 | // Talc doesn't have enough memory, and we just got called! 158 | // We'll go through an example of how to handle this situation. 159 | 160 | // We can inspect `layout` to estimate how much we should free up for this allocation 161 | // or we can extend by any amount (increasing powers of two has good time complexity). 162 | // (Creating another heap with `claim` will also work.) 163 | 164 | // This function will be repeatedly called until we free up enough memory or 165 | // we return Err(()) causing allocation failure. Be careful to avoid conditions where 166 | // the heap isn't sufficiently extended indefinitely, causing an infinite loop. 167 | 168 | // an arbitrary address limit for the sake of example 169 | const HEAP_TOP_LIMIT: *mut u8 = 0x80000000 as *mut u8; 170 | 171 | let old_heap: Span = talc.oom_handler.heap; 172 | 173 | // we're going to extend the heap upward, doubling its size 174 | // but we'll be sure not to extend past the limit 175 | let new_heap: Span = old_heap.extend(0, old_heap.size()).below(HEAP_TOP_LIMIT); 176 | 177 | if new_heap == old_heap { 178 | // we won't be extending the heap, so we should return Err 179 | return Err(()); 180 | } 181 | 182 | unsafe { 183 | // we're assuming the new memory up to HEAP_TOP_LIMIT is unused and allocatable 184 | talc.oom_handler.heap = talc.extend(old_heap, new_heap); 185 | } 186 | 187 | Ok(()) 188 | } 189 | } 190 | ``` 191 | 192 | ## Conditional Features 193 | * `"lock_api"` (default): Provides the `Talck` locking wrapper type that implements `GlobalAlloc`. 194 | * `"allocator"` (default, requires nightly): Provides an `Allocator` trait implementation via `Talck`. 195 | * `"nightly_api"` (default, requires nightly): Provides the `Span::from(*mut [T])` and `Span::from_slice` functions. 196 | * `"counters"`: `Talc` will track heap and allocation metrics. Use `Talc::get_counters` to access them. 197 | * `"allocator-api2"`: `Talck` will implement `allocator_api2::alloc::Allocator` if `"allocator"` is not active. 198 | 199 | ## Stable Rust and MSRV 200 | Talc can be built on stable Rust by disabling `"allocator"` and `"nightly_api"`. The MSRV is 1.67.1. 201 | 202 | Disabling `"nightly_api"` disables `Span::from(*mut [T])`, `Span::from(*const [T])`, `Span::from_const_slice` and `Span::from_slice`. 203 | 204 | ## Algorithm 205 | This is a dlmalloc-style linked list allocator with boundary tagging and bucketing, aimed at general-purpose use cases. Allocation is O(n) worst case (but in practice its near-constant time, see microbenchmarks), while in-place reallocations and deallocations are O(1). 206 | 207 | Additionally, the layout of chunk metadata is rearranged to allow for smaller minimum-size chunks to reduce memory overhead of small allocations. The minimum chunk size is `3 * usize`, with a single `usize` being reserved per allocation. This is more efficient than `dlmalloc` and `galloc`, despite using a similar algorithm. 208 | 209 | ## Future Development 210 | - Support better concurrency, as it's the main deficit of the allocator 211 | - Change the default features to be stable by default 212 | - Add out-of-the-box support for more systems 213 | - Allow for integrating with a backing allocator & (deferred) freeing of unused memory (e.g. better integration with mmap/paging) 214 | 215 | Update: All of this is currently in the works. No guarantees on when it will be done, but significant progress has been made. 216 | 217 | ## Changelog 218 | 219 | 220 | #### v4.4.2 221 | 222 | - [polarathene](https://github.com/polarathene): Replace README relative links with fully-qualified links. 223 | - [polarathene](https://github.com/polarathene): Improve docs for `stable_examples/examples/std_global_allocator.rs`. 224 | 225 | - Improved docs for `stable_examples/examples/stable_allocator_api.rs` and `stable_examples/examples/std_global_allocator.rs`. 226 | - Deprecated the `Span::from*` function for converting from shared references and const pointers, as they make committing UB easy. These will be removed in v5. 227 | - Fixed up a bunch of warnings all over the project. 228 | 229 | #### v4.4.1 230 | 231 | - Added utility function `except` to `Span`, which takes the set difference, potentially splitting the `Span`. Thanks [bjorn3](https://github.com/bjorn3) for the suggestion! 232 | 233 | #### v4.4.0 234 | 235 | - Added feature `allocator-api2` which allows using the `Allocator` trait on stable via the [`allocator-api2`](https://github.com/zakarumych/allocator-api2) crate. Thanks [jess-sol](https://github.com/jess-sol)! 236 | 237 | #### v4.3.1 238 | 239 | - Updated the README a little 240 | 241 | #### v4.3.0 242 | 243 | - Added an implementation for `Display` for the counters. Hopefully this makes your logs a bit prettier. 244 | - Bug me if you have opinions about the current layout, I'm open to changing it. 245 | 246 | - Added Frusa and RLSF to the benchmarks. 247 | - Good showing by RLSF all around, and Frusa has particular workloads it excels at. 248 | - Changed random actions benchmark to measure over various allocation sizes. 249 | 250 | #### v4.2.0 251 | 252 | - Optimized reallocation to allows other allocation operations to occur while memcopy-ing if an in-place reallocation failed. 253 | - As a side effect Talc now has a `grow_in_place` function that returns `Err` if growing the memory in-place isn't possible. 254 | - A graph of the random actions benchmark with a workload that benefits from this has been included in the [benchmarks](#benchmarks) section. 255 | 256 | - Added `Span::from_*` and `From<>` functions for const pointers and shared references. 257 | - This makes creating a span in static contexts on stable much easier: `Span::from_const_array(addr_of!(MEMORY))` 258 | - Fix: Made `Talck` derive `Debug` again. 259 | 260 | - Contribution by [Ken Hoover](https://github.com/khoover): add Talc arena-style allocation size and perf WASM benchmarks 261 | - This might be a great option if you have a known dynamic memory requirement and would like to reduce your WASM size a little more. 262 | 263 | - `wasm-size` now uses _wasm-opt_, giving more realistic size differences for users of _wasm-pack_ 264 | - Improved shell scripts 265 | - Overhauled microbenchmarks 266 | - No longer simulates high-heap pressure as tolerating allocation failure is rare 267 | - Data is now displayed using box-and-whisker plots 268 | 269 | #### v4.1.1 270 | 271 | - Fix: Reset MSRV to 1.67.1 and added a check to `test.sh` for it 272 | 273 | #### v4.1.0 (yanked, use 4.1.1) 274 | 275 | - Added optional tracking of allocation metrics. Thanks [Ken Hoover](https://github.com/khoover) for the suggestion! 276 | - Enable the `"counters"` feature. Access the data via `talc.get_counters()` 277 | - Metrics include allocation count, bytes available, fragmentation, overhead, and more. 278 | - Improvements to documentation 279 | - Improved and updated benchmarks 280 | - Integrated the WASM performance benchmark into the project. Use `wasm-bench.sh` to run (requires _wasm-pack_ and _deno_) 281 | - Improved `wasm-size` and `wasm-size.sh` 282 | 283 | #### v4.0.0 284 | - Changed `Talck`'s API to be more inline with Rust norms. 285 | - `Talck` now hides its internal structure (no more `.0`). 286 | - `Talck::talc()` has been replaced by `Talck::lock()`. 287 | - `Talck::new()` and `Talck::into_inner(self)` have been added. 288 | - Removed `TalckRef` and implemented the `Allocator` trait on `Talck` directly. No need to call `talck.allocator()` anymore. 289 | - Changed API for provided locking mechanism 290 | - Moved `AssumeUnlockable` into `talc::locking::AssumeUnlockable` 291 | - Removed `Talc::lock_assume_single_threaded`, use `.lock::()` if necessary. 292 | - Improvements to documentation here and there. Thanks [polarathene](https://github.com/polarathene) for the contribution! 293 | 294 | #### v3.1.2 295 | - Some improvements to documentation. 296 | 297 | #### v3.1.1 298 | - Changed the WASM OOM handler's behavior to be more robust if other code calls `memory.grow` during the allocator's use. 299 | 300 | #### v3.1.0 301 | - Reduced use of nightly-only features, and feature-gated the remainder (`Span::from(*mut [T])` and `Span::from_slice`) behind `nightly_api`. 302 | - `nightly_api` feature is default-enabled 303 | - *WARNING:* use of `default-features = false` may cause unexpected errors if the gated functions are used. Consider adding `nightly_api` or using another function. 304 | 305 | #### v3.0.1 306 | - Improved documentation 307 | - Improved and updated benchmarks 308 | - Increased the range of allocation sizes on Random Actions. (sorry Buddy Allocator!) 309 | - Increased the number of iterations the Heap Efficiency benchmark does to produce more accurate and stable values. 310 | 311 | #### v3.0.0 312 | - Added support for multiple discontinuous heaps! This required some major API changes 313 | - `new_arena` no longer exists (use `new` and then `claim`) 314 | - `init` has been replaced with `claim` 315 | - `claim`, `extend` and `truncate` now return the new heap extent 316 | - `InitOnOom` is now `ClaimOnOom`. 317 | - All of the above now have different behavior and documentation. 318 | - Each heap now has a fixed overhead of one `usize` at the bottom. 319 | 320 | To migrate from v2 to v3, keep in mind that you must keep track of the heaps if you want to resize them, by storing the returned `Span`s. Read [`claim`](https://docs.rs/talc/latest/talc/struct.Talc.html#method.claim), [`extend`](https://docs.rs/talc/latest/talc/struct.Talc.html#method.extend) and [`truncate`](https://docs.rs/talc/latest/talc/struct.Talc.html#method.truncate)'s documentation for all the details. 321 | 322 | #### v2.2.1 323 | - Rewrote the allocator internals to place allocation metadata above the allocation. 324 | - This will have the largest impact on avoiding false sharing, where previously, the allocation metadata for one allocation would infringe on the cache-line of the allocation before it, even if a sufficiently high alignment was demanded. Single-threaded performance marginally increased, too. 325 | - Removed heap_exhaustion and replaced heap_efficiency benchmarks. 326 | - Improved documentation and other resources. 327 | - Changed the WASM size measurement to include slightly less overhead. 328 | 329 | #### v2.2.0 330 | - Added `dlmalloc` to the benchmarks. 331 | - WASM should now be fully supported via `TalckWasm`. Let me know what breaks ;) 332 | - Find more details [here](https://github.com/SFBdragon/talc/README_WASM.md). 333 | 334 | 335 | #### v2.1.0 336 | - Tests are now passing on 32 bit targets. 337 | - Documentation fixes and improvements for various items. 338 | - Fixed using `lock_api` without `allocator`. 339 | - Experimental WASM support has been added via `TalckWasm` on WASM targets. 340 | 341 | 342 | #### v2.0.0 343 | - Removed dependency on `spin` and switched to using `lock_api` (thanks [Stefan Lankes](https://github.com/stlankes)) 344 | - You can specify the lock you want to use with `talc.lock::>()` for example. 345 | - Removed the requirement that the `Talc` struct must not be moved, and removed the `mov` function. 346 | - The arena is now used to store metadata, so extremely small arenas will result in allocation failure. 347 | - Made the OOM handling system use generics and traits instead of a function pointer. 348 | - Use `ErrOnOom` to do what it says on the tin. `InitOnOom` is similar but inits to the given span if completely uninitialized. Implement `OomHandler` on any struct to implement your own behaviour (the OOM handler state can be accessed from `handle_oom` via `talc.oom_handler`). 349 | - Changed the API and internals of `Span` and other changes to pass `miri`'s Stacked Borrows checks. 350 | - Span now uses pointers exclusively and carries provenance. 351 | - Updated the benchmarks in a number of ways, notably adding `buddy_alloc` and removing `simple_chunk_allocator`. 352 | 353 | -------------------------------------------------------------------------------- /talc/README_WASM.md: -------------------------------------------------------------------------------- 1 | # Talc for WebAssembly 2 | 3 | Talc is also a drop-in replacement for the default Rust WebAssembly allocator, dlmalloc. The two main configurations's usage and benchmarks are below. Both provide a decent middleground by being faster than `lol_alloc` and `dlmalloc` while inbetweening them in size. 4 | 5 | ## Usage 6 | Set the global allocator in your project after running `cargo add talc` as follows: 7 | 8 | ```rust 9 | /// SAFETY: The runtime environment must be single-threaded WASM. 10 | #[global_allocator] 11 | static ALLOCATOR: talc::TalckWasm = unsafe { talc::TalckWasm::new_global() }; 12 | ``` 13 | 14 | Or if your arena size is statically known, for example 16 MiB, `0x1000000`: 15 | 16 | ```rust 17 | #[global_allocator] 18 | static ALLOCATOR: talc::Talck = { 19 | static mut MEMORY: [u8; 0x1000000] = [0; 0x1000000]; 20 | let span = talc::Span::from_array(std::ptr::addr_of!(MEMORY).cast_mut()); 21 | talc::Talc::new(unsafe { talc::ClaimOnOom::new(span) }).lock() 22 | }; 23 | ``` 24 | 25 | ## Configuration features for WebAssembly: 26 | - If default features are disabled, make sure to enable `"lock_api"`. 27 | - Turn on `"counters"` for allocation statistics accessible via `ALLOCATOR.lock().get_counters()` 28 | - You can turn off default features to remove `"nightly_api"`, allowing stable Rust builds. 29 | 30 | e.g. `default-features = false, features = ["lock_api", "counters"]` 31 | 32 | ## Relative WASM Binary Size 33 | 34 | Rough measurements of allocator size for relative comparison using `/wasm-size`. 35 | 36 | | Allocator | WASM Size/bytes | 37 | | --------- | --------------- | 38 | | lol_alloc | 11655 | 39 | | rlsf | 12242 | 40 | | **talc** (arena\*) | 13543 | 41 | | **talc** | 14467 | 42 | | dlmalloc (default) | 16767 | 43 | 44 | \* uses a static arena instead of dynamically managing the heap 45 | 46 | ## WASM Benchmarks 47 | 48 | Rough measurements of allocator speed for relative comparison using `/wasm-bench`. 49 | 50 | | Allocator | Average Actions/us | 51 | |-----------|--------------------| 52 | | **talc** | 6.7| 53 | | **talc** (arena\*) | 6.8 | 54 | | rlsf | 5.7 | 55 | | dlmalloc (default) | 5.9 | 56 | | lol_alloc | 4.4 | 57 | 58 | \* uses a static arena instead of dynamically managing the heap 59 | 60 | 61 | If you'd like to see comparisons to other allocators in this space, consider creating a pull request or opening an issue. 62 | -------------------------------------------------------------------------------- /talc/benchmark_graphs/microbench.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SFBdragon/talc/75d049aaafe8a936f554a8930d90812719e3b0e8/talc/benchmark_graphs/microbench.png -------------------------------------------------------------------------------- /talc/benchmark_graphs/random_actions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SFBdragon/talc/75d049aaafe8a936f554a8930d90812719e3b0e8/talc/benchmark_graphs/random_actions.png -------------------------------------------------------------------------------- /talc/benchmark_graphs/random_actions_multi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SFBdragon/talc/75d049aaafe8a936f554a8930d90812719e3b0e8/talc/benchmark_graphs/random_actions_multi.png -------------------------------------------------------------------------------- /talc/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! The Talc allocator crate. 2 | //! 3 | //! For getting started: 4 | //! - Check out the crate's [README](https://github.com/SFBdragon/talc) 5 | //! - Read check out the `Talc` and `Talck` structures. 6 | //! 7 | //! Your first step will be `Talc::new(...)`, then `claim`. 8 | //! Calling `Talc::lock()` on it will yield a `Talck` which implements 9 | //! [`GlobalAlloc`] and [`Allocator`] (if the appropriate feature flags are set). 10 | 11 | #![cfg_attr(not(any(test, feature = "fuzzing")), no_std)] 12 | #![cfg_attr(feature = "allocator", feature(allocator_api))] 13 | 14 | mod oom_handler; 15 | mod ptr_utils; 16 | mod span; 17 | mod talc; 18 | 19 | #[cfg(feature = "lock_api")] 20 | pub mod locking; 21 | #[cfg(feature = "lock_api")] 22 | mod talck; 23 | 24 | pub use oom_handler::{ClaimOnOom, ErrOnOom, OomHandler}; 25 | pub use span::Span; 26 | pub use talc::Talc; 27 | 28 | #[cfg(feature = "lock_api")] 29 | pub use talck::Talck; 30 | #[cfg(all(target_family = "wasm", feature = "lock_api"))] 31 | pub use talck::TalckWasm; 32 | 33 | #[cfg(all(target_family = "wasm", feature = "lock_api"))] 34 | pub use oom_handler::WasmHandler; 35 | -------------------------------------------------------------------------------- /talc/src/locking.rs: -------------------------------------------------------------------------------- 1 | //! Note this only contains [`AssumeUnlockable`] which is not generally recommended. 2 | //! Use of the `spin` crate's mutex with [`Talck`](crate::Talc) is a good default. 3 | 4 | /// #### WARNING: [`AssumeUnlockable`] may cause undefined behaviour without `unsafe` code! 5 | /// 6 | /// A dummy [`RawMutex`](lock_api::RawMutex) implementation to skip synchronization on single threaded systems. 7 | /// 8 | /// # Safety 9 | /// [`AssumeUnlockable`] is highly unsafe and may cause undefined behaviour if multiple 10 | /// threads enter a critical section it guards, even without explicit unsafe code. 11 | /// 12 | /// Note that uncontended spin locks are cheap. Usage is only recommended on 13 | /// platforms that don't have atomics or are exclusively single threaded. 14 | /// 15 | /// Through no fault of its own, `lock_api`'s API does not allow for safe 16 | /// encapsulation of this functionality. This is a hack for backwards compatibility. 17 | pub struct AssumeUnlockable; 18 | 19 | // SAFETY: nope 20 | unsafe impl lock_api::RawMutex for AssumeUnlockable { 21 | const INIT: AssumeUnlockable = AssumeUnlockable; 22 | 23 | // A spinlock guard can be sent to another thread and unlocked there 24 | type GuardMarker = lock_api::GuardSend; 25 | 26 | fn lock(&self) {} 27 | 28 | fn try_lock(&self) -> bool { 29 | true 30 | } 31 | 32 | unsafe fn unlock(&self) {} 33 | } 34 | -------------------------------------------------------------------------------- /talc/src/oom_handler.rs: -------------------------------------------------------------------------------- 1 | use core::alloc::Layout; 2 | 3 | use crate::{Span, Talc}; 4 | 5 | pub trait OomHandler: Sized { 6 | /// Given the allocator and the `layout` of the allocation that caused 7 | /// OOM, resize or claim and return `Ok(())` or fail by returning `Err(())`. 8 | /// 9 | /// This function is called repeatedly if the allocator is still out of memory. 10 | /// Therefore an infinite loop will occur if `Ok(())` is repeatedly returned 11 | /// without extending or claiming new memory. 12 | fn handle_oom(talc: &mut Talc, layout: Layout) -> Result<(), ()>; 13 | } 14 | 15 | /// Doesn't handle out-of-memory conditions, immediate allocation error occurs. 16 | pub struct ErrOnOom; 17 | 18 | impl OomHandler for ErrOnOom { 19 | fn handle_oom(_: &mut Talc, _: Layout) -> Result<(), ()> { 20 | Err(()) 21 | } 22 | } 23 | 24 | /// An out-of-memory handler that attempts to claim the 25 | /// memory within the given [`Span`] upon OOM. 26 | /// 27 | /// The contained span is then overwritten with an empty span. 28 | /// 29 | /// If the span is empty or `claim` fails, allocation failure occurs. 30 | pub struct ClaimOnOom(Span); 31 | 32 | impl ClaimOnOom { 33 | /// # Safety 34 | /// The memory within the given [`Span`] must conform to 35 | /// the requirements laid out by [`claim`](Talc::claim). 36 | pub const unsafe fn new(span: Span) -> Self { 37 | ClaimOnOom(span) 38 | } 39 | } 40 | 41 | impl OomHandler for ClaimOnOom { 42 | fn handle_oom(talc: &mut Talc, _: Layout) -> Result<(), ()> { 43 | if !talc.oom_handler.0.is_empty() { 44 | unsafe { 45 | talc.claim(talc.oom_handler.0)?; 46 | } 47 | 48 | talc.oom_handler.0 = Span::empty(); 49 | 50 | Ok(()) 51 | } else { 52 | Err(()) 53 | } 54 | } 55 | } 56 | 57 | #[cfg(all(target_family = "wasm", feature = "lock_api"))] 58 | pub struct WasmHandler { 59 | prev_heap: Span, 60 | } 61 | 62 | #[cfg(all(target_family = "wasm", feature = "lock_api"))] 63 | unsafe impl Send for WasmHandler {} 64 | 65 | #[cfg(all(target_family = "wasm", feature = "lock_api"))] 66 | impl WasmHandler { 67 | /// Create a new WASM handler. 68 | /// # Safety 69 | /// [`WasmHandler`] expects to have full control over WASM memory 70 | /// and be running in a single-threaded environment. 71 | pub const unsafe fn new() -> Self { 72 | Self { prev_heap: Span::empty() } 73 | } 74 | } 75 | 76 | #[cfg(all(target_family = "wasm", feature = "lock_api"))] 77 | impl OomHandler for WasmHandler { 78 | fn handle_oom(talc: &mut Talc, layout: Layout) -> Result<(), ()> { 79 | /// WASM page size is 64KiB 80 | const PAGE_SIZE: usize = 1024 * 64; 81 | 82 | // growth strategy: just try to grow enough to avoid OOM again on this allocation 83 | let required = (layout.size() + 8).max(layout.align() * 2); 84 | let mut delta_pages = (required + (PAGE_SIZE - 1)) / PAGE_SIZE; 85 | 86 | let prev = 'prev: { 87 | // This performs a scan, trying to find a smaller possible 88 | // growth if the previous one was unsuccessful. Return 89 | // any successful allocated to memory. 90 | // If not quite enough, talc will invoke handle_oom again. 91 | 92 | // if we're about to fail because of allocation failure 93 | // we may as well try as hard as we can to probe what's permissable 94 | // which can be done with a log2(n)-ish algorithm 95 | // (factoring in repeated called to handle_oom) 96 | while delta_pages != 0 { 97 | // use `core::arch::wasm` instead once it doesn't 98 | // require the unstable feature wasm_simd64? 99 | let result = core::arch::wasm32::memory_grow::<0>(delta_pages); 100 | 101 | if result != usize::MAX { 102 | break 'prev result; 103 | } else { 104 | delta_pages >>= 1; 105 | continue; 106 | } 107 | } 108 | 109 | return Err(()); 110 | }; 111 | 112 | let prev_heap_acme = (prev * PAGE_SIZE) as *mut u8; 113 | let new_heap_acme = prev_heap_acme.wrapping_add(delta_pages * PAGE_SIZE); 114 | 115 | // try to get base & acme, which will fail if prev_heap is empty 116 | // otherwise the allocator has been initialized previously 117 | if let Some((prev_base, prev_acme)) = talc.oom_handler.prev_heap.get_base_acme() { 118 | if prev_acme == prev_heap_acme { 119 | talc.oom_handler.prev_heap = unsafe { 120 | talc.extend(talc.oom_handler.prev_heap, Span::new(prev_base, new_heap_acme)) 121 | }; 122 | 123 | return Ok(()); 124 | } 125 | } 126 | 127 | talc.oom_handler.prev_heap = unsafe { 128 | // delta_pages is always greater than zero 129 | // thus one page is enough space for metadata 130 | // therefore we can unwrap the result 131 | talc.claim(Span::new(prev_heap_acme, new_heap_acme)).unwrap() 132 | }; 133 | 134 | Ok(()) 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /talc/src/ptr_utils.rs: -------------------------------------------------------------------------------- 1 | //! Generic utilities for pointer handling and sizing. 2 | 3 | pub const WORD_SIZE: usize = core::mem::size_of::(); 4 | pub const WORD_BITS: usize = usize::BITS as usize; 5 | pub const ALIGN: usize = core::mem::align_of::(); 6 | 7 | /// Aligns `ptr` up to the next `align_mask + 1`. 8 | /// 9 | /// `align_mask` must be a power of two minus one. 10 | #[inline] 11 | pub fn align_up_by(ptr: *mut u8, align_mask: usize) -> *mut u8 { 12 | debug_assert!((align_mask + 1).is_power_of_two()); 13 | 14 | // this incantation maintains provenance of ptr 15 | // while allowing the compiler to see through the wrapping_add and optimize it 16 | ptr.wrapping_add(((ptr as usize + align_mask) & !align_mask) - ptr as usize) 17 | // equivalent to the following: 18 | // ((ptr as usize + align_mask) & !align_mask) as *mut u8 19 | // i.e. just align up to the next align_mask + 1 20 | } 21 | 22 | pub fn align_down(ptr: *mut u8) -> *mut u8 { 23 | ptr.wrapping_sub(ptr as usize % ALIGN) 24 | } 25 | pub fn align_up_overflows(ptr: *mut u8) -> bool { 26 | ALIGN - 1 > usize::MAX - ptr as usize 27 | } 28 | pub fn align_up(ptr: *mut u8) -> *mut u8 { 29 | debug_assert!(!align_up_overflows(ptr)); 30 | 31 | let offset_ptr = ptr.wrapping_add(ALIGN - 1); 32 | offset_ptr.wrapping_sub(offset_ptr as usize % ALIGN) 33 | } 34 | 35 | #[cfg(test)] 36 | mod tests { 37 | use core::ptr::null_mut; 38 | 39 | use super::*; 40 | 41 | #[test] 42 | fn align_ptr_test() { 43 | assert!(!align_up_overflows(null_mut())); 44 | assert!(!align_up_overflows(null_mut::().wrapping_sub(ALIGN))); 45 | assert!(align_up_overflows(null_mut::().wrapping_sub(ALIGN - 1))); 46 | assert!(align_up_overflows(null_mut::().wrapping_sub(ALIGN - 2))); 47 | assert!(align_up_overflows(null_mut::().wrapping_sub(ALIGN - 3))); 48 | 49 | assert!(align_up(null_mut()) == null_mut()); 50 | assert!(align_down(null_mut()) == null_mut()); 51 | 52 | assert!(align_up(null_mut::().wrapping_add(1)) == null_mut::().wrapping_add(ALIGN)); 53 | assert!(align_up(null_mut::().wrapping_add(2)) == null_mut::().wrapping_add(ALIGN)); 54 | assert!(align_up(null_mut::().wrapping_add(3)) == null_mut::().wrapping_add(ALIGN)); 55 | assert!( 56 | align_up(null_mut::().wrapping_add(ALIGN)) == null_mut::().wrapping_add(ALIGN) 57 | ); 58 | 59 | assert!(align_down(null_mut::().wrapping_add(1)) == null_mut::()); 60 | assert!(align_down(null_mut::().wrapping_add(2)) == null_mut::()); 61 | assert!(align_down(null_mut::().wrapping_add(3)) == null_mut::()); 62 | assert!( 63 | align_down(null_mut::().wrapping_add(ALIGN)) 64 | == null_mut::().wrapping_add(ALIGN) 65 | ); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /talc/src/span.rs: -------------------------------------------------------------------------------- 1 | use core::ops::Range; 2 | 3 | use crate::ptr_utils::*; 4 | 5 | /// Represents an interval of memory `[base, acme)` 6 | /// 7 | /// Use `get_base_acme` to retrieve `base` and `acme` directly. 8 | /// 9 | /// # Empty Spans 10 | /// Note that where `base >= acme`, the [`Span`] is considered empty, in which case 11 | /// the specific values of `base` and `acme` are considered meaningless. 12 | /// * Empty spans contain nothing and overlap with nothing. 13 | /// * Empty spans are contained by any sized span. 14 | #[derive(Clone, Copy, Hash)] 15 | pub struct Span { 16 | base: *mut u8, 17 | acme: *mut u8, 18 | } 19 | 20 | unsafe impl Send for Span {} 21 | 22 | impl Default for Span { 23 | fn default() -> Self { 24 | Self::empty() 25 | } 26 | } 27 | 28 | impl core::fmt::Debug for Span { 29 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 30 | f.write_fmt(format_args!("{:p}..[{}]..{:p}", self.base, self.size(), self.acme)) 31 | } 32 | } 33 | 34 | impl core::fmt::Display for Span { 35 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 36 | match self.get_base_acme() { 37 | Some((base, acme)) => f.write_fmt(format_args!("{:p}..{:p}", base, acme)), 38 | None => f.write_str("Empty Span"), 39 | } 40 | } 41 | } 42 | 43 | impl From> for Span { 44 | fn from(value: Range<*mut T>) -> Self { 45 | Self { base: value.start.cast(), acme: value.end.cast() } 46 | } 47 | } 48 | 49 | // NOTE: This should be removed in a future release as it encouraged UB. 50 | // Once `const_mut_refs` is stabilized in Rust, this will no longer be useful anyway. 51 | // See: https://github.com/SFBdragon/talc/issues/33 52 | impl From> for Span { 53 | fn from(value: Range<*const T>) -> Self { 54 | Self { base: value.start.cast_mut().cast(), acme: value.end.cast_mut().cast() } 55 | } 56 | } 57 | 58 | impl From<&mut [T]> for Span { 59 | fn from(value: &mut [T]) -> Self { 60 | Self::from(value.as_mut_ptr_range()) 61 | } 62 | } 63 | 64 | // NOTE: This should be removed in a future release as it encouraged UB. 65 | // Once `const_mut_refs` is stabilized in Rust, this will no longer be useful anyway. 66 | // See: https://github.com/SFBdragon/talc/issues/33 67 | impl From<&[T]> for Span { 68 | fn from(value: &[T]) -> Self { 69 | Self::from(value.as_ptr_range()) 70 | } 71 | } 72 | 73 | impl From<&mut [T; N]> for Span { 74 | fn from(value: &mut [T; N]) -> Self { 75 | Self::from(value as *mut [T; N]) 76 | } 77 | } 78 | 79 | // NOTE: This should be removed in a future release as it encouraged UB. 80 | // Once `const_mut_refs` is stabilized in Rust, this will no longer be useful anyway. 81 | // See: https://github.com/SFBdragon/talc/issues/33 82 | impl From<&[T; N]> for Span { 83 | fn from(value: &[T; N]) -> Self { 84 | Self::from(value as *const [T; N]) 85 | } 86 | } 87 | 88 | #[cfg(feature = "nightly_api")] 89 | impl From<*mut [T]> for Span { 90 | fn from(value: *mut [T]) -> Self { 91 | Self::from_slice(value) 92 | } 93 | } 94 | 95 | // NOTE: This should be removed in a future release as it encouraged UB. 96 | // Once `const_mut_refs` is stabilized in Rust, this will no longer be useful anyway. 97 | // See: https://github.com/SFBdragon/talc/issues/33 98 | #[cfg(feature = "nightly_api")] 99 | impl From<*const [T]> for Span { 100 | fn from(value: *const [T]) -> Self { 101 | #[expect(deprecated)] // This impl is 'deprecated' too. 102 | Self::from_const_slice(value) 103 | } 104 | } 105 | 106 | impl From<*mut [T; N]> for Span { 107 | fn from(value: *mut [T; N]) -> Self { 108 | Self::from_array(value) 109 | } 110 | } 111 | 112 | // NOTE: This should be removed in a future release as it encouraged UB. 113 | // Once `const_mut_refs` is stabilized in Rust, this will no longer be useful anyway. 114 | // See: https://github.com/SFBdragon/talc/issues/33 115 | impl From<*const [T; N]> for Span { 116 | fn from(value: *const [T; N]) -> Self { 117 | Self::from_array(value.cast_mut()) 118 | } 119 | } 120 | 121 | impl PartialEq for Span { 122 | fn eq(&self, other: &Self) -> bool { 123 | self.is_empty() && other.is_empty() || self.base == other.base && self.acme == other.acme 124 | } 125 | } 126 | impl Eq for Span {} 127 | 128 | impl Span { 129 | /// Returns whether `base >= acme`. 130 | #[inline] 131 | pub fn is_empty(self) -> bool { 132 | self.acme <= self.base 133 | } 134 | 135 | /// Returns whether `base < acme`. 136 | #[inline] 137 | pub fn is_sized(self) -> bool { 138 | !self.is_empty() 139 | } 140 | 141 | /// Returns the size of the span, else zero if `base >= span`. 142 | #[inline] 143 | pub fn size(self) -> usize { 144 | if self.is_empty() { 0 } else { self.acme as usize - self.base as usize } 145 | } 146 | 147 | /// If `self` isn't empty, returns `(base, acme)` 148 | #[inline] 149 | pub fn get_base_acme(self) -> Option<(*mut u8, *mut u8)> { 150 | if self.is_empty() { None } else { Some((self.base, self.acme)) } 151 | } 152 | 153 | /// Create an empty span. 154 | #[inline] 155 | pub const fn empty() -> Self { 156 | Self { base: core::ptr::null_mut(), acme: core::ptr::null_mut() } 157 | } 158 | 159 | /// Create a new span. 160 | #[inline] 161 | pub const fn new(base: *mut u8, acme: *mut u8) -> Self { 162 | Self { base, acme } 163 | } 164 | 165 | /// Creates a [`Span`] given a `base` and a `size`. 166 | /// 167 | /// If `base + size` overflows, the result is empty. 168 | #[inline] 169 | pub const fn from_base_size(base: *mut u8, size: usize) -> Self { 170 | Self { base, acme: base.wrapping_add(size) } 171 | } 172 | 173 | #[cfg(feature = "nightly_api")] 174 | #[inline] 175 | pub const fn from_slice(slice: *mut [T]) -> Self { 176 | Self { 177 | base: slice as *mut T as *mut u8, 178 | // SAFETY: pointing directly after an object is considered 179 | // within the same object 180 | acme: unsafe { (slice as *mut T).add(slice.len()).cast() }, 181 | } 182 | } 183 | 184 | // NOTE: This should be removed in a future release as it encouraged UB. 185 | // Once `const_mut_refs` is stabilized in Rust, this will no longer be useful anyway. 186 | // See: https://github.com/SFBdragon/talc/issues/33 187 | #[deprecated = "Conversion from const references encourages UB. This will be removed in a future release."] 188 | #[cfg(feature = "nightly_api")] 189 | #[inline] 190 | pub const fn from_const_slice(slice: *const [T]) -> Self { 191 | Self { 192 | base: slice as *mut T as *mut u8, 193 | // SAFETY: pointing directly after an object is considered 194 | // within the same object 195 | acme: unsafe { (slice as *mut T).add(slice.len()).cast() }, 196 | } 197 | } 198 | 199 | #[inline] 200 | pub const fn from_array(array: *mut [T; N]) -> Self { 201 | Self { 202 | base: array as *mut T as *mut u8, 203 | // SAFETY: pointing directly after an object is considered 204 | // within the same object 205 | acme: unsafe { (array as *mut T).add(N).cast() }, 206 | } 207 | } 208 | 209 | // NOTE: This should be removed in a future release as it encouraged UB. 210 | // Once `const_mut_refs` is stabilized in Rust, this will no longer be useful anyway. 211 | // See: https://github.com/SFBdragon/talc/issues/33 212 | #[deprecated = "Conversion from const references encourages UB. This will be removed in a future release."] 213 | #[inline] 214 | pub const fn from_const_array(array: *const [T; N]) -> Self { 215 | Self { 216 | base: array as *mut T as *mut u8, 217 | // SAFETY: pointing directly after an object is considered 218 | // within the same object 219 | acme: unsafe { (array as *mut T).add(N).cast() }, 220 | } 221 | } 222 | 223 | /// Returns `None` if `self` is empty. 224 | #[inline] 225 | pub fn to_ptr_range(self) -> Option> { 226 | if self.is_empty() { None } else { Some(self.base..self.acme) } 227 | } 228 | 229 | /// Returns `None` if `self` is empty. 230 | #[inline] 231 | pub fn to_slice(self) -> Option<*mut [u8]> { 232 | if self.is_empty() { 233 | None 234 | } else { 235 | Some(core::ptr::slice_from_raw_parts_mut(self.base, self.size())) 236 | } 237 | } 238 | 239 | /// Returns whether `self` contains `addr`. 240 | /// 241 | /// Empty spans contain nothing. 242 | #[inline] 243 | pub fn contains(self, ptr: *mut u8) -> bool { 244 | // if self is empty, this always evaluates to false 245 | self.base <= ptr && ptr < self.acme 246 | } 247 | 248 | /// Returns whether `self` contains `other`. 249 | /// 250 | /// Empty spans are contained by any span, even empty ones. 251 | #[inline] 252 | pub fn contains_span(self, other: Span) -> bool { 253 | other.is_empty() || self.base <= other.base && other.acme <= self.acme 254 | } 255 | 256 | /// Returns whether some of `self` overlaps with `other`. 257 | /// 258 | /// Empty spans don't overlap with anything. 259 | #[inline] 260 | pub fn overlaps(self, other: Span) -> bool { 261 | self.is_sized() && other.is_sized() && !(other.base >= self.acme || self.base >= other.acme) 262 | } 263 | 264 | /// Aligns `base` upward and `acme` downward by `align_of::()`. 265 | #[inline] 266 | pub fn word_align_inward(self) -> Self { 267 | if ALIGN > usize::MAX - self.base as usize { 268 | Self::empty() 269 | } else { 270 | Self { base: align_up(self.base), acme: align_down(self.acme) } 271 | } 272 | } 273 | /// Aligns `base` downward and `acme` upward by `align_of::()`. 274 | #[inline] 275 | pub fn word_align_outward(self) -> Self { 276 | if ALIGN > usize::MAX - self.acme as usize { 277 | panic!("aligning acme upward would overflow!"); 278 | } 279 | 280 | Self { base: align_down(self.base), acme: align_up(self.acme) } 281 | } 282 | 283 | /// Raises `base` if `base` is smaller than `min`. 284 | #[inline] 285 | pub fn above(self, min: *mut u8) -> Self { 286 | Self { base: if min > self.base { min } else { self.base }, acme: self.acme } 287 | } 288 | /// Lowers `acme` if `acme` is greater than `max`. 289 | #[inline] 290 | pub fn below(self, max: *mut u8) -> Self { 291 | Self { base: self.base, acme: if max < self.acme { max } else { self.acme } } 292 | } 293 | 294 | /// Returns the [`Span`]s of `self` below and above the `exclude` span, respectively. 295 | /// Alternatively worded, the set difference `self`\\`exclude`. 296 | /// 297 | /// If `exclude` is empty, `self` and an empty `Span` are returned. 298 | #[inline] 299 | pub fn except(self, exclude: Span) -> (Self, Self) { 300 | match exclude.get_base_acme() { 301 | Some((base, acme)) => (self.below(base), self.above(acme)), 302 | None => (self, Span::empty()), 303 | } 304 | } 305 | 306 | /// Returns a span that `other` contains by raising `base` or lowering `acme`. 307 | /// 308 | /// If `other` is empty, returns `other`. 309 | #[inline] 310 | pub fn fit_within(self, other: Span) -> Self { 311 | if other.is_empty() { 312 | other 313 | } else { 314 | Self { 315 | base: if other.base > self.base { other.base } else { self.base }, 316 | acme: if other.acme < self.acme { other.acme } else { self.acme }, 317 | } 318 | } 319 | } 320 | /// Returns a span that contains `other` by extending `self`. 321 | /// 322 | /// If `other` is empty, returns `self`, as all spans contain any empty span. 323 | #[inline] 324 | pub fn fit_over(self, other: Self) -> Self { 325 | if other.is_empty() { 326 | self 327 | } else { 328 | Self { 329 | base: if other.base < self.base { other.base } else { self.base }, 330 | acme: if other.acme > self.acme { other.acme } else { self.acme }, 331 | } 332 | } 333 | } 334 | 335 | /// Lower `base` by `low` and raise `acme` by `high`. 336 | /// 337 | /// Does nothing if `self` is empty. 338 | /// 339 | /// # Panics 340 | /// Panics if lowering `base` by `low` or raising `acme` by `high` under/overflows. 341 | #[inline] 342 | pub fn extend(self, low: usize, high: usize) -> Self { 343 | if self.is_empty() { 344 | self 345 | } else { 346 | assert!((self.base as usize).checked_sub(low).is_some()); 347 | assert!((self.acme as usize).checked_add(high).is_some()); 348 | 349 | Self { base: self.base.wrapping_sub(low), acme: self.acme.wrapping_add(high) } 350 | } 351 | } 352 | 353 | /// Raise `base` by `low` and lower `acme` by `high`. 354 | /// 355 | /// If `self` is empty, `self` is returned. 356 | /// 357 | /// If either operation would wrap around the address space, an empty span is returned. 358 | #[inline] 359 | pub fn truncate(self, low: usize, high: usize) -> Span { 360 | if self.is_empty() { 361 | self 362 | } else if (self.base as usize).checked_add(low).is_none() 363 | || (self.acme as usize).checked_sub(high).is_none() 364 | { 365 | Span::empty() 366 | } else { 367 | Self { 368 | // if either boundary saturates, the span will be empty thereafter, as expected 369 | base: self.base.wrapping_add(low), 370 | acme: self.acme.wrapping_sub(high), 371 | } 372 | } 373 | } 374 | } 375 | 376 | #[cfg(test)] 377 | mod test { 378 | use super::*; 379 | 380 | fn ptr(addr: usize) -> *mut u8 { 381 | // don't ` as usize` to avoid upsetting miri too much 382 | core::ptr::null_mut::().wrapping_add(addr) 383 | } 384 | 385 | #[test] 386 | fn test_span() { 387 | let base = 1234usize; 388 | let acme = 5678usize; 389 | 390 | let bptr = ptr(base); 391 | let aptr = ptr(acme); 392 | 393 | let span = Span::from(bptr..aptr); 394 | assert!(!span.is_empty()); 395 | assert!(span.size() == acme - base); 396 | 397 | assert!( 398 | span.word_align_inward() 399 | == Span::new( 400 | bptr.wrapping_add(ALIGN - 1) 401 | .wrapping_sub(bptr.wrapping_add(ALIGN - 1) as usize & (ALIGN - 1)), 402 | aptr.wrapping_sub(acme & (ALIGN - 1)) 403 | ) 404 | ); 405 | assert!( 406 | span.word_align_outward() 407 | == Span::new( 408 | bptr.wrapping_sub(base & (ALIGN - 1)), 409 | aptr.wrapping_add(ALIGN - 1) 410 | .wrapping_sub(aptr.wrapping_add(ALIGN - 1) as usize & (ALIGN - 1)) 411 | ) 412 | ); 413 | 414 | assert_eq!(span.above(ptr(2345)), Span::new(ptr(2345), aptr)); 415 | assert_eq!(span.below(ptr(7890)), Span::new(bptr, aptr)); 416 | assert_eq!(span.below(ptr(3456)), Span::new(bptr, ptr(3456))); 417 | assert_eq!(span.below(ptr(0123)), Span::empty()); 418 | assert_eq!(span.above(ptr(7890)), Span::empty()); 419 | 420 | assert_eq!( 421 | span.except(Span::new(ptr(base + 1111), ptr(acme - 1111))), 422 | (Span::new(bptr, ptr(base + 1111)), Span::new(ptr(acme - 1111), aptr)) 423 | ); 424 | assert_eq!( 425 | span.except(Span::new(ptr(base + 1111), ptr(acme + 1111))), 426 | (Span::new(bptr, ptr(base + 1111)), Span::empty()) 427 | ); 428 | assert_eq!( 429 | span.except(Span::new(ptr(base - 1111), ptr(acme + 1111))), 430 | (Span::empty(), Span::empty()) 431 | ); 432 | assert_eq!(span.except(Span::empty()), (span, Span::empty())); 433 | 434 | assert!(span.fit_over(Span::empty()) == span); 435 | assert!(span.fit_within(Span::empty()).is_empty()); 436 | assert!(span.fit_within(Span::new(ptr(0), ptr(10000))) == span); 437 | assert!(span.fit_over(Span::new(ptr(0), ptr(10000))) == Span::new(ptr(0), ptr(10000))); 438 | assert!(span.fit_within(Span::new(ptr(4000), ptr(10000))) == Span::new(ptr(4000), aptr)); 439 | assert!(span.fit_over(Span::new(ptr(4000), ptr(10000))) == Span::new(bptr, ptr(10000))); 440 | 441 | assert!(span.extend(1234, 1010) == Span::new(ptr(0), ptr(5678 + 1010))); 442 | assert!(span.truncate(1234, 1010) == Span::new(ptr(1234 + 1234), ptr(5678 - 1010))); 443 | assert!(span.truncate(235623, 45235772).is_empty()); 444 | } 445 | } 446 | -------------------------------------------------------------------------------- /talc/src/talc.rs: -------------------------------------------------------------------------------- 1 | mod llist; 2 | mod tag; 3 | 4 | #[cfg(feature = "counters")] 5 | pub mod counters; 6 | 7 | use crate::{ptr_utils::*, OomHandler, Span}; 8 | use core::{ 9 | alloc::Layout, 10 | ptr::{null_mut, NonNull}, 11 | }; 12 | use llist::LlistNode; 13 | use tag::Tag; 14 | 15 | const NODE_SIZE: usize = core::mem::size_of::(); 16 | const TAG_SIZE: usize = core::mem::size_of::(); 17 | 18 | const MIN_TAG_OFFSET: usize = NODE_SIZE; 19 | const MIN_CHUNK_SIZE: usize = MIN_TAG_OFFSET + TAG_SIZE; 20 | const MIN_HEAP_SIZE: usize = MIN_CHUNK_SIZE + TAG_SIZE; 21 | 22 | const BIN_COUNT: usize = usize::BITS as usize * 2; 23 | 24 | type Bin = Option>; 25 | 26 | // Free chunk (3x ptr size minimum): 27 | // ?? | NODE: LlistNode (2 * ptr), SIZE: usize, ..???.., SIZE: usize | ?? 28 | // Reserved chunk (1x ptr size of overhead): 29 | // ?? | ??????? , TAG: Tag (ptr) | ?? 30 | 31 | // TAG contains a pointer to the bottom of the reserved chunk, 32 | // a is_allocated (set) bit flag differentiating itself from a free chunk 33 | // (the LlistNode contains well-aligned pointers, thus does not have that bit set), 34 | // as well as a is_low_free bit flag which does what is says on the tin 35 | 36 | const GAP_NODE_OFFSET: usize = 0; 37 | const GAP_LOW_SIZE_OFFSET: usize = NODE_SIZE; 38 | const GAP_HIGH_SIZE_OFFSET: usize = WORD_SIZE; 39 | 40 | // WASM perf tanks if these #[inline]'s are not present 41 | #[inline] 42 | unsafe fn gap_base_to_node(base: *mut u8) -> *mut LlistNode { 43 | base.add(GAP_NODE_OFFSET).cast() 44 | } 45 | #[inline] 46 | unsafe fn gap_base_to_size(base: *mut u8) -> *mut usize { 47 | base.add(GAP_LOW_SIZE_OFFSET).cast() 48 | } 49 | #[inline] 50 | unsafe fn gap_base_to_acme(base: *mut u8) -> *mut u8 { 51 | gap_base_to_acme_size(base).0 52 | } 53 | #[inline] 54 | unsafe fn gap_base_to_acme_size(base: *mut u8) -> (*mut u8, usize) { 55 | let size = gap_base_to_size(base).read(); 56 | (base.add(size), size) 57 | } 58 | #[inline] 59 | unsafe fn gap_acme_to_size(acme: *mut u8) -> *mut usize { 60 | acme.sub(GAP_HIGH_SIZE_OFFSET).cast() 61 | } 62 | #[inline] 63 | unsafe fn gap_acme_to_base(acme: *mut u8) -> *mut u8 { 64 | gap_acme_to_base_size(acme).0 65 | } 66 | #[inline] 67 | unsafe fn gap_acme_to_base_size(acme: *mut u8) -> (*mut u8, usize) { 68 | let size = gap_acme_to_size(acme).read(); 69 | (acme.sub(size), size) 70 | } 71 | #[inline] 72 | unsafe fn gap_node_to_base(node: NonNull) -> *mut u8 { 73 | node.as_ptr().cast::().sub(GAP_NODE_OFFSET).cast() 74 | } 75 | #[inline] 76 | unsafe fn gap_node_to_size(node: NonNull) -> *mut usize { 77 | node.as_ptr().cast::().sub(GAP_NODE_OFFSET).add(GAP_LOW_SIZE_OFFSET).cast() 78 | } 79 | #[inline] 80 | unsafe fn is_gap_below(acme: *mut u8) -> bool { 81 | // gap size will never have bit 1 set, but a tag will 82 | gap_acme_to_size(acme).read() & Tag::ALLOCATED_FLAG == 0 83 | } 84 | #[inline] 85 | unsafe fn is_gap_above_heap_base(heap_base: *mut u8) -> bool { 86 | // there's a tag at every heap base 87 | heap_base.cast::().read().is_above_free() 88 | } 89 | 90 | /// Determines the tag pointer and retrieves the tag, given the allocated pointer. 91 | #[inline] 92 | unsafe fn tag_from_alloc_ptr(ptr: *mut u8, size: usize) -> (*mut u8, Tag) { 93 | let post_alloc_ptr = align_up(ptr.add(size)); 94 | // we're either reading a tag_ptr or a Tag with the base pointer + metadata in the low bits 95 | let tag_or_tag_ptr = post_alloc_ptr.cast::<*mut u8>().read(); 96 | 97 | // if the pointer is greater, it's a tag_ptr 98 | // if it's less, it's a tag, effectively a base pointer 99 | // (the low bits of metadata in a tag don't effect the inequality) 100 | if tag_or_tag_ptr > post_alloc_ptr { 101 | (tag_or_tag_ptr, tag_or_tag_ptr.cast::().read()) 102 | } else { 103 | (post_alloc_ptr, Tag(tag_or_tag_ptr)) 104 | } 105 | } 106 | 107 | /// Returns whether the two pointers are greater than `MIN_CHUNK_SIZE` apart. 108 | #[inline] 109 | fn is_chunk_size(base: *mut u8, acme: *mut u8) -> bool { 110 | debug_assert!(acme >= base, "!(acme {:p} >= base {:p})", acme, base); 111 | acme as usize - base as usize >= MIN_CHUNK_SIZE 112 | } 113 | 114 | /// `size` should be larger or equal to MIN_CHUNK_SIZE 115 | #[inline] 116 | unsafe fn bin_of_size(size: usize) -> usize { 117 | // this mess determines the bucketing strategy used by the allocator 118 | // the default is to have a bucket per multiple of word size from the minimum 119 | // chunk size up to WORD_BUCKETED_SIZE and double word gap (sharing two sizes) 120 | // up to DOUBLE_BUCKETED_SIZE, and from there on use pseudo-logarithmic sizes. 121 | 122 | // such sizes are as follows: begin at some power of two (DOUBLE_BUCKETED_SIZE) 123 | // and increase by some power of two fraction (quarters, on 64 bit machines) 124 | // until reaching the next power of two, and repeat: 125 | // e.g. begin at 32, increase by quarters: 32, 40, 48, 56, 64, 80, 96, 112, 128, ... 126 | 127 | // note to anyone adding support for another word size: use buckets.py to figure it out 128 | const ERRMSG: &str = "Unsupported system word size, open an issue/create a PR!"; 129 | 130 | /// up to what size do we use a bin for every multiple of a word 131 | const WORD_BIN_LIMIT: usize = match WORD_SIZE { 132 | 8 => 256, 133 | 4 => 64, 134 | _ => panic!("{}", ERRMSG), 135 | }; 136 | /// up to what size beyond that do we use a bin for every multiple of a doubleword 137 | const DOUBLE_BIN_LIMIT: usize = match WORD_SIZE { 138 | 8 => 512, 139 | 4 => 128, 140 | _ => panic!("{}", ERRMSG), 141 | }; 142 | /// how many buckets are linearly spaced among each power of two magnitude (how many divisions) 143 | const DIVS_PER_POW2: usize = match WORD_SIZE { 144 | 8 => 4, 145 | 4 => 2, 146 | _ => panic!("{}", ERRMSG), 147 | }; 148 | /// how many bits are used to determine the division 149 | const DIV_BITS: usize = DIVS_PER_POW2.ilog2() as usize; 150 | 151 | /// the bucket index at which the doubleword separated buckets start 152 | const DBL_BUCKET: usize = (WORD_BIN_LIMIT - MIN_CHUNK_SIZE) / WORD_SIZE; 153 | /// the bucket index at which the peudo-exponentially separated buckets start 154 | const EXP_BUCKET: usize = DBL_BUCKET + (DOUBLE_BIN_LIMIT - WORD_BIN_LIMIT) / WORD_SIZE / 2; 155 | /// Log 2 of (minimum pseudo-exponential chunk size) 156 | const MIN_EXP_BITS_LESS_ONE: usize = DOUBLE_BIN_LIMIT.ilog2() as usize; 157 | 158 | debug_assert!(size >= MIN_CHUNK_SIZE); 159 | 160 | if size < WORD_BIN_LIMIT { 161 | // single word separated bucket 162 | 163 | (size - MIN_CHUNK_SIZE) / WORD_SIZE 164 | } else if size < DOUBLE_BIN_LIMIT { 165 | // double word separated bucket 166 | 167 | // equiv to (size - WORD_BIN_LIMIT) / 2WORD_SIZE + DBL_BUCKET 168 | // but saves an instruction 169 | size / (2 * WORD_SIZE) - WORD_BIN_LIMIT / (2 * WORD_SIZE) + DBL_BUCKET 170 | } else { 171 | // pseudo-exponentially separated bucket 172 | 173 | // here's what a size is, bit by bit: 1_div_extra 174 | // e.g. with four divisions 1_01_00010011000 175 | // the bucket is determined by the magnitude and the division 176 | // mag 0 div 0, mag 0 div 1, mag 0 div 2, mag 0 div 3, mag 1 div 0, ... 177 | 178 | let bits_less_one = size.ilog2() as usize; 179 | 180 | // the magnitude the size belongs to. 181 | // calculate the difference in bit count i.e. difference in power 182 | let magnitude = bits_less_one - MIN_EXP_BITS_LESS_ONE; 183 | // the division of the magnitude the size belongs to. 184 | // slide the size to get the division bits at the bottom and remove the top bit 185 | let division = (size >> (bits_less_one - DIV_BITS)) - DIVS_PER_POW2; 186 | // the index into the pseudo-exponential buckets. 187 | let bucket_offset = magnitude * DIVS_PER_POW2 + division; 188 | 189 | // cap the max bucket at the last bucket 190 | (bucket_offset + EXP_BUCKET).min(BIN_COUNT - 1) 191 | } 192 | } 193 | 194 | /// The Talc Allocator! 195 | /// 196 | /// One way to get started: 197 | /// 1. Construct with [`new`](Talc::new) (supply [`ErrOnOom`] to ignore OOM handling). 198 | /// 2. Establish any number of heaps with [`claim`](Talc::claim). 199 | /// 3. Call [`lock`](Talc::lock) to get a [`Talck`] which supports the 200 | /// [`GlobalAlloc`](core::alloc::GlobalAlloc) and [`Allocator`](core::alloc::Allocator) traits. 201 | /// 202 | /// Check out the associated functions `new`, `claim`, `lock`, `extend`, and `truncate`. 203 | pub struct Talc { 204 | /// The low bits of the availability flags. 205 | availability_low: usize, 206 | /// The high bits of the availability flags. 207 | availability_high: usize, 208 | /// Linked list heads. 209 | bins: *mut Bin, 210 | 211 | /// The user-specified OOM handler. 212 | /// 213 | /// Its state is entirely maintained by the user. 214 | pub oom_handler: O, 215 | 216 | #[cfg(feature = "counters")] 217 | /// Allocation stats. 218 | counters: counters::Counters, 219 | } 220 | 221 | unsafe impl Send for Talc {} 222 | 223 | impl core::fmt::Debug for Talc { 224 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 225 | f.debug_struct("Talc") 226 | .field("availability_low", &format_args!("{:x}", self.availability_low)) 227 | .field("availability_high", &format_args!("{:x}", self.availability_high)) 228 | .field("metadata_ptr", &self.bins) 229 | .finish() 230 | } 231 | } 232 | 233 | impl Talc { 234 | #[inline] 235 | const fn required_chunk_size(size: usize) -> usize { 236 | if size <= MIN_CHUNK_SIZE - TAG_SIZE { 237 | MIN_CHUNK_SIZE 238 | } else { 239 | (size + TAG_SIZE + (ALIGN - 1)) & !(ALIGN - 1) 240 | } 241 | } 242 | 243 | /// Get the pointer to the `bin`th bin. 244 | /// # Safety 245 | /// `bin` must be smaller than `BIN_COUNT`. 246 | #[inline] 247 | unsafe fn get_bin_ptr(&self, bin: usize) -> *mut Bin { 248 | debug_assert!(bin < BIN_COUNT); 249 | 250 | self.bins.add(bin) 251 | } 252 | 253 | /// Sets the availability flag for bin `b`. 254 | /// 255 | /// This is done when a chunk is added to an empty bin. 256 | #[inline] 257 | fn set_avails(&mut self, b: usize) { 258 | debug_assert!(b < BIN_COUNT); 259 | 260 | if b < WORD_BITS { 261 | debug_assert!(self.availability_low & 1 << b == 0); 262 | self.availability_low ^= 1 << b; 263 | } else { 264 | debug_assert!(self.availability_high & 1 << (b - WORD_BITS) == 0); 265 | self.availability_high ^= 1 << (b - WORD_BITS); 266 | } 267 | } 268 | /// Clears the availability flag for bin `b`. 269 | /// 270 | /// This is done when a bin becomes empty. 271 | #[inline] 272 | fn clear_avails(&mut self, b: usize) { 273 | debug_assert!(b < BIN_COUNT); 274 | 275 | // if head is the last node 276 | if b < WORD_BITS { 277 | self.availability_low ^= 1 << b; 278 | debug_assert!(self.availability_low & 1 << b == 0); 279 | } else { 280 | self.availability_high ^= 1 << (b - WORD_BITS); 281 | debug_assert!(self.availability_high & 1 << (b - WORD_BITS) == 0); 282 | } 283 | } 284 | 285 | /// Registers a gap in memory which is allocatable. 286 | #[inline] 287 | unsafe fn register_gap(&mut self, base: *mut u8, acme: *mut u8) { 288 | debug_assert!(is_chunk_size(base, acme)); 289 | 290 | let size = acme as usize - base as usize; 291 | let bin = bin_of_size(size); 292 | 293 | let bin_ptr = self.get_bin_ptr(bin); 294 | 295 | if (*bin_ptr).is_none() { 296 | self.set_avails(bin); 297 | } 298 | 299 | LlistNode::insert(gap_base_to_node(base), bin_ptr, *bin_ptr); 300 | 301 | debug_assert!((*bin_ptr).is_some()); 302 | 303 | gap_base_to_size(base).write(size); 304 | gap_acme_to_size(acme).write(size); 305 | 306 | #[cfg(feature = "counters")] 307 | self.counters.account_register_gap(size); 308 | } 309 | 310 | /// Deregisters memory, not allowing it to be allocated. 311 | #[inline] 312 | unsafe fn deregister_gap(&mut self, base: *mut u8, bin: usize) { 313 | debug_assert!((*self.get_bin_ptr(bin)).is_some()); 314 | #[cfg(feature = "counters")] 315 | self.counters.account_deregister_gap(gap_base_to_size(base).read()); 316 | 317 | LlistNode::remove(gap_base_to_node(base)); 318 | 319 | if (*self.get_bin_ptr(bin)).is_none() { 320 | self.clear_avails(bin); 321 | } 322 | } 323 | 324 | /// Allocate a contiguous region of memory according to `layout`, if possible. 325 | /// # Safety 326 | /// `layout.size()` must be nonzero. 327 | pub unsafe fn malloc(&mut self, layout: Layout) -> Result, ()> { 328 | debug_assert!(layout.size() != 0); 329 | self.scan_for_errors(); 330 | 331 | let (mut free_base, free_acme, alloc_base) = loop { 332 | // this returns None if there are no heaps or allocatable memory 333 | match self.get_sufficient_chunk(layout) { 334 | Some(payload) => break payload, 335 | None => _ = O::handle_oom(self, layout)?, 336 | } 337 | }; 338 | 339 | // determine the base of the allocated chunk 340 | // if the amount of memory below the chunk is too small, subsume it, else free it 341 | let chunk_base_ceil = alloc_base.min(free_acme.sub(MIN_CHUNK_SIZE)); 342 | if is_chunk_size(free_base, chunk_base_ceil) { 343 | self.register_gap(free_base, chunk_base_ceil); 344 | free_base = chunk_base_ceil; 345 | } else { 346 | Tag::clear_above_free(free_base.sub(TAG_SIZE).cast()); 347 | } 348 | 349 | // the word immediately after the allocation 350 | let post_alloc_ptr = align_up(alloc_base.add(layout.size())); 351 | // the tag position, accounting for the minimum size of a chunk 352 | let mut tag_ptr = free_base.add(MIN_TAG_OFFSET).max(post_alloc_ptr); 353 | // the pointer after the lowest possible tag pointer 354 | let min_alloc_chunk_acme = tag_ptr.add(TAG_SIZE); 355 | 356 | // handle the space above the required allocation span 357 | if is_chunk_size(min_alloc_chunk_acme, free_acme) { 358 | self.register_gap(min_alloc_chunk_acme, free_acme); 359 | Tag::write(tag_ptr.cast(), free_base, true); 360 | } else { 361 | tag_ptr = free_acme.sub(TAG_SIZE); 362 | Tag::write(tag_ptr.cast(), free_base, false); 363 | } 364 | 365 | if tag_ptr != post_alloc_ptr { 366 | // write the real tag ptr where the tag is expected to be 367 | post_alloc_ptr.cast::<*mut u8>().write(tag_ptr); 368 | } 369 | 370 | #[cfg(feature = "counters")] 371 | self.counters.account_alloc(layout.size()); 372 | 373 | Ok(NonNull::new_unchecked(alloc_base)) 374 | } 375 | 376 | /// Returns `(chunk_base, chunk_acme, alloc_base)` 377 | unsafe fn get_sufficient_chunk( 378 | &mut self, 379 | layout: Layout, 380 | ) -> Option<(*mut u8, *mut u8, *mut u8)> { 381 | let required_chunk_size = Self::required_chunk_size(layout.size()); 382 | 383 | // if there are no valid heaps, availability is zero, and next_available_bin returns None 384 | let mut bin = self.next_available_bin(bin_of_size(required_chunk_size))?; 385 | 386 | if layout.align() <= ALIGN { 387 | // the required alignment is most often the machine word size (or less) 388 | // a faster loop without alignment checking is used in this case 389 | loop { 390 | for node_ptr in LlistNode::iter_mut(*self.get_bin_ptr(bin)) { 391 | let size = gap_node_to_size(node_ptr).read(); 392 | 393 | // if the chunk size is sufficient, remove from bookkeeping data structures and return 394 | if size >= required_chunk_size { 395 | let base = gap_node_to_base(node_ptr); 396 | self.deregister_gap(base, bin); 397 | return Some((base, base.add(size), base)); 398 | } 399 | } 400 | 401 | bin = self.next_available_bin(bin + 1)?; 402 | } 403 | } else { 404 | // a larger than word-size alignment is demanded 405 | // therefore each chunk is manually checked to be sufficient accordingly 406 | let align_mask = layout.align() - 1; 407 | let required_size = layout.size() + TAG_SIZE; 408 | 409 | loop { 410 | for node_ptr in LlistNode::iter_mut(*self.get_bin_ptr(bin)) { 411 | let size = gap_node_to_size(node_ptr).read(); 412 | 413 | if size >= required_chunk_size { 414 | let base = gap_node_to_base(node_ptr); 415 | let acme = base.add(size); 416 | // calculate the lowest aligned pointer above the tag-offset free chunk pointer 417 | let aligned_ptr = align_up_by(base, align_mask); 418 | 419 | // if the remaining size is sufficient, remove the chunk from the books and return 420 | if aligned_ptr.add(required_size) <= acme { 421 | self.deregister_gap(base, bin); 422 | return Some((base, acme, aligned_ptr)); 423 | } 424 | } 425 | } 426 | 427 | bin = self.next_available_bin(bin + 1)?; 428 | } 429 | } 430 | } 431 | 432 | #[inline(always)] 433 | fn next_available_bin(&self, next_bin: usize) -> Option { 434 | if next_bin < usize::BITS as usize { 435 | // shift flags such that only flags for larger buckets are kept 436 | let shifted_avails = self.availability_low >> next_bin; 437 | 438 | // find the next up, grab from the high flags, or quit 439 | if shifted_avails != 0 { 440 | Some(next_bin + shifted_avails.trailing_zeros() as usize) 441 | } else if self.availability_high != 0 { 442 | Some(self.availability_high.trailing_zeros() as usize + WORD_BITS) 443 | } else { 444 | None 445 | } 446 | } else if next_bin < BIN_COUNT { 447 | // similar process to the above, but the low flags are irrelevant 448 | let shifted_avails = self.availability_high >> (next_bin - WORD_BITS); 449 | 450 | if shifted_avails != 0 { 451 | Some(next_bin + shifted_avails.trailing_zeros() as usize) 452 | } else { 453 | return None; 454 | } 455 | } else { 456 | None 457 | } 458 | } 459 | 460 | /// Free previously allocated/reallocated memory. 461 | /// # Safety 462 | /// `ptr` must have been previously allocated given `layout`. 463 | pub unsafe fn free(&mut self, ptr: NonNull, layout: Layout) { 464 | self.scan_for_errors(); 465 | #[cfg(feature = "counters")] 466 | self.counters.account_dealloc(layout.size()); 467 | 468 | let (tag_ptr, tag) = tag_from_alloc_ptr(ptr.as_ptr(), layout.size()); 469 | let mut chunk_base = tag.chunk_base(); 470 | let mut chunk_acme = tag_ptr.add(TAG_SIZE); 471 | 472 | debug_assert!(tag.is_allocated()); 473 | debug_assert!(is_chunk_size(chunk_base, chunk_acme)); 474 | 475 | // try recombine below 476 | if is_gap_below(chunk_base) { 477 | let (below_base, below_size) = gap_acme_to_base_size(chunk_base); 478 | self.deregister_gap(below_base, bin_of_size(below_size)); 479 | 480 | chunk_base = below_base; 481 | } else { 482 | Tag::set_above_free(chunk_base.sub(TAG_SIZE).cast()); 483 | } 484 | 485 | // try recombine above 486 | if tag.is_above_free() { 487 | let above_size = gap_base_to_size(chunk_acme).read(); 488 | self.deregister_gap(chunk_acme, bin_of_size(above_size)); 489 | 490 | chunk_acme = chunk_acme.add(above_size); 491 | } 492 | 493 | // add the full recombined free chunk back into the books 494 | self.register_gap(chunk_base, chunk_acme); 495 | } 496 | 497 | /// Grow a previously allocated/reallocated region of memory to `new_size`. 498 | /// # Safety 499 | /// `ptr` must have been previously allocated or reallocated given `layout`. 500 | /// `new_size` must be larger or equal to `layout.size()`. 501 | pub unsafe fn grow( 502 | &mut self, 503 | ptr: NonNull, 504 | old_layout: Layout, 505 | new_size: usize, 506 | ) -> Result, ()> { 507 | match self.grow_in_place(ptr, old_layout, new_size) { 508 | Err(_) => { 509 | // grow in-place failed; reallocate the slow way 510 | let new_layout = Layout::from_size_align_unchecked(new_size, old_layout.align()); 511 | let allocation = self.malloc(new_layout)?; 512 | allocation.as_ptr().copy_from_nonoverlapping(ptr.as_ptr(), old_layout.size()); 513 | self.free(ptr, old_layout); 514 | 515 | Ok(allocation) 516 | } 517 | res => res, 518 | } 519 | } 520 | 521 | /// Attempt to grow a previously allocated/reallocated region of memory to `new_size`. 522 | /// 523 | /// Returns `Err` if reallocation could not occur in-place. 524 | /// Ownership of the memory remains with the caller. 525 | /// # Safety 526 | /// `ptr` must have been previously allocated or reallocated given `layout`. 527 | /// `new_size` must be larger or equal to `layout.size()`. 528 | pub unsafe fn grow_in_place( 529 | &mut self, 530 | ptr: NonNull, 531 | old_layout: Layout, 532 | new_size: usize, 533 | ) -> Result, ()> { 534 | debug_assert!(new_size >= old_layout.size()); 535 | self.scan_for_errors(); 536 | 537 | let old_post_alloc_ptr = align_up(ptr.as_ptr().add(old_layout.size())); 538 | let new_post_alloc_ptr = align_up(ptr.as_ptr().add(new_size)); 539 | 540 | if old_post_alloc_ptr == new_post_alloc_ptr { 541 | // this handles a rare short-circuit, but more helpfully 542 | // also guarantees that we'll never need to add padding to 543 | // reach minimum chunk size with new_tag_ptr later as 544 | // min alloc size (1) rounded up to (WORD) + post_alloc_ptr (WORD) + new_tag_ptr (WORD) >= MIN_CHUNK_SIZE 545 | 546 | #[cfg(feature = "counters")] 547 | self.counters.account_grow_in_place(old_layout.size(), new_size); 548 | 549 | return Ok(ptr); 550 | } 551 | 552 | let (tag_ptr, tag) = tag_from_alloc_ptr(ptr.as_ptr(), old_layout.size()); 553 | 554 | // tag_ptr may be greater where extra free space needed to be reserved 555 | if new_post_alloc_ptr <= tag_ptr { 556 | if new_post_alloc_ptr < tag_ptr { 557 | new_post_alloc_ptr.cast::<*mut u8>().write(tag_ptr); 558 | } 559 | 560 | #[cfg(feature = "counters")] 561 | self.counters.account_grow_in_place(old_layout.size(), new_size); 562 | 563 | return Ok(ptr); 564 | } 565 | 566 | let new_tag_ptr = new_post_alloc_ptr; 567 | 568 | let base = tag.chunk_base(); 569 | let acme = tag_ptr.add(TAG_SIZE); 570 | 571 | debug_assert!(tag.is_allocated()); 572 | debug_assert!(is_chunk_size(base, acme)); 573 | 574 | // otherwise, check if 1) is free 2) is large enough 575 | // because free chunks don't border free chunks, this needn't be recursive 576 | if tag.is_above_free() { 577 | let above_size = gap_base_to_size(acme).read(); 578 | let above_tag_ptr = tag_ptr.add(above_size); 579 | 580 | if new_tag_ptr <= above_tag_ptr { 581 | self.deregister_gap(acme, bin_of_size(above_size)); 582 | 583 | // finally, determine if the remainder of the free block is big enough 584 | // to be freed again, or if the entire region should be allocated 585 | if is_chunk_size(new_tag_ptr, above_tag_ptr) { 586 | self.register_gap(new_tag_ptr.add(TAG_SIZE), above_tag_ptr.add(TAG_SIZE)); 587 | Tag::write(new_tag_ptr.cast(), base, true); 588 | } else { 589 | Tag::write(above_tag_ptr.cast(), base, false); 590 | 591 | if new_post_alloc_ptr != above_tag_ptr { 592 | new_post_alloc_ptr.cast::<*mut u8>().write(above_tag_ptr); 593 | } 594 | } 595 | 596 | #[cfg(feature = "counters")] 597 | self.counters.account_grow_in_place(old_layout.size(), new_size); 598 | 599 | return Ok(ptr); 600 | } 601 | } 602 | 603 | Err(()) 604 | } 605 | 606 | /// Shrink a previously allocated/reallocated region of memory to `new_size`. 607 | /// 608 | /// This function is infallible given valid inputs, and the reallocation will always be 609 | /// done in-place, maintaining the validity of the pointer. 610 | /// 611 | /// # Safety 612 | /// - `ptr` must have been previously allocated or reallocated given `layout`. 613 | /// - `new_size` must be smaller or equal to `layout.size()`. 614 | /// - `new_size` should be nonzero. 615 | pub unsafe fn shrink(&mut self, ptr: NonNull, layout: Layout, new_size: usize) { 616 | debug_assert!(new_size != 0); 617 | debug_assert!(new_size <= layout.size()); 618 | self.scan_for_errors(); 619 | 620 | let (tag_ptr, tag) = tag_from_alloc_ptr(ptr.as_ptr(), layout.size()); 621 | let chunk_base = tag.chunk_base(); 622 | 623 | debug_assert!(tag.is_allocated()); 624 | debug_assert!(is_chunk_size(chunk_base, tag_ptr.add(TAG_SIZE))); 625 | 626 | // the word immediately after the allocation 627 | let new_post_alloc_ptr = align_up(ptr.as_ptr().add(new_size)); 628 | // the tag position, accounting for the minimum size of a chunk 629 | let mut new_tag_ptr = chunk_base.add(MIN_TAG_OFFSET).max(new_post_alloc_ptr); 630 | 631 | // if the remainder between the new required size and the originally allocated 632 | // size is large enough, free the remainder, otherwise leave it 633 | if is_chunk_size(new_tag_ptr, tag_ptr) { 634 | let mut acme = tag_ptr.add(TAG_SIZE); 635 | let new_acme = new_tag_ptr.add(TAG_SIZE); 636 | 637 | if tag.is_above_free() { 638 | let above_size = gap_base_to_size(acme).read(); 639 | self.deregister_gap(acme, bin_of_size(above_size)); 640 | 641 | acme = acme.add(above_size); 642 | } 643 | 644 | self.register_gap(new_acme, acme); 645 | Tag::write(new_tag_ptr.cast(), chunk_base, true); 646 | } else { 647 | new_tag_ptr = tag_ptr; 648 | } 649 | 650 | if new_tag_ptr != new_post_alloc_ptr { 651 | new_post_alloc_ptr.cast::<*mut u8>().write(new_tag_ptr); 652 | } 653 | 654 | #[cfg(feature = "counters")] 655 | self.counters.account_shrink_in_place(layout.size(), new_size); 656 | } 657 | 658 | /// Returns an uninitialized [`Talc`]. 659 | /// 660 | /// If you don't want to handle OOM, use [`ErrOnOom`]. 661 | /// 662 | /// In order to make this allocator useful, `claim` some memory. 663 | pub const fn new(oom_handler: O) -> Self { 664 | Self { 665 | oom_handler, 666 | availability_low: 0, 667 | availability_high: 0, 668 | bins: null_mut(), 669 | 670 | #[cfg(feature = "counters")] 671 | counters: counters::Counters::new(), 672 | } 673 | } 674 | 675 | /// Returns the minimum [`Span`] containing this heap's allocated memory. 676 | /// # Safety 677 | /// `heap` must be the return value of a heap manipulation function. 678 | pub unsafe fn get_allocated_span(&self, heap: Span) -> Span { 679 | assert!(heap.size() >= MIN_HEAP_SIZE); 680 | 681 | let (mut base, mut acme) = heap.get_base_acme().unwrap(); 682 | 683 | // check for free space at the heap's top 684 | if is_gap_below(acme) { 685 | acme = gap_acme_to_base(acme); 686 | } 687 | 688 | // check for free memory at the bottom of the heap using the base tag 689 | if is_gap_above_heap_base(base) { 690 | base = gap_base_to_acme(base.add(TAG_SIZE)).sub(TAG_SIZE); 691 | } 692 | 693 | // base might be greater that acme for an empty heap 694 | // but that's fine, this'll just become an empty span 695 | Span::new(base, acme) 696 | } 697 | 698 | /// Attempt to initialize a new heap for the allocator. 699 | /// 700 | /// Note: 701 | /// * Each heap reserves a `usize` at the bottom as fixed overhead. 702 | /// * Metadata will be placed into the bottom of the first successfully established heap. 703 | /// It is currently ~1KiB on 64-bit systems (less on 32-bit). This is subject to change. 704 | /// 705 | /// # Return Values 706 | /// The resulting [`Span`] is the actual heap extent, and may 707 | /// be slightly smaller than requested. Use this to resize the heap. 708 | /// Any memory outside the claimed heap is free to use. 709 | /// 710 | /// Returns [`Err`] where 711 | /// * allocator metadata is not yet established, and there's insufficient memory to do so. 712 | /// * allocator metadata is established, but the heap is too small 713 | /// (less than around `4 * usize` for now). 714 | /// 715 | /// # Safety 716 | /// - The memory within the `memory` must be valid for reads and writes, 717 | /// and memory therein (when not allocated to the user) must not be mutated 718 | /// while the allocator is in use. 719 | /// - `memory` should not overlap with any other active heap. 720 | /// 721 | /// # Panics 722 | /// Panics if `memory` contains the null address. 723 | pub unsafe fn claim(&mut self, memory: Span) -> Result { 724 | self.scan_for_errors(); 725 | 726 | const BIN_ARRAY_SIZE: usize = core::mem::size_of::() * BIN_COUNT; 727 | 728 | // create a new heap 729 | // if bins is null, we will need to try put the metadata in this heap 730 | // this metadata is allocated 'by hand' to be isomorphic with other chunks 731 | 732 | assert!(!memory.contains(null_mut()), "heap covers the null address!"); 733 | 734 | let aligned_heap = memory.word_align_inward(); 735 | 736 | // if this fails, there's no space to work with 737 | if let Some((base, acme)) = aligned_heap.get_base_acme() { 738 | // check if the allocator has already successfully placed its metadata 739 | if !self.bins.is_null() { 740 | // check if there's enough space to establish a free chunk 741 | if acme as usize - base as usize >= MIN_HEAP_SIZE { 742 | // write in the base tag 743 | Tag::write(base.cast(), null_mut(), true); 744 | 745 | // register the free memory 746 | let chunk_base = base.wrapping_add(TAG_SIZE); 747 | self.register_gap(chunk_base, acme); 748 | 749 | self.scan_for_errors(); 750 | 751 | #[cfg(feature = "counters")] 752 | self.counters.account_claim(aligned_heap.size()); 753 | 754 | return Ok(aligned_heap); 755 | } 756 | } else { 757 | // check if there's enough space to allocate metadata and establish a free chunk 758 | if acme as usize - base as usize >= TAG_SIZE + BIN_ARRAY_SIZE + TAG_SIZE { 759 | Tag::write(base.cast(), null_mut(), false); 760 | 761 | // align the metadata pointer against the base of the heap 762 | let metadata_ptr = base.add(TAG_SIZE); 763 | // align the tag pointer against the top of the metadata 764 | let post_metadata_ptr = metadata_ptr.add(BIN_ARRAY_SIZE); 765 | 766 | // initialize the bins to None 767 | for i in 0..BIN_COUNT { 768 | let bin_ptr = metadata_ptr.cast::().add(i); 769 | bin_ptr.write(None); 770 | } 771 | 772 | self.bins = metadata_ptr.cast::(); 773 | 774 | // check whether there's enough room on top to free 775 | // add_chunk_to_record only depends on self.bins 776 | let metadata_chunk_acme = post_metadata_ptr.add(TAG_SIZE); 777 | if is_chunk_size(metadata_chunk_acme, acme) { 778 | self.register_gap(metadata_chunk_acme, acme); 779 | Tag::write(post_metadata_ptr.cast(), base, true); 780 | } else { 781 | let tag_ptr = acme.sub(TAG_SIZE).cast::(); 782 | 783 | if tag_ptr != post_metadata_ptr.cast() { 784 | post_metadata_ptr.cast::<*mut Tag>().write(tag_ptr); 785 | } 786 | Tag::write(tag_ptr, base, false); 787 | } 788 | 789 | self.scan_for_errors(); 790 | 791 | #[cfg(feature = "counters")] 792 | self.counters.account_claim(aligned_heap.size()); 793 | 794 | return Ok(aligned_heap); 795 | } 796 | } 797 | } 798 | 799 | // fallthrough from insufficient size 800 | 801 | Err(()) 802 | } 803 | 804 | /// Increase the extent of a heap. The new extent of the heap is returned, 805 | /// and will be equal to or slightly smaller than requested. 806 | /// 807 | /// # Safety 808 | /// - `old_heap` must be the return value of a heap-manipulation function 809 | /// of this allocator instance. 810 | /// - The entire `req_heap` memory but be readable and writable 811 | /// and unmutated besides that which is allocated so long as the heap is in use. 812 | /// 813 | /// # Panics 814 | /// This function panics if: 815 | /// - `old_heap` is too small or heap metadata is not yet allocated 816 | /// - `req_heap` doesn't contain `old_heap` 817 | /// - `req_heap` contains the null address 818 | /// 819 | /// A recommended pattern for satisfying these criteria is: 820 | /// ```rust 821 | /// # use talc::*; 822 | /// # let mut talc = Talc::new(ErrOnOom); 823 | /// let mut heap = [0u8; 2000]; 824 | /// let old_heap = Span::from(&mut heap[300..1700]); 825 | /// let old_heap = unsafe { talc.claim(old_heap).unwrap() }; 826 | /// 827 | /// // compute the new heap span as an extension of the old span 828 | /// let new_heap = old_heap.extend(250, 500).fit_within((&mut heap[..]).into()); 829 | /// 830 | /// // SAFETY: be sure not to extend into memory we can't use 831 | /// let new_heap = unsafe { talc.extend(old_heap, new_heap) }; 832 | /// ``` 833 | pub unsafe fn extend(&mut self, old_heap: Span, req_heap: Span) -> Span { 834 | assert!(!self.bins.is_null()); 835 | assert!(old_heap.size() >= MIN_HEAP_SIZE); 836 | assert!(req_heap.contains_span(old_heap), "new_heap must contain old_heap"); 837 | assert!(!req_heap.contains(null_mut()), "new_heap covers the null address!"); 838 | 839 | self.scan_for_errors(); 840 | 841 | let (old_base, old_acme) = old_heap.word_align_inward().get_base_acme().unwrap(); 842 | let (new_base, new_acme) = req_heap.word_align_inward().get_base_acme().unwrap(); 843 | let new_chunk_base = new_base.add(TAG_SIZE); 844 | let mut ret_base = new_base; 845 | let mut ret_acme = new_acme; 846 | 847 | // if the top chunk is free, extend the block to cover the new extra area 848 | // otherwise allocate above if possible 849 | if is_gap_below(old_acme) { 850 | let (top_base, top_size) = gap_acme_to_base_size(old_acme); 851 | self.deregister_gap(top_base, bin_of_size(top_size)); 852 | self.register_gap(top_base, new_acme); 853 | } else if is_chunk_size(old_acme, new_acme) { 854 | self.register_gap(old_acme, new_acme); 855 | Tag::set_above_free(old_acme.sub(TAG_SIZE).cast()); 856 | } else { 857 | ret_acme = old_acme; 858 | } 859 | 860 | // extend the bottom chunk if it's free, else add free chunk below if possible 861 | if is_gap_above_heap_base(old_base) { 862 | let bottom_base = old_base.add(TAG_SIZE); 863 | let bottom_size = gap_base_to_size(bottom_base).read(); 864 | self.deregister_gap(bottom_base, bin_of_size(bottom_size)); 865 | self.register_gap(new_chunk_base, bottom_base.add(bottom_size)); 866 | Tag::write(new_base.cast(), null_mut(), true); 867 | } else if is_chunk_size(new_base, old_base) { 868 | self.register_gap(new_base.add(TAG_SIZE), old_base.add(TAG_SIZE)); 869 | Tag::write(new_base.cast(), null_mut(), true); 870 | } else { 871 | ret_base = old_base; 872 | } 873 | 874 | let ret_heap = Span::new(ret_base, ret_acme); 875 | 876 | #[cfg(feature = "counters")] 877 | self.counters.account_extend(old_heap.size(), ret_heap.size()); 878 | 879 | ret_heap 880 | } 881 | 882 | /// Reduce the extent of a heap. 883 | /// The new extent must encompass all current allocations. See below. 884 | /// 885 | /// The resultant heap is always equal to or slightly smaller than `req_heap`. 886 | /// 887 | /// Truncating to an empty [`Span`] is valid for heaps where no memory is 888 | /// currently allocated within it. 889 | /// 890 | /// In all cases where the return value is empty, the heap no longer exists. 891 | /// You may do what you like with the heap memory. The empty span should not be 892 | /// used as input to [`truncate`](Talc::truncate), [`extend`](Talc::extend), 893 | /// or [`get_allocated_span`](Talc::get_allocated_span). 894 | /// 895 | /// # Safety 896 | /// `old_heap` must be the return value of a heap-manipulation function 897 | /// of this allocator instance. 898 | /// 899 | /// # Panics: 900 | /// This function panics if: 901 | /// - `old_heap` doesn't contain `req_heap` 902 | /// - `req_heap` doesn't contain all the allocated memory in `old_heap` 903 | /// - the heap metadata is not yet allocated, see [`claim`](Talc::claim) 904 | /// 905 | /// # Usage 906 | /// 907 | /// A recommended pattern for satisfying these criteria is: 908 | /// ```rust 909 | /// # use talc::*; 910 | /// # let mut talc = Talc::new(ErrOnOom); 911 | /// let mut heap = [0u8; 2000]; 912 | /// let old_heap = Span::from(&mut heap[300..1700]); 913 | /// let old_heap = unsafe { talc.claim(old_heap).unwrap() }; 914 | /// 915 | /// // note: lock a `Talck` here otherwise a race condition may occur 916 | /// // in between Talc::get_allocated_span and Talc::truncate 917 | /// 918 | /// // compute the new heap span as a truncation of the old span 919 | /// let new_heap = old_heap 920 | /// .truncate(250, 300) 921 | /// .fit_over(unsafe { talc.get_allocated_span(old_heap) }); 922 | /// 923 | /// // truncate the heap 924 | /// unsafe { talc.truncate(old_heap, new_heap); } 925 | /// ``` 926 | pub unsafe fn truncate(&mut self, old_heap: Span, req_heap: Span) -> Span { 927 | assert!(!self.bins.is_null(), "no heaps have been successfully established!"); 928 | 929 | self.scan_for_errors(); 930 | 931 | let new_heap = req_heap.word_align_inward(); 932 | 933 | // check that the new_heap is valid 934 | assert!(old_heap.contains_span(new_heap), "the old_heap must contain new_heap!"); 935 | assert!( 936 | new_heap.contains_span(unsafe { self.get_allocated_span(old_heap) }), 937 | "new_heap must contain all the heap's allocated memory! see `get_allocated_span`" 938 | ); 939 | 940 | let (old_base, old_acme) = old_heap.get_base_acme().unwrap(); 941 | let old_chunk_base = old_base.add(TAG_SIZE); 942 | 943 | // if the entire heap is decimated, just return an empty span 944 | if new_heap.size() < MIN_HEAP_SIZE { 945 | self.deregister_gap( 946 | old_chunk_base, 947 | bin_of_size(old_acme as usize - old_chunk_base as usize), 948 | ); 949 | 950 | #[cfg(feature = "counters")] 951 | self.counters.account_truncate(old_heap.size(), 0); 952 | 953 | return Span::empty(); 954 | } 955 | 956 | let (new_base, new_acme) = new_heap.get_base_acme().unwrap(); 957 | let new_chunk_base = new_base.add(TAG_SIZE); 958 | let mut ret_base = new_base; 959 | let mut ret_acme = new_acme; 960 | 961 | // trim the top 962 | if new_acme < old_acme { 963 | let (top_base, top_size) = gap_acme_to_base_size(old_acme); 964 | self.deregister_gap(top_base, bin_of_size(top_size)); 965 | 966 | if is_chunk_size(top_base, new_acme) { 967 | self.register_gap(top_base, new_acme); 968 | } else { 969 | ret_acme = top_base; 970 | Tag::clear_above_free(top_base.sub(TAG_SIZE).cast()); 971 | } 972 | } 973 | 974 | // no need to check if the entire heap vanished; 975 | // we eliminated this possibility earlier 976 | 977 | // trim the bottom 978 | if old_base < new_base { 979 | debug_assert!(is_gap_above_heap_base(old_base)); 980 | 981 | let (bottom_acme, bottom_size) = gap_base_to_acme_size(old_chunk_base); 982 | self.deregister_gap(old_chunk_base, bin_of_size(bottom_size)); 983 | 984 | if is_chunk_size(new_chunk_base, bottom_acme) { 985 | self.register_gap(new_chunk_base, bottom_acme); 986 | Tag::write(new_base.cast(), null_mut(), true); 987 | } else { 988 | ret_base = bottom_acme.sub(TAG_SIZE); 989 | Tag::write(ret_base.cast(), null_mut(), false); 990 | } 991 | } 992 | 993 | let ret_heap = Span::new(ret_base, ret_acme); 994 | 995 | #[cfg(feature = "counters")] 996 | self.counters.account_truncate(old_heap.size(), ret_heap.size()); 997 | 998 | ret_heap 999 | } 1000 | 1001 | #[cfg(not(debug_assertions))] 1002 | fn scan_for_errors(&self) {} 1003 | 1004 | #[cfg(debug_assertions)] 1005 | /// Debugging function for checking various assumptions. 1006 | fn scan_for_errors(&self) { 1007 | #[cfg(any(test, feature = "fuzzing"))] 1008 | let mut vec = std::vec::Vec::::new(); 1009 | 1010 | if !self.bins.is_null() { 1011 | for b in 0..BIN_COUNT { 1012 | let mut any = false; 1013 | unsafe { 1014 | for node in LlistNode::iter_mut(*self.get_bin_ptr(b)) { 1015 | any = true; 1016 | if b < WORD_BITS { 1017 | assert!(self.availability_low & 1 << b != 0); 1018 | } else { 1019 | assert!(self.availability_high & 1 << (b - WORD_BITS) != 0); 1020 | } 1021 | 1022 | let base = gap_node_to_base(node); 1023 | let (acme, size) = gap_base_to_acme_size(base); 1024 | let low_size = gap_acme_to_size(acme).read(); 1025 | assert!(low_size == size); 1026 | 1027 | let lower_tag = base.sub(TAG_SIZE).cast::().read(); 1028 | assert!(lower_tag.is_allocated()); 1029 | assert!(lower_tag.is_above_free()); 1030 | 1031 | #[cfg(any(test, feature = "fuzzing"))] 1032 | { 1033 | let span = Span::new(base, acme); 1034 | //dbg!(span); 1035 | for other in &vec { 1036 | assert!(!span.overlaps(*other), "{} intersects {}", span, other); 1037 | } 1038 | vec.push(span); 1039 | } 1040 | } 1041 | } 1042 | 1043 | if !any { 1044 | if b < WORD_BITS { 1045 | assert!(self.availability_low & 1 << b == 0); 1046 | } else { 1047 | assert!(self.availability_high & 1 << (b - WORD_BITS) == 0); 1048 | } 1049 | } 1050 | } 1051 | } else { 1052 | assert!(self.availability_low == 0); 1053 | assert!(self.availability_high == 0); 1054 | } 1055 | } 1056 | } 1057 | 1058 | #[cfg(test)] 1059 | mod tests { 1060 | use super::*; 1061 | 1062 | #[test] 1063 | fn alignment_assumptions_hold() { 1064 | // claim assumes this 1065 | assert!(ALIGN == std::mem::align_of::() && ALIGN == std::mem::size_of::()); 1066 | } 1067 | 1068 | #[test] 1069 | fn alloc_dealloc_test() { 1070 | const ARENA_SIZE: usize = 10000000; 1071 | 1072 | let arena = Box::leak(vec![0u8; ARENA_SIZE].into_boxed_slice()) as *mut [_]; 1073 | 1074 | let mut talc = Talc::new(crate::ErrOnOom); 1075 | 1076 | unsafe { 1077 | talc.claim(arena.as_mut().unwrap().into()).unwrap(); 1078 | } 1079 | 1080 | let layout = Layout::from_size_align(1243, 8).unwrap(); 1081 | 1082 | let a = unsafe { talc.malloc(layout) }; 1083 | assert!(a.is_ok()); 1084 | unsafe { 1085 | a.unwrap().as_ptr().write_bytes(255, layout.size()); 1086 | } 1087 | 1088 | let mut x = vec![NonNull::dangling(); 100]; 1089 | 1090 | for _ in 0..1 { 1091 | for i in 0..100 { 1092 | let allocation = unsafe { talc.malloc(layout) }; 1093 | assert!(allocation.is_ok()); 1094 | unsafe { 1095 | allocation.unwrap().as_ptr().write_bytes(0xab, layout.size()); 1096 | } 1097 | x[i] = allocation.unwrap(); 1098 | } 1099 | 1100 | for i in 0..50 { 1101 | unsafe { 1102 | talc.free(x[i], layout); 1103 | } 1104 | } 1105 | for i in (50..100).rev() { 1106 | unsafe { 1107 | talc.free(x[i], layout); 1108 | } 1109 | } 1110 | } 1111 | 1112 | unsafe { 1113 | talc.free(a.unwrap(), layout); 1114 | } 1115 | 1116 | unsafe { 1117 | drop(Box::from_raw(arena)); 1118 | } 1119 | } 1120 | 1121 | #[test] 1122 | fn claim_truncate_extend_test() { 1123 | // not big enough to fit the metadata 1124 | let mut tiny_heap = [0u8; BIN_COUNT * WORD_SIZE / 2]; 1125 | let tiny_heap_span: Span = Span::from(&mut tiny_heap); 1126 | 1127 | // big enough with plenty of extra 1128 | let big_heap = Box::leak(vec![0u8; BIN_COUNT * WORD_SIZE + 100000].into_boxed_slice()); 1129 | let big_heap_span = Span::from(big_heap.as_mut()); 1130 | 1131 | let mut talc = Talc::new(crate::ErrOnOom); 1132 | 1133 | unsafe { 1134 | talc.claim(tiny_heap_span).unwrap_err(); 1135 | } 1136 | 1137 | assert!(talc.bins.is_null()); 1138 | assert!(talc.availability_low == 0 && talc.availability_high == 0); 1139 | 1140 | let alloc_big_heap = unsafe { talc.claim(big_heap_span).unwrap() }; 1141 | 1142 | assert!(!talc.bins.is_null()); 1143 | 1144 | let alloc_big_heap = unsafe { 1145 | talc.truncate( 1146 | alloc_big_heap, 1147 | alloc_big_heap.truncate(500, 500).fit_over(talc.get_allocated_span(alloc_big_heap)), 1148 | ) 1149 | }; 1150 | 1151 | let _alloc_tiny_heap = unsafe { talc.claim(tiny_heap_span).unwrap() }; 1152 | 1153 | let allocation = unsafe { 1154 | let allocation = talc.malloc(Layout::new::()).unwrap(); 1155 | allocation.as_ptr().write_bytes(0, Layout::new::().size()); 1156 | allocation 1157 | }; 1158 | 1159 | let alloc_big_heap = unsafe { 1160 | talc.truncate( 1161 | alloc_big_heap, 1162 | alloc_big_heap 1163 | .truncate(100000, 100000) 1164 | .fit_over(talc.get_allocated_span(alloc_big_heap)), 1165 | ) 1166 | }; 1167 | 1168 | unsafe { 1169 | talc.extend( 1170 | alloc_big_heap, 1171 | alloc_big_heap.extend(10000, 10000).fit_within(big_heap_span), 1172 | ); 1173 | } 1174 | 1175 | unsafe { 1176 | talc.free(allocation, Layout::new::()); 1177 | } 1178 | 1179 | unsafe { 1180 | drop(Box::from_raw(big_heap)); 1181 | } 1182 | } 1183 | } 1184 | -------------------------------------------------------------------------------- /talc/src/talc/counters.rs: -------------------------------------------------------------------------------- 1 | //! Track allocation counters for Talc. 2 | 3 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] 4 | pub struct Counters { 5 | /// Number of active allocations. 6 | pub allocation_count: usize, 7 | /// Total number of allocations. 8 | pub total_allocation_count: u64, 9 | 10 | /// Sum of active allocations' layouts' size. 11 | pub allocated_bytes: usize, 12 | /// Sum of all allocations' layouts' maximum size. 13 | /// 14 | /// In-place reallocations's unchanged bytes are not recounted. 15 | pub total_allocated_bytes: u64, 16 | 17 | /// Number of bytes available for allocation. 18 | pub available_bytes: usize, 19 | /// Number of holes/gaps between allocations. 20 | pub fragment_count: usize, 21 | 22 | /// Number of active established heaps. 23 | pub heap_count: usize, 24 | /// Total number of established heaps. 25 | pub total_heap_count: u64, 26 | 27 | /// Sum of bytes actively claimed. 28 | pub claimed_bytes: usize, 29 | /// Sum of bytes ever claimed. Reclaimed bytes included. 30 | pub total_claimed_bytes: u64, 31 | } 32 | 33 | impl Counters { 34 | pub const fn new() -> Self { 35 | Self { 36 | allocation_count: 0, 37 | total_allocation_count: 0, 38 | allocated_bytes: 0, 39 | total_allocated_bytes: 0, 40 | available_bytes: 0, 41 | fragment_count: 0, 42 | heap_count: 0, 43 | total_heap_count: 0, 44 | claimed_bytes: 0, 45 | total_claimed_bytes: 0, 46 | } 47 | } 48 | 49 | /// Returns the number of bytes unavailable due to padding/metadata/etc. 50 | pub const fn overhead_bytes(&self) -> usize { 51 | self.claimed_bytes - self.available_bytes - self.allocated_bytes 52 | } 53 | 54 | /// Returns the total number of allocated bytes freed. 55 | pub const fn total_freed_bytes(&self) -> u64 { 56 | self.total_allocated_bytes - self.allocated_bytes as u64 57 | } 58 | 59 | /// Returns the total number of claimed bytes released. 60 | pub const fn total_released_bytes(&self) -> u64 { 61 | self.total_claimed_bytes - self.claimed_bytes as u64 62 | } 63 | 64 | pub(crate) fn account_register_gap(&mut self, size: usize) { 65 | self.available_bytes += size; 66 | self.fragment_count += 1; 67 | } 68 | pub(crate) fn account_deregister_gap(&mut self, size: usize) { 69 | self.available_bytes -= size; 70 | self.fragment_count -= 1; 71 | } 72 | 73 | pub(crate) fn account_alloc(&mut self, alloc_size: usize) { 74 | self.allocation_count += 1; 75 | self.allocated_bytes += alloc_size; 76 | 77 | self.total_allocation_count += 1; 78 | self.total_allocated_bytes += alloc_size as u64; 79 | } 80 | 81 | pub(crate) fn account_dealloc(&mut self, alloc_size: usize) { 82 | self.allocation_count -= 1; 83 | self.allocated_bytes -= alloc_size; 84 | } 85 | 86 | pub(crate) fn account_grow_in_place(&mut self, old_alloc_size: usize, new_alloc_size: usize) { 87 | self.allocated_bytes += new_alloc_size - old_alloc_size; 88 | self.total_allocated_bytes += (new_alloc_size - old_alloc_size) as u64; 89 | } 90 | 91 | pub(crate) fn account_shrink_in_place(&mut self, old_alloc_size: usize, new_alloc_size: usize) { 92 | self.allocated_bytes -= old_alloc_size - new_alloc_size; 93 | self.total_allocated_bytes -= (old_alloc_size - new_alloc_size) as u64; 94 | } 95 | 96 | pub(crate) fn account_claim(&mut self, claimed_size: usize) { 97 | self.heap_count += 1; 98 | self.claimed_bytes += claimed_size; 99 | 100 | self.total_heap_count += 1; 101 | self.total_claimed_bytes += claimed_size as u64; 102 | } 103 | 104 | pub(crate) fn account_extend(&mut self, old_claimed_size: usize, new_claimed_size: usize) { 105 | self.claimed_bytes += new_claimed_size - old_claimed_size; 106 | self.total_claimed_bytes += (new_claimed_size - old_claimed_size) as u64; 107 | } 108 | 109 | pub(crate) fn account_truncate(&mut self, old_claimed_size: usize, new_claimed_size: usize) { 110 | if old_claimed_size != 0 && new_claimed_size == 0 { 111 | self.heap_count -= 1; 112 | } 113 | 114 | self.claimed_bytes -= old_claimed_size - new_claimed_size; 115 | } 116 | } 117 | 118 | impl core::fmt::Display for Counters { 119 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 120 | f.write_fmt(format_args!( 121 | r#"Stat | Running Total | Accumulative Total 122 | ---------------------|---------------------|-------------------- 123 | # of Allocations | {:>19} | {:>19} 124 | # of Allocated Bytes | {:>19} | {:>19} 125 | # of Available Bytes | {:>19} | N/A 126 | # of Claimed Bytes | {:>19} | {:>19} 127 | # of Heaps | {:>19} | {:>19} 128 | # of Fragments | {:>19} | N/A"#, 129 | self.allocation_count, 130 | self.total_allocation_count, 131 | self.allocated_bytes, 132 | self.total_allocated_bytes, 133 | self.available_bytes, 134 | self.claimed_bytes, 135 | self.total_claimed_bytes, 136 | self.heap_count, 137 | self.total_heap_count, 138 | self.fragment_count 139 | )) 140 | } 141 | } 142 | 143 | impl super::Talc { 144 | pub fn get_counters(&self) -> &Counters { 145 | &self.counters 146 | } 147 | } 148 | 149 | #[cfg(test)] 150 | mod tests { 151 | use core::alloc::Layout; 152 | 153 | use ptr_utils::{WORD_BITS, WORD_SIZE}; 154 | 155 | use crate::{talc::TAG_SIZE, *}; 156 | 157 | #[test] 158 | fn test_claim_alloc_free_truncate() { 159 | let mut arena = [0u8; 1000000]; 160 | 161 | let mut talc = Talc::new(ErrOnOom); 162 | 163 | let low = 99; 164 | let high = 10001; 165 | let heap1 = unsafe { talc.claim(arena.get_mut(low..high).unwrap().into()).unwrap() }; 166 | 167 | let pre_alloc_claimed_bytes = talc.get_counters().claimed_bytes; 168 | assert!(talc.get_counters().claimed_bytes == heap1.size()); 169 | assert!(talc.get_counters().claimed_bytes <= high - low); 170 | assert!(talc.get_counters().claimed_bytes >= high - low - 16); 171 | assert!(talc.get_counters().claimed_bytes == talc.get_counters().total_claimed_bytes as _); 172 | 173 | let pre_alloc_avl_bytes = talc.get_counters().available_bytes; 174 | dbg!(pre_alloc_avl_bytes); 175 | assert!(talc.get_counters().available_bytes < high - low - WORD_SIZE * WORD_BITS * 2); 176 | assert!(talc.get_counters().available_bytes >= high - low - WORD_SIZE * WORD_BITS * 2 - 64); 177 | 178 | assert!(talc.get_counters().allocated_bytes == 0); 179 | assert!(talc.get_counters().total_allocated_bytes == 0); 180 | 181 | assert!(talc.get_counters().allocation_count == 0); 182 | assert!(talc.get_counters().total_allocation_count == 0); 183 | assert!(talc.get_counters().fragment_count == 1); 184 | assert!(talc.get_counters().overhead_bytes() >= TAG_SIZE + WORD_SIZE * WORD_BITS * 2); 185 | assert!(talc.get_counters().overhead_bytes() <= TAG_SIZE + WORD_SIZE * WORD_BITS * 2 + 64); 186 | 187 | let alloc_layout = Layout::new::<[u128; 3]>(); 188 | let alloc = unsafe { talc.malloc(alloc_layout).unwrap() }; 189 | 190 | assert!(talc.get_counters().claimed_bytes == pre_alloc_claimed_bytes); 191 | assert!(talc.get_counters().available_bytes < pre_alloc_avl_bytes - alloc_layout.size()); 192 | assert!(talc.get_counters().available_bytes < pre_alloc_avl_bytes - alloc_layout.size()); 193 | assert!(talc.get_counters().allocated_bytes == alloc_layout.size()); 194 | assert!(talc.get_counters().total_allocated_bytes == alloc_layout.size() as _); 195 | assert!(talc.get_counters().allocation_count == 1); 196 | assert!(talc.get_counters().total_allocation_count == 1); 197 | dbg!(talc.get_counters().fragment_count); 198 | assert!(matches!(talc.get_counters().fragment_count, 1..=2)); 199 | 200 | assert!(talc.get_counters().overhead_bytes() >= 2 * TAG_SIZE); 201 | 202 | unsafe { 203 | talc.free(alloc, alloc_layout); 204 | } 205 | 206 | assert!(talc.get_counters().claimed_bytes == pre_alloc_claimed_bytes); 207 | assert!(talc.get_counters().total_claimed_bytes == pre_alloc_claimed_bytes as _); 208 | assert!(talc.get_counters().available_bytes == pre_alloc_avl_bytes); 209 | assert!(talc.get_counters().allocated_bytes == 0); 210 | assert!(talc.get_counters().total_allocated_bytes == alloc_layout.size() as _); 211 | assert!(talc.get_counters().allocation_count == 0); 212 | assert!(talc.get_counters().total_allocation_count == 1); 213 | assert!(talc.get_counters().fragment_count == 1); 214 | 215 | let heap1 = unsafe { talc.truncate(heap1, talc.get_allocated_span(heap1)) }; 216 | 217 | assert!(heap1.size() <= TAG_SIZE + WORD_SIZE * WORD_BITS * 2 + 64); 218 | 219 | assert!(talc.get_counters().claimed_bytes == heap1.size()); 220 | assert!(talc.get_counters().overhead_bytes() == talc.get_counters().claimed_bytes); 221 | assert!(talc.get_counters().total_claimed_bytes == pre_alloc_claimed_bytes as _); 222 | assert!(talc.get_counters().available_bytes == 0); 223 | assert!(talc.get_counters().allocated_bytes == 0); 224 | assert!(talc.get_counters().total_allocated_bytes == alloc_layout.size() as _); 225 | assert!(talc.get_counters().allocation_count == 0); 226 | assert!(talc.get_counters().total_allocation_count == 1); 227 | assert!(talc.get_counters().fragment_count == 0); 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /talc/src/talc/llist.rs: -------------------------------------------------------------------------------- 1 | use core::ptr::NonNull; 2 | 3 | /// Describes a linked list node. 4 | /// 5 | /// # Safety: 6 | /// `LlistNode`s are inherently unsafe due to the referencial dependency between nodes. This requires 7 | /// that `LlistNode`s are never moved manually, otherwise using the list becomes memory 8 | /// unsafe and may lead to undefined behaviour. 9 | /// 10 | /// This data structure is not thread-safe, use mutexes/locks to mutually exclude data access. 11 | #[derive(Debug)] 12 | #[repr(C)] 13 | pub struct LlistNode { 14 | pub next: Option>, 15 | pub next_of_prev: *mut Option>, 16 | } 17 | 18 | impl LlistNode { 19 | #[inline] 20 | pub fn next_ptr(ptr: *mut Self) -> *mut Option> { 21 | ptr.cast() /* .cast::().wrapping_add(core::mem::offset_of!(LlistNode, next)) */ 22 | } 23 | 24 | /// Create a new node as a member of an existing linked list at `node`. 25 | /// 26 | /// Warning: This will not call `remove` on `node`, regardless of initialization. 27 | /// It is your responsibility to make sure `node` gets `remove`d if necessary. 28 | /// Failing to do so when is not undefined behaviour or memory unsafe, but 29 | /// may cause unexpected linkages. 30 | /// 31 | /// # Safety 32 | /// * `node` must be `ptr::write`-able. 33 | /// * `next_of_prev` must be dereferencable and valid. 34 | pub unsafe fn insert( 35 | node: *mut Self, 36 | next_of_prev: *mut Option>, 37 | next: Option>, 38 | ) { 39 | debug_assert!(!node.is_null()); 40 | debug_assert!(!next_of_prev.is_null()); 41 | 42 | node.write(Self { next_of_prev, next }); 43 | 44 | *next_of_prev = Some(NonNull::new_unchecked(node)); 45 | 46 | if let Some(next) = next { 47 | (*next.as_ptr()).next_of_prev = Self::next_ptr(node); 48 | } 49 | } 50 | 51 | /// Remove `node` from it's linked list. 52 | /// 53 | /// Note that this does not modify `node`; it should be considered invalid. 54 | /// 55 | /// # Safety 56 | /// * `self` must be dereferencable and valid. 57 | pub unsafe fn remove(node: *mut Self) { 58 | debug_assert!(!node.is_null()); 59 | let LlistNode { next, next_of_prev } = node.read(); 60 | 61 | debug_assert!(!next_of_prev.is_null()); 62 | *next_of_prev = next; 63 | 64 | if let Some(next) = next { 65 | (*next.as_ptr()).next_of_prev = next_of_prev; 66 | } 67 | } 68 | 69 | /// Creates an iterator over the circular linked list, exclusive of 70 | /// the sentinel. 71 | /// # Safety 72 | /// `start`'s linked list must remain in a valid state during iteration. 73 | /// Modifying `LlistNode`s already returned by the iterator is okay. 74 | pub unsafe fn iter_mut(first: Option>) -> IterMut { 75 | IterMut::new(first) 76 | } 77 | } 78 | 79 | /// An iterator over the circular linked list `LlistNode`s, excluding the 'head'. 80 | /// 81 | /// This `struct` is created by `LlistNode::iter_mut`. See its documentation for more. 82 | #[derive(Debug, Clone, Copy)] 83 | #[must_use = "iterators are lazy and do nothing unless consumed"] 84 | pub struct IterMut(Option>); 85 | 86 | impl IterMut { 87 | /// Create a new iterator over the linked list from `first`. 88 | pub unsafe fn new(first: Option>) -> Self { 89 | Self(first) 90 | } 91 | } 92 | 93 | impl Iterator for IterMut { 94 | type Item = NonNull; 95 | 96 | fn next(&mut self) -> Option { 97 | let current = self.0?; 98 | self.0 = unsafe { (*current.as_ptr()).next }; 99 | Some(current) 100 | } 101 | } 102 | 103 | #[cfg(test)] 104 | mod tests { 105 | use std::ptr::null_mut; 106 | 107 | use super::*; 108 | 109 | #[test] 110 | fn test_llist() { 111 | unsafe { 112 | let x = Box::into_raw(Box::new(LlistNode { next: None, next_of_prev: null_mut() })); 113 | let y = Box::into_raw(Box::new(LlistNode { next: None, next_of_prev: null_mut() })); 114 | let z = Box::into_raw(Box::new(LlistNode { next: None, next_of_prev: null_mut() })); 115 | 116 | LlistNode::insert(y, LlistNode::next_ptr(x), None); 117 | LlistNode::insert(z, LlistNode::next_ptr(x), Some(NonNull::new(y).unwrap())); 118 | 119 | let mut iter = LlistNode::iter_mut(Some(NonNull::new(x)).unwrap()); 120 | assert!(iter.next().is_some_and(|n| n.as_ptr() == x)); 121 | assert!(iter.next().is_some_and(|n| n.as_ptr() == z)); 122 | assert!(iter.next().is_some_and(|n| n.as_ptr() == y)); 123 | assert!(iter.next().is_none()); 124 | 125 | let mut iter = LlistNode::iter_mut(Some(NonNull::new(y).unwrap())); 126 | assert!(iter.next().is_some_and(|n| n.as_ptr() == y)); 127 | assert!(iter.next().is_none()); 128 | 129 | LlistNode::remove(z); 130 | 131 | let mut iter = LlistNode::iter_mut(Some(NonNull::new(x).unwrap())); 132 | assert!(iter.next().is_some_and(|n| n.as_ptr() == x)); 133 | assert!(iter.next().is_some_and(|n| n.as_ptr() == y)); 134 | assert!(iter.next().is_none()); 135 | 136 | LlistNode::insert(z, LlistNode::next_ptr(x), Some(NonNull::new(y).unwrap())); 137 | 138 | let mut iter = LlistNode::iter_mut(Some(NonNull::new(x).unwrap())); 139 | assert!(iter.next().is_some_and(|n| n.as_ptr() == x)); 140 | assert!(iter.next().is_some_and(|n| n.as_ptr() == z)); 141 | assert!(iter.next().is_some_and(|n| n.as_ptr() == y)); 142 | assert!(iter.next().is_none()); 143 | 144 | LlistNode::remove(z); 145 | LlistNode::remove(y); 146 | 147 | let mut iter = LlistNode::iter_mut(Some(NonNull::new(x).unwrap())); 148 | assert!(iter.next().is_some_and(|n| n.as_ptr() == x)); 149 | assert!(iter.next().is_none()); 150 | 151 | drop(Box::from_raw(x)); 152 | drop(Box::from_raw(y)); 153 | drop(Box::from_raw(z)); 154 | } 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /talc/src/talc/tag.rs: -------------------------------------------------------------------------------- 1 | //! A `Tag` is a size with flags in the least significant 2 | //! bits and most significant bit for allocated chunks. 3 | 4 | // const UNUSED_BITS: usize = 2; //crate::ALIGN.ilog2(); 5 | // on 64 bit machines we have unused 3 bits to work with but 6 | // let's keep it more portable for now. 7 | 8 | use crate::ptr_utils::ALIGN; 9 | 10 | /// Tag for allocated chunk metadata. 11 | #[derive(Clone, Copy, PartialEq, Eq)] 12 | #[repr(transparent)] 13 | pub struct Tag(pub *mut u8); 14 | 15 | impl core::fmt::Debug for Tag { 16 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 17 | f.debug_struct("Tag") 18 | .field("is_allocated", &self.is_allocated()) 19 | .field("is_above_free", &self.is_above_free()) 20 | .field("base_ptr", &format_args!("{:p}", self.chunk_base())) 21 | .finish() 22 | } 23 | } 24 | 25 | impl Tag { 26 | pub const ALLOCATED_FLAG: usize = 1 << 0; // pointers are always aligned to 4 bytes at least 27 | pub const IS_ABOVE_FREE_FLAG: usize = 1 << 1; // pointers are always aligned to 4 bytes at least 28 | 29 | const BASE: usize = !(Self::ALLOCATED_FLAG | Self::IS_ABOVE_FREE_FLAG); 30 | 31 | pub unsafe fn write(chunk_tag: *mut Tag, chunk_base: *mut u8, is_above_free: bool) { 32 | debug_assert!(chunk_base as usize & !Self::BASE == 0); 33 | 34 | chunk_tag.write(if is_above_free { 35 | Self(chunk_base.wrapping_add(Self::IS_ABOVE_FREE_FLAG | Self::ALLOCATED_FLAG)) 36 | } else { 37 | Self(chunk_base.wrapping_add(Self::ALLOCATED_FLAG)) 38 | }) 39 | } 40 | 41 | pub fn chunk_base(self) -> *mut u8 { 42 | self.0.wrapping_sub(self.0 as usize % ALIGN) 43 | } 44 | 45 | pub fn is_above_free(self) -> bool { 46 | self.0 as usize & Self::IS_ABOVE_FREE_FLAG != 0 47 | } 48 | 49 | pub fn is_allocated(self) -> bool { 50 | self.0 as usize & Self::ALLOCATED_FLAG != 0 51 | } 52 | 53 | pub unsafe fn set_above_free(ptr: *mut Self) { 54 | let mut tag = ptr.read(); 55 | debug_assert!(!tag.is_above_free()); 56 | tag = Self(tag.0.wrapping_add(Self::IS_ABOVE_FREE_FLAG)); 57 | debug_assert!(tag.is_above_free()); 58 | ptr.write(tag); 59 | } 60 | 61 | pub unsafe fn clear_above_free(ptr: *mut Self) { 62 | let mut tag = ptr.read(); 63 | debug_assert!(tag.is_above_free()); 64 | tag = Self(tag.0.wrapping_sub(Self::IS_ABOVE_FREE_FLAG)); 65 | debug_assert!(!tag.is_above_free()); 66 | ptr.write(tag); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /talc/src/talck.rs: -------------------------------------------------------------------------------- 1 | //! Home of Talck, a mutex-locked wrapper of Talc. 2 | 3 | use crate::{talc::Talc, OomHandler}; 4 | 5 | use core::{ 6 | alloc::{GlobalAlloc, Layout}, 7 | cmp::Ordering, 8 | ptr::{null_mut, NonNull}, 9 | }; 10 | 11 | #[cfg(feature = "allocator")] 12 | use core::alloc::{AllocError, Allocator}; 13 | 14 | #[cfg(all(feature = "allocator-api2", not(feature = "allocator")))] 15 | use allocator_api2::alloc::{AllocError, Allocator}; 16 | 17 | #[cfg(any(feature = "allocator", feature = "allocator-api2"))] 18 | pub(crate) fn is_aligned_to(ptr: *mut u8, align: usize) -> bool { 19 | (ptr as usize).trailing_zeros() >= align.trailing_zeros() 20 | } 21 | 22 | const RELEASE_LOCK_ON_REALLOC_LIMIT: usize = 0x10000; 23 | 24 | /// Talc lock, contains a mutex-locked [`Talc`]. 25 | /// 26 | /// # Example 27 | /// ```rust 28 | /// # use talc::*; 29 | /// let talc = Talc::new(ErrOnOom); 30 | /// let talck = talc.lock::>(); 31 | /// ``` 32 | #[derive(Debug)] 33 | pub struct Talck { 34 | mutex: lock_api::Mutex>, 35 | } 36 | 37 | impl Talck { 38 | /// Create a new `Talck`. 39 | pub const fn new(talc: Talc) -> Self { 40 | Self { mutex: lock_api::Mutex::new(talc) } 41 | } 42 | 43 | /// Lock the mutex and access the inner `Talc`. 44 | pub fn lock(&self) -> lock_api::MutexGuard> { 45 | self.mutex.lock() 46 | } 47 | 48 | /// Try to lock the mutex and access the inner `Talc`. 49 | pub fn try_lock(&self) -> Option>> { 50 | self.mutex.try_lock() 51 | } 52 | 53 | /// Retrieve the inner `Talc`. 54 | pub fn into_inner(self) -> Talc { 55 | self.mutex.into_inner() 56 | } 57 | } 58 | 59 | unsafe impl GlobalAlloc for Talck { 60 | unsafe fn alloc(&self, layout: Layout) -> *mut u8 { 61 | self.lock().malloc(layout).map_or(null_mut(), |nn| nn.as_ptr()) 62 | } 63 | 64 | unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { 65 | self.lock().free(NonNull::new_unchecked(ptr), layout) 66 | } 67 | 68 | unsafe fn realloc(&self, ptr: *mut u8, old_layout: Layout, new_size: usize) -> *mut u8 { 69 | let nn_ptr = NonNull::new_unchecked(ptr); 70 | 71 | match new_size.cmp(&old_layout.size()) { 72 | Ordering::Greater => { 73 | // first try to grow in-place before manually re-allocating 74 | 75 | if let Ok(nn) = self.lock().grow_in_place(nn_ptr, old_layout, new_size) { 76 | return nn.as_ptr(); 77 | } 78 | 79 | // grow in-place failed, reallocate manually 80 | 81 | let new_layout = Layout::from_size_align_unchecked(new_size, old_layout.align()); 82 | 83 | let mut lock = self.lock(); 84 | let allocation = match lock.malloc(new_layout) { 85 | Ok(ptr) => ptr, 86 | Err(_) => return null_mut(), 87 | }; 88 | 89 | if old_layout.size() > RELEASE_LOCK_ON_REALLOC_LIMIT { 90 | drop(lock); 91 | allocation.as_ptr().copy_from_nonoverlapping(ptr, old_layout.size()); 92 | lock = self.lock(); 93 | } else { 94 | allocation.as_ptr().copy_from_nonoverlapping(ptr, old_layout.size()); 95 | } 96 | 97 | lock.free(nn_ptr, old_layout); 98 | allocation.as_ptr() 99 | } 100 | 101 | Ordering::Less => { 102 | self.lock().shrink(NonNull::new_unchecked(ptr), old_layout, new_size); 103 | ptr 104 | } 105 | 106 | Ordering::Equal => ptr, 107 | } 108 | } 109 | } 110 | 111 | /// Convert a nonnull and length to a nonnull slice. 112 | #[cfg(any(feature = "allocator", feature = "allocator-api2"))] 113 | fn nonnull_slice_from_raw_parts(ptr: NonNull, len: usize) -> NonNull<[u8]> { 114 | unsafe { NonNull::new_unchecked(core::ptr::slice_from_raw_parts_mut(ptr.as_ptr(), len)) } 115 | } 116 | 117 | #[cfg(any(feature = "allocator", feature = "allocator-api2"))] 118 | unsafe impl Allocator for Talck { 119 | fn allocate(&self, layout: Layout) -> Result, AllocError> { 120 | if layout.size() == 0 { 121 | return Ok(nonnull_slice_from_raw_parts(NonNull::dangling(), 0)); 122 | } 123 | 124 | unsafe { self.lock().malloc(layout) } 125 | .map(|nn| nonnull_slice_from_raw_parts(nn, layout.size())) 126 | .map_err(|_| AllocError) 127 | } 128 | 129 | unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { 130 | if layout.size() != 0 { 131 | self.lock().free(ptr, layout); 132 | } 133 | } 134 | 135 | unsafe fn grow( 136 | &self, 137 | ptr: NonNull, 138 | old_layout: Layout, 139 | new_layout: Layout, 140 | ) -> Result, AllocError> { 141 | debug_assert!(new_layout.size() >= old_layout.size()); 142 | 143 | if old_layout.size() == 0 { 144 | return self.allocate(new_layout); 145 | } else if is_aligned_to(ptr.as_ptr(), new_layout.align()) { 146 | // alignment is fine, try to allocate in-place 147 | if let Ok(nn) = self.lock().grow_in_place(ptr, old_layout, new_layout.size()) { 148 | return Ok(nonnull_slice_from_raw_parts(nn, new_layout.size())); 149 | } 150 | } 151 | 152 | // can't grow in place, reallocate manually 153 | 154 | let mut lock = self.lock(); 155 | let allocation = lock.malloc(new_layout).map_err(|_| AllocError)?; 156 | 157 | if old_layout.size() > RELEASE_LOCK_ON_REALLOC_LIMIT { 158 | drop(lock); 159 | allocation.as_ptr().copy_from_nonoverlapping(ptr.as_ptr(), old_layout.size()); 160 | lock = self.lock(); 161 | } else { 162 | allocation.as_ptr().copy_from_nonoverlapping(ptr.as_ptr(), old_layout.size()); 163 | } 164 | 165 | lock.free(ptr, old_layout); 166 | 167 | Ok(nonnull_slice_from_raw_parts(allocation, new_layout.size())) 168 | } 169 | 170 | unsafe fn grow_zeroed( 171 | &self, 172 | ptr: NonNull, 173 | old_layout: Layout, 174 | new_layout: Layout, 175 | ) -> Result, AllocError> { 176 | let res = self.grow(ptr, old_layout, new_layout); 177 | 178 | if let Ok(allocation) = res { 179 | allocation 180 | .as_ptr() 181 | .cast::() 182 | .add(old_layout.size()) 183 | .write_bytes(0, new_layout.size() - old_layout.size()); 184 | } 185 | 186 | res 187 | } 188 | 189 | unsafe fn shrink( 190 | &self, 191 | ptr: NonNull, 192 | old_layout: Layout, 193 | new_layout: Layout, 194 | ) -> Result, AllocError> { 195 | debug_assert!(new_layout.size() <= old_layout.size()); 196 | 197 | if new_layout.size() == 0 { 198 | if old_layout.size() > 0 { 199 | self.lock().free(ptr, old_layout); 200 | } 201 | 202 | return Ok(nonnull_slice_from_raw_parts(NonNull::dangling(), 0)); 203 | } 204 | 205 | if !is_aligned_to(ptr.as_ptr(), new_layout.align()) { 206 | let mut lock = self.lock(); 207 | let allocation = lock.malloc(new_layout).map_err(|_| AllocError)?; 208 | 209 | if new_layout.size() > RELEASE_LOCK_ON_REALLOC_LIMIT { 210 | drop(lock); 211 | allocation.as_ptr().copy_from_nonoverlapping(ptr.as_ptr(), new_layout.size()); 212 | lock = self.lock(); 213 | } else { 214 | allocation.as_ptr().copy_from_nonoverlapping(ptr.as_ptr(), new_layout.size()); 215 | } 216 | 217 | lock.free(ptr, old_layout); 218 | return Ok(nonnull_slice_from_raw_parts(allocation, new_layout.size())); 219 | } 220 | 221 | self.lock().shrink(ptr, old_layout, new_layout.size()); 222 | 223 | Ok(nonnull_slice_from_raw_parts(ptr, new_layout.size())) 224 | } 225 | } 226 | 227 | impl Talc { 228 | /// Wrap in `Talck`, a mutex-locked wrapper struct using [`lock_api`]. 229 | /// 230 | /// This implements the [`GlobalAlloc`](core::alloc::GlobalAlloc) trait and provides 231 | /// access to the [`Allocator`](core::alloc::Allocator) API. 232 | /// 233 | /// # Examples 234 | /// ``` 235 | /// # use talc::*; 236 | /// # use core::alloc::{GlobalAlloc, Layout}; 237 | /// use spin::Mutex; 238 | /// let talc = Talc::new(ErrOnOom); 239 | /// let talck = talc.lock::>(); 240 | /// 241 | /// unsafe { 242 | /// talck.alloc(Layout::from_size_align_unchecked(32, 4)); 243 | /// } 244 | /// ``` 245 | pub const fn lock(self) -> Talck { 246 | Talck::new(self) 247 | } 248 | } 249 | 250 | #[cfg(all(target_family = "wasm"))] 251 | impl TalckWasm { 252 | /// Create a [`Talck`] instance that takes control of WASM memory management. 253 | /// 254 | /// # Safety 255 | /// The runtime environment must be single-threaded WASM. 256 | /// 257 | /// Note: calls to memory.grow during use of the allocator is allowed. 258 | pub const unsafe fn new_global() -> Self { 259 | Talc::new(crate::WasmHandler::new()).lock() 260 | } 261 | } 262 | 263 | #[cfg(all(target_family = "wasm"))] 264 | pub type TalckWasm = Talck; 265 | -------------------------------------------------------------------------------- /wasm-perf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script runs a benchmark on global alloctors for WASM. 4 | # requires wasm-pack and deno 5 | 6 | ALLOCATORS="talc talc_arena rlsf dlmalloc lol_alloc" 7 | for ALLOCATOR in ${ALLOCATORS}; do 8 | echo "${ALLOCATOR}" 9 | wasm-pack --log-level warn build wasm-perf --release --target web --features ${ALLOCATOR} 10 | 11 | if [[ $1 != "check" ]]; then 12 | cd wasm-perf 13 | deno run --allow-read bench.js 14 | cd - 15 | fi 16 | done 17 | -------------------------------------------------------------------------------- /wasm-perf/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "wasm-perf" 3 | version = "0.0.0" 4 | authors = ["Shaun Beautement "] 5 | edition = "2021" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ["cdylib", "rlib"] 10 | 11 | [dependencies] 12 | console_error_panic_hook = "0.1.7" 13 | wasm-bindgen = "0.2.84" 14 | fastrand = "2.0.0" 15 | web-sys = { version = "0.3.67", features = ["Window", "Performance"] } 16 | 17 | rlsf = { version = "0.2.1", optional = true } 18 | lol_alloc = { version = "0.4.0", optional = true } 19 | 20 | [dependencies.talc] 21 | path = "../talc" 22 | default-features = false 23 | features = ["lock_api"] 24 | optional = true 25 | 26 | [features] 27 | talc_arena = ["talc"] 28 | dlmalloc = [] # dummy feature for wasm-perf.sh 29 | 30 | -------------------------------------------------------------------------------- /wasm-perf/bench.js: -------------------------------------------------------------------------------- 1 | import init, {bench} from "./pkg/wasm_perf.js"; 2 | await init(Deno.readFile('./pkg/wasm_perf_bg.wasm')); 3 | bench(); 4 | -------------------------------------------------------------------------------- /wasm-perf/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::alloc::Layout; 2 | 3 | use wasm_bindgen::prelude::*; 4 | 5 | 6 | #[cfg(all(feature = "talc", not(feature = "talc_arena")))] 7 | #[global_allocator] 8 | static TALCK: talc::TalckWasm = unsafe { talc::TalckWasm::new_global() }; 9 | 10 | #[cfg(all(feature = "rlsf"))] 11 | #[global_allocator] 12 | static RLSF: rlsf::SmallGlobalTlsf = rlsf::SmallGlobalTlsf::new(); 13 | 14 | #[cfg(feature = "talc_arena")] 15 | #[global_allocator] 16 | static ALLOCATOR: talc::Talck = { 17 | static mut MEMORY: [std::mem::MaybeUninit; 32 * 1024 * 1024] 18 | = [std::mem::MaybeUninit::uninit(); 32 * 1024 * 1024]; 19 | let span = talc::Span::from_const_array(unsafe { std::ptr::addr_of!(MEMORY) }); 20 | talc::Talc::new(unsafe { talc::ClaimOnOom::new(span) }).lock() 21 | }; 22 | 23 | #[cfg(feature = "lol_alloc")] 24 | #[global_allocator] static ALLOC: lol_alloc::AssumeSingleThreaded = 25 | unsafe { lol_alloc::AssumeSingleThreaded::new(lol_alloc::FreeListAllocator::new()) }; 26 | 27 | 28 | #[wasm_bindgen] 29 | extern "C" { 30 | #[wasm_bindgen(js_namespace = console)] 31 | fn log(s: &str); 32 | } 33 | 34 | const ACTIONS: usize = 100000; 35 | const ITERATIONS: usize = 100; 36 | 37 | #[wasm_bindgen] 38 | pub fn bench() { 39 | console_error_panic_hook::set_once(); 40 | 41 | let timer = web_sys::window().unwrap().performance().unwrap(); 42 | 43 | // warm up 44 | random_actions(); 45 | 46 | // go! 47 | let start = timer.now(); 48 | for _ in 0..ITERATIONS { random_actions(); } 49 | let end = timer.now(); 50 | 51 | // log durations 52 | let total_ms = end - start; 53 | let average_ms = total_ms / ITERATIONS as f64; 54 | let apms = ACTIONS as f64 / average_ms / 1000.0; 55 | log(format!(" total time: {} ms", total_ms).as_str()); 56 | log(format!(" average time for {} actions: {} ms", ACTIONS, average_ms).as_str()); 57 | log(format!(" average actions/us: {:.1}", apms).as_str()); 58 | } 59 | 60 | fn random_actions() { 61 | let mut score = 0; 62 | let mut v = Vec::with_capacity(10000); 63 | 64 | while score < ACTIONS { 65 | let action = fastrand::usize(0..3); 66 | 67 | match action { 68 | 0 => { 69 | let size = fastrand::usize(100..=1000); 70 | let align = 8 << fastrand::u16(..).trailing_zeros() / 2; 71 | let layout = Layout::from_size_align(size, align).unwrap(); 72 | 73 | let allocation = unsafe { std::alloc::alloc(layout) }; 74 | 75 | if !allocation.is_null() { 76 | v.push((allocation, layout)); 77 | score += 1; 78 | } 79 | } 80 | 1 => { 81 | if !v.is_empty() { 82 | let index = fastrand::usize(0..v.len()); 83 | let (ptr, layout) = v.swap_remove(index); 84 | 85 | unsafe { 86 | std::alloc::dealloc(ptr, layout); 87 | } 88 | 89 | score += 1; 90 | } 91 | } 92 | 2 => { 93 | if !v.is_empty() { 94 | let index = fastrand::usize(0..v.len()); 95 | if let Some((ptr, layout)) = v.get_mut(index) { 96 | let new_size = fastrand::usize(100..=10000); 97 | 98 | unsafe { 99 | let realloc = std::alloc::realloc(*ptr, *layout, new_size); 100 | 101 | if !realloc.is_null() { 102 | *ptr = realloc; 103 | *layout = Layout::from_size_align_unchecked(new_size, layout.align()); 104 | score += 1; 105 | } 106 | } 107 | } 108 | } 109 | } 110 | _ => unreachable!(), 111 | } 112 | } 113 | 114 | for (ptr, layout) in v { 115 | unsafe { std::alloc::dealloc(ptr, layout); } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /wasm-size.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script calculates a weight heurisitic for WASM allocators. 4 | 5 | 6 | COMMAND="" 7 | if [[ $1 == "check" ]]; then 8 | COMMAND="check" 9 | else 10 | COMMAND="build" 11 | fi 12 | 13 | ALLOCATORS="talc talc_arena rlsf dlmalloc lol_alloc" 14 | for ALLOCATOR in ${ALLOCATORS}; do 15 | echo "${ALLOCATOR}" 16 | 17 | # turn on LTO via RUSTFLAGS 18 | RUSTFLAGS="-C lto -C embed-bitcode=yes -C linker-plugin-lto" \ 19 | cargo +nightly $COMMAND -p wasm-size --quiet --release --target wasm32-unknown-unknown --features ${ALLOCATOR} 20 | 21 | if [[ $1 != "check" ]]; then 22 | wasm-opt -Oz -o target/wasm32-unknown-unknown/release/wasm_size_opt.wasm target/wasm32-unknown-unknown/release/wasm_size.wasm 23 | echo -n " " 24 | wc -c ./target/wasm32-unknown-unknown/release/wasm_size_opt.wasm 25 | fi 26 | done 27 | -------------------------------------------------------------------------------- /wasm-size/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "wasm-size" 3 | version = "0.0.0" 4 | authors = ["Shaun Beautement "] 5 | edition = "2021" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ["cdylib"] 10 | 11 | [dependencies] 12 | lol_alloc = { version = "0.4.0", optional = true } 13 | dlmalloc = { version = "0.2.4", features = ["global"], optional = true } 14 | rlsf = { version = "0.2.1", optional = true } 15 | 16 | [dependencies.talc] 17 | path = "../talc" 18 | default-features = false 19 | features = ["lock_api"] 20 | optional = true 21 | 22 | [features] 23 | talc_arena = ["talc"] 24 | -------------------------------------------------------------------------------- /wasm-size/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | 3 | use core::alloc::Layout; 4 | 5 | extern crate alloc; 6 | 7 | #[cfg(not(target_family = "wasm"))] 8 | compile_error!("Requires targetting WASM"); 9 | 10 | #[cfg(all(not(feature = "talc"), not(feature = "dlmalloc"), not(feature = "lol_alloc"), not(feature = "rlsf")))] 11 | mod no_alloc { 12 | use core::alloc::{GlobalAlloc, Layout}; 13 | 14 | struct NoAlloc; 15 | unsafe impl GlobalAlloc for NoAlloc { 16 | unsafe fn alloc(&self, _: Layout) -> *mut u8 { core::ptr::null_mut() } 17 | unsafe fn dealloc(&self, _: *mut u8, _: Layout) { } 18 | } 19 | 20 | #[global_allocator] 21 | static NOALLOC: NoAlloc = NoAlloc; 22 | } 23 | 24 | #[cfg(all(feature = "talc", not(feature = "talc_arena")))] 25 | #[global_allocator] 26 | static TALC: talc::TalckWasm = unsafe { talc::TalckWasm::new_global() }; 27 | 28 | #[cfg(all(feature = "rlsf"))] 29 | #[global_allocator] 30 | static RLSF: rlsf::SmallGlobalTlsf = rlsf::SmallGlobalTlsf::new(); 31 | 32 | #[cfg(all(feature = "talc", feature = "talc_arena"))] 33 | #[global_allocator] 34 | static ALLOCATOR: talc::Talck = { 35 | use core::{mem::MaybeUninit, ptr::addr_of_mut}; 36 | 37 | const MEMORY_SIZE: usize = 128 * 1024 * 1024; 38 | static mut MEMORY: [MaybeUninit; MEMORY_SIZE] = [MaybeUninit::uninit(); MEMORY_SIZE]; 39 | let span = talc::Span::from_array(addr_of_mut!(MEMORY)); 40 | let oom_handler = unsafe { talc::ClaimOnOom::new(span) }; 41 | talc::Talc::new(oom_handler).lock() 42 | }; 43 | 44 | #[cfg(feature = "lol_alloc")] 45 | #[global_allocator] 46 | static LOL_ALLOC: lol_alloc::AssumeSingleThreaded = 47 | unsafe { lol_alloc::AssumeSingleThreaded::new(lol_alloc::FreeListAllocator::new()) }; 48 | 49 | #[cfg(feature = "dlmalloc")] 50 | #[global_allocator] 51 | static DLMALLOC: dlmalloc::GlobalDlmalloc = dlmalloc::GlobalDlmalloc; 52 | 53 | // this is necessary, despite rust-analyzer's protests 54 | #[panic_handler] 55 | fn panic_handler(_: &core::panic::PanicInfo) -> ! { 56 | loop { } 57 | } 58 | 59 | #[no_mangle] 60 | pub unsafe extern "C" fn alloc(size: usize) -> *mut u8 { 61 | alloc::alloc::alloc(Layout::from_size_align_unchecked(size, 8)) 62 | } 63 | 64 | #[no_mangle] 65 | pub unsafe extern "C" fn dealloc(ptr: *mut u8, size: usize) { 66 | alloc::alloc::dealloc(ptr, Layout::from_size_align_unchecked(size, 8)) 67 | } 68 | 69 | #[no_mangle] 70 | pub unsafe extern "C" fn realloc(ptr: *mut u8, old_size: usize, new_size: usize) -> *mut u8 { 71 | alloc::alloc::realloc(ptr, Layout::from_size_align_unchecked(old_size, 8), new_size) 72 | } 73 | --------------------------------------------------------------------------------