├── .gitignore ├── llmalloc-test ├── src │ └── lib.rs └── Cargo.toml ├── llmalloc-core ├── src │ ├── internals │ │ ├── huge_page │ │ │ ├── number_pages.rs │ │ │ ├── page_index.rs │ │ │ ├── foreign.rs │ │ │ └── page_sizes.rs │ │ ├── blocks │ │ │ ├── test.rs │ │ │ ├── block_ptr.rs │ │ │ ├── block_foreign.rs │ │ │ ├── atomic_block_foreign.rs │ │ │ ├── block_foreign_list.rs │ │ │ ├── block_local.rs │ │ │ └── atomic_block_foreign_list.rs │ │ ├── blocks.rs │ │ ├── atomic.rs │ │ ├── large_page │ │ │ ├── test.rs │ │ │ ├── adrift.rs │ │ │ └── local.rs │ │ ├── socket_local │ │ │ └── test.rs │ │ ├── huge_page.rs │ │ └── atomic_stack.rs │ ├── internals.rs │ ├── api.rs │ ├── lib.rs │ ├── utils.rs │ ├── api │ │ ├── platform.rs │ │ ├── domain.rs │ │ ├── thread.rs │ │ ├── configuration.rs │ │ └── socket.rs │ └── utils │ │ └── power_of_2.rs └── Cargo.toml ├── llmalloc-c ├── Cargo.toml └── src │ └── lib.rs ├── Cargo.toml ├── llmalloc ├── tests │ ├── basic.rs │ └── multi.rs ├── src │ ├── platform.rs │ ├── lib.rs │ ├── platform │ │ ├── api.rs │ │ └── linux.rs │ └── allocator.rs └── Cargo.toml ├── LICENSE-MIT ├── README.md ├── LICENSE-APACHE └── doc └── Design.md /.gitignore: -------------------------------------------------------------------------------- 1 | .cargo 2 | target 3 | Cargo.lock 4 | -------------------------------------------------------------------------------- /llmalloc-test/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! A test-support library. 2 | 3 | mod bursty; 4 | 5 | pub use bursty::{Bursty, BurstyBuilder}; 6 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/huge_page/number_pages.rs: -------------------------------------------------------------------------------- 1 | //! Utilities for HugePage. 2 | 3 | #[derive(Clone, Copy, Default)] 4 | pub(crate) struct NumberPages(pub(crate) usize); 5 | -------------------------------------------------------------------------------- /llmalloc-test/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "llmalloc-test" 3 | version = "0.1.0" 4 | authors = ["Matthieu M. "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | -------------------------------------------------------------------------------- /llmalloc-c/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "llmalloc-c" 3 | version = "0.1.0" 4 | authors = ["Matthieu M. "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | 9 | llmalloc = { path = "../llmalloc" } 10 | -------------------------------------------------------------------------------- /llmalloc-core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "llmalloc-core" 3 | version = "0.1.0" 4 | authors = ["Matthieu M. "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | 9 | [dev-dependencies] 10 | 11 | llmalloc-test = { path = "../llmalloc-test" } 12 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | 3 | members = [ 4 | "llmalloc-core", 5 | "llmalloc", 6 | "llmalloc-c", 7 | "llmalloc-test", 8 | ] 9 | 10 | [profile.bench] 11 | 12 | debug = true 13 | 14 | [profile.release] 15 | 16 | debug = true 17 | 18 | [profile.test] 19 | 20 | debug = true 21 | -------------------------------------------------------------------------------- /llmalloc/tests/basic.rs: -------------------------------------------------------------------------------- 1 | use llmalloc::LLAllocator; 2 | 3 | #[test] 4 | fn warm_up() { 5 | let allocator = LLAllocator::new(); 6 | allocator.warm_up().expect("Warmed up!"); 7 | } 8 | 9 | // FIXME: use sys crates... properly configured for system libraries. 10 | #[link(name = "numa")] 11 | extern "C" {} 12 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals.rs: -------------------------------------------------------------------------------- 1 | //! The internals of llmalloc-core. 2 | //! 3 | //! The internals provide all the heavy-lifting. 4 | 5 | pub mod blocks; 6 | pub mod huge_allocator; 7 | pub mod huge_page; 8 | pub mod large_page; 9 | pub mod socket_local; 10 | pub mod thread_local; 11 | 12 | mod atomic; 13 | mod atomic_stack; 14 | -------------------------------------------------------------------------------- /llmalloc/src/platform.rs: -------------------------------------------------------------------------------- 1 | //! Abstraction over OS differences. 2 | 3 | mod api; 4 | 5 | pub(crate) use api::{NumaNodeIndex, Configuration, Platform, ThreadLocal}; 6 | 7 | #[cfg(target_os = "linux")] 8 | mod linux; 9 | 10 | #[cfg(target_os = "linux")] 11 | pub(crate) use linux::{LLConfiguration, LLPlatform, LLThreadLocal}; 12 | -------------------------------------------------------------------------------- /llmalloc-core/src/api.rs: -------------------------------------------------------------------------------- 1 | //! The API of llmalloc-core. 2 | 3 | mod configuration; 4 | mod description; 5 | mod domain; 6 | mod platform; 7 | mod socket; 8 | mod thread; 9 | 10 | pub use configuration::{Configuration, Properties}; 11 | pub use description::{AllocationSize, Category, ClassSize, Layout, PowerOf2}; 12 | pub use domain::DomainHandle; 13 | pub use platform::Platform; 14 | pub use socket::{AtomicSocketHandle, SocketHandle}; 15 | pub use thread::ThreadHandle; 16 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/blocks/test.rs: -------------------------------------------------------------------------------- 1 | //! Test utilities; 2 | 3 | use core::ptr::NonNull; 4 | 5 | #[repr(align(256))] 6 | #[derive(Default)] 7 | pub(crate) struct AlignedArray([AlignedElement; 32]); 8 | 9 | impl AlignedArray { 10 | pub(crate) fn get(&self, index: usize) -> NonNull { 11 | let cell = &self.0[index]; 12 | NonNull::from(cell).cast() 13 | } 14 | } 15 | 16 | #[repr(align(16))] 17 | #[derive(Default)] 18 | struct AlignedElement(T); 19 | -------------------------------------------------------------------------------- /llmalloc/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "llmalloc" 3 | version = "0.1.0" 4 | authors = ["Matthieu M. "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | 9 | llmalloc-core = { path = "../llmalloc-core" } 10 | 11 | [target.'cfg(target_os = "linux")'.dependencies] 12 | 13 | libc = { version = "0.2.76", default-features = false } 14 | 15 | [dev-dependencies] 16 | 17 | criterion = "0.3" 18 | num_cpus = "1.13.0" 19 | serial_test = "0.5.0" 20 | 21 | llmalloc-test = { path = "../llmalloc-test" } 22 | 23 | [[bench]] 24 | 25 | name = "benchmark" 26 | harness = false 27 | -------------------------------------------------------------------------------- /llmalloc-core/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(not(test), no_std)] 2 | 3 | #![deny(missing_docs)] 4 | 5 | //! Building blocks for a low-latency allocator. 6 | //! 7 | //! llmalloc-core is a set of building blocks to build a custom low-latency malloc replacement with ease. It contains: 8 | //! - A platform trait, used to allocate large raw blocks of memory to be carved up. 9 | //! - A handful of user-facing types representing NUMA node and thread data, leaving it up to the user to arrange 10 | //! those as desired in memory. 11 | 12 | mod api; 13 | mod internals; 14 | mod utils; 15 | 16 | pub use api::*; 17 | -------------------------------------------------------------------------------- /llmalloc/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | #![deny(missing_docs)] 3 | 4 | //! A Low-Latency Memory Allocator library. 5 | //! 6 | //! The type `LLAllocator` provides a low-latency memory allocator, as a drop-in replacement for regular allocators. 7 | //! 8 | //! # Warning 9 | //! 10 | //! This low-latency memory allocator is not suitable for all applications. 11 | //! 12 | //! See the README.md file for the limitations and trade-offs made. 13 | 14 | mod allocator; 15 | mod platform; 16 | 17 | pub use allocator::LLAllocator; 18 | 19 | use platform::{LLConfiguration, Platform, LLPlatform, ThreadLocal, LLThreadLocal}; 20 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright 2020 matthieu-m 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/blocks.rs: -------------------------------------------------------------------------------- 1 | //! Blocks 2 | //! 3 | //! A Block represent a unit of allocation. 4 | //! 5 | //! Whilst allocated, the content of the block is purely in the hands of the user. Whilst deallocated, however, the Block 6 | //! storage is reused to store meta-data used to manage the memory. 7 | //! 8 | //! Specifically, the blocks within a LargePage are maintained into a tail-list structure. 9 | //! 10 | //! Note: Blocks are never _constructed_, instead raw memory is reinterpreted as blocks. 11 | 12 | mod atomic_block_foreign_list; 13 | mod atomic_block_foreign; 14 | mod block_foreign; 15 | mod block_foreign_list; 16 | mod block_local; 17 | mod block_ptr; 18 | 19 | pub(crate) use atomic_block_foreign_list::AtomicBlockForeignList; 20 | pub(crate) use atomic_block_foreign::AtomicBlockForeign; 21 | pub(crate) use block_foreign::BlockForeign; 22 | pub(crate) use block_foreign_list::BlockForeignList; 23 | pub(crate) use block_local::{BlockLocal, BlockLocalStack}; 24 | pub(crate) use block_ptr::BlockPtr; 25 | 26 | use super::atomic::{AtomicLength, AtomicPtr}; 27 | 28 | #[cfg(test)] 29 | mod test; 30 | 31 | #[cfg(test)] 32 | use test::AlignedArray; 33 | -------------------------------------------------------------------------------- /llmalloc-core/src/utils.rs: -------------------------------------------------------------------------------- 1 | //! A collection of utilities. 2 | 3 | use core::ptr::NonNull; 4 | 5 | mod power_of_2; 6 | 7 | pub use power_of_2::PowerOf2; 8 | 9 | /// Returns whether the pointer is sufficiently aligned for the given alignment. 10 | pub(crate) fn is_sufficiently_aligned_for(ptr: NonNull, alignment: PowerOf2) -> bool { 11 | (ptr.as_ptr() as usize) % alignment == 0 12 | } 13 | 14 | // The Prefetch Guard is used to prevent pre-fetching on a previous page from accidentally causing false-sharing with 15 | // the thread currently using the LargePage. 16 | #[repr(align(128))] 17 | #[derive(Default)] 18 | pub(crate) struct PrefetchGuard(u8); 19 | 20 | #[cfg(test)] 21 | mod tests { 22 | 23 | use crate::PowerOf2; 24 | 25 | use super::*; 26 | 27 | #[test] 28 | fn is_sufficiently_aligned_for() { 29 | fn is_aligned_for(ptr: usize, alignment: usize) -> bool { 30 | let alignment = PowerOf2::new(alignment).unwrap(); 31 | let ptr = NonNull::new(ptr as *mut u8).unwrap(); 32 | super::is_sufficiently_aligned_for(ptr, alignment) 33 | } 34 | 35 | fn is_not_aligned_for(ptr: usize, alignment: usize) -> bool { 36 | !is_aligned_for(ptr, alignment) 37 | } 38 | 39 | assert!(is_aligned_for(1, 1)); 40 | assert!(is_aligned_for(2, 1)); 41 | assert!(is_aligned_for(3, 1)); 42 | assert!(is_aligned_for(4, 1)); 43 | 44 | assert!(is_not_aligned_for(1, 2)); 45 | assert!(is_aligned_for(2, 2)); 46 | assert!(is_not_aligned_for(3, 2)); 47 | assert!(is_aligned_for(4, 2)); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /llmalloc/src/platform/api.rs: -------------------------------------------------------------------------------- 1 | //! API of OS required services. 2 | 3 | use core::ptr::NonNull; 4 | 5 | pub use llmalloc_core::Configuration; 6 | 7 | /// Abstraction over OS services. 8 | pub(crate) trait Platform : llmalloc_core::Platform + Send + Sync { 9 | /// Returns the current NUMA node on which the thread is running. 10 | /// 11 | /// As the thread may migrate to another node at the scheduler's whim, the actual result has no impact on 12 | /// correctness. It does, however, impact performance: it is better for a node's thread to access memory 13 | /// stored in the node's memory banks, rather than another node. 14 | fn current_node(&self) -> NumaNodeIndex; 15 | } 16 | 17 | /// Abstraction over thread-local storage. 18 | pub(crate) trait ThreadLocal { 19 | /// Returns a pointer to the thread-local value associated to this instance. 20 | /// 21 | /// May return a null pointer if no prior value was set, or it was already destructed. 22 | fn get(&self) -> Option>; 23 | 24 | /// Sets the pointer to the thread-local value associated to this instance. 25 | /// 26 | /// # Safety 27 | /// 28 | /// - Assumes that the value is not already set. 29 | fn set(&self, value: NonNull); 30 | } 31 | 32 | /// Index of a NUMA node. 33 | #[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Eq, Ord, Hash)] 34 | pub(crate) struct NumaNodeIndex(u32); 35 | 36 | impl NumaNodeIndex { 37 | /// Creates a NumaNodeIndex. 38 | pub(crate) fn new(value: u32) -> Self { Self(value) } 39 | 40 | /// Retrieves the index. 41 | pub(crate) fn value(&self) -> u32 { self.0 } 42 | } 43 | -------------------------------------------------------------------------------- /llmalloc-core/src/api/platform.rs: -------------------------------------------------------------------------------- 1 | //! Platform 2 | //! 3 | //! The Platform trait is used to request memory directly from the Platform. By abstracting the underlying platform, 4 | //! it becomes possible to easily port the code to a different OS, or even to a bare-metal target. 5 | 6 | use core::{ 7 | alloc::Layout, 8 | ptr::NonNull, 9 | }; 10 | 11 | /// Abstraction of platform specific memory allocation and deallocation. 12 | pub trait Platform { 13 | /// Allocates a fresh block of memory as per the specified layout. 14 | /// 15 | /// May return a null pointer if the allocation request cannot be satisfied. 16 | /// 17 | /// # Safety 18 | /// 19 | /// The caller may assume that if the returned pointer is not null then: 20 | /// - The number of usable bytes is _at greater than or equal_ to `layout.size()`. 21 | /// - The pointer is _at least_ aligned to `layout.align()`. 22 | /// 23 | /// `allocate` assumes that: 24 | /// - `layout.size()` is a multiple of `layout.align()`. 25 | /// - `layout.align()` is non-zero, and is a power of 2. 26 | unsafe fn allocate(&self, layout: Layout) -> Option>; 27 | 28 | /// Deallocates the supplied block of memory. 29 | /// 30 | /// # Safety 31 | /// 32 | /// The caller should no longer reference the memory after calling this function. 33 | /// 34 | /// `deallocate` assumes that: 35 | /// - `pointer` was allocated by this instance of `Platform`, with `layout` as argument. 36 | /// - `pointer` is the value returned by `Plaform`, and not an interior pointer. 37 | unsafe fn deallocate(&self, pointer: NonNull, layout: Layout); 38 | } 39 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/huge_page/page_index.rs: -------------------------------------------------------------------------------- 1 | //! The index of a Huge Page. 2 | 3 | use core::num::NonZeroUsize; 4 | 5 | #[derive(Clone, Copy)] 6 | pub(crate) struct PageIndex(NonZeroUsize); 7 | 8 | impl PageIndex { 9 | /// Creates an instance of PageIndex, or None if `index` is zero. 10 | pub(crate) fn new(index: usize) -> Option { NonZeroUsize::new(index).map(PageIndex) } 11 | 12 | /// Creates an instance of PageIndex. 13 | /// 14 | /// # Safety 15 | /// 16 | /// - Assumes that `index` is non-zero. 17 | pub(crate) unsafe fn new_unchecked(index: usize) -> PageIndex { 18 | debug_assert!(index > 0); 19 | 20 | PageIndex(NonZeroUsize::new_unchecked(index)) 21 | } 22 | 23 | /// Returns the inner value. 24 | pub(crate) fn value(&self) -> usize { self.0.get() } 25 | } 26 | 27 | #[cfg(test)] 28 | mod tests { 29 | 30 | use super::*; 31 | 32 | #[test] 33 | fn page_index_new() { 34 | fn new(index: usize) -> usize { PageIndex::new(index).unwrap().value() } 35 | 36 | assert!(PageIndex::new(0).is_none()); 37 | 38 | assert_eq!(1, new(1)); 39 | assert_eq!(3, new(3)); 40 | assert_eq!(42, new(42)); 41 | assert_eq!(99, new(99)); 42 | assert_eq!(1023, new(1023)); 43 | } 44 | 45 | #[test] 46 | fn page_index_new_unchecked() { 47 | fn new(index: usize) -> usize { 48 | assert_ne!(0, index); 49 | 50 | // Safety: 51 | // - `index` is not 0. 52 | unsafe { PageIndex::new_unchecked(index) }.value() 53 | } 54 | 55 | assert_eq!(1, new(1)); 56 | assert_eq!(3, new(3)); 57 | assert_eq!(42, new(42)); 58 | assert_eq!(99, new(99)); 59 | assert_eq!(1023, new(1023)); 60 | } 61 | 62 | } // mod tests 63 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/atomic.rs: -------------------------------------------------------------------------------- 1 | //! Building brick for List and Stack. 2 | 3 | use core::{ 4 | ptr::{self, NonNull}, 5 | sync::atomic::{self, Ordering}, 6 | }; 7 | 8 | // Automatically uses Acquire/Release, to synchronize before CellLocal conversion. 9 | #[derive(Default)] 10 | pub(crate) struct AtomicLength(atomic::AtomicUsize); 11 | 12 | impl AtomicLength { 13 | pub(crate) fn load(&self) -> usize { self.0.load(Ordering::Acquire) } 14 | 15 | pub(crate) fn store(&self, len: usize) { self.0.store(len, Ordering::Release) } 16 | } 17 | 18 | // Automatically uses Acquire/Release, to synchronize before CellLocal conversion. 19 | pub(crate) struct AtomicPtr(atomic::AtomicPtr); 20 | 21 | impl AtomicPtr { 22 | pub(crate) fn load(&self) -> Option> { NonNull::new(self.0.load(Ordering::Acquire)) } 23 | 24 | pub(crate) fn store(&self, ptr: Option>) { self.0.store(into_raw(ptr), Ordering::Release) } 25 | 26 | pub(crate) fn exchange(&self, ptr: Option>) -> Option> { 27 | NonNull::new(self.0.swap(into_raw(ptr), Ordering::AcqRel)) 28 | } 29 | 30 | pub(crate) fn compare_exchange(&self, current: Option>, new: Option>) 31 | -> Result>, Option>> 32 | { 33 | self.0.compare_exchange(into_raw(current), into_raw(new), Ordering::AcqRel, Ordering::Acquire) 34 | .map(NonNull::new) 35 | .map_err(NonNull::new) 36 | } 37 | } 38 | 39 | impl Default for AtomicPtr { 40 | fn default() -> Self { Self(atomic::AtomicPtr::new(ptr::null_mut())) } 41 | } 42 | 43 | // 44 | // Implementation 45 | // 46 | 47 | #[inline(always)] 48 | fn into_raw(ptr: Option>) -> *mut T { 49 | ptr.map(|t| t.as_ptr()) 50 | .unwrap_or(ptr::null_mut()) 51 | } 52 | -------------------------------------------------------------------------------- /llmalloc-core/src/api/domain.rs: -------------------------------------------------------------------------------- 1 | //! Domain Handle. 2 | //! 3 | //! An instance of the Domain Handle defines an allocation Domain: 4 | //! 5 | //! - A single allocation Domain is connected to 1 to N Sockets. 6 | //! - A single Socket is connected to 1 to N Threads. 7 | //! 8 | //! The allocation Domain is its own island, memory wise: 9 | //! 10 | //! - Any piece of memory allocated by a connected Socket (and Thread) is served by the Domain. 11 | //! - In exchange, pieces of memory MUST be deallocated by a Socket (and Thread) connected to its original Domain. 12 | //! 13 | //! Typically, applications will use a global Domain. 14 | 15 | use crate::internals::huge_allocator::HugeAllocator; 16 | 17 | /// Domain Handle. 18 | /// 19 | /// By design, `free` only takes a pointer. At the same time, the `Platform` abstraction requires that the layout of 20 | /// the pointer to be deallocated is passed. 21 | /// 22 | /// The `DomainHandle` bridges the gap by recording the original layout on allocation and providing it back on deallocation. 23 | /// 24 | /// # Limitation 25 | /// 26 | /// A single `DomainHandle` is limited to 128 allocations above C::HUGE_PAGE_SIZE. 27 | pub struct DomainHandle(HugeAllocator); 28 | 29 | impl DomainHandle { 30 | /// Creates a Domain. 31 | /// 32 | /// The Domain created will allocate memory from the `platform`, and return it to the `platform`. 33 | pub const fn new(platform: P) -> Self { Self(HugeAllocator::new(platform)) } 34 | 35 | /// Returns a reference to the underlying platform. 36 | pub fn platform(&self) -> &P { self.0.platform() } 37 | 38 | /// Returns a reference to the raw handle. 39 | pub(crate) fn as_raw(&self) -> &HugeAllocator { &self.0 } 40 | } 41 | 42 | impl Default for DomainHandle 43 | where 44 | P: Default 45 | { 46 | fn default() -> Self { Self::new(P::default()) } 47 | } 48 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/blocks/block_ptr.rs: -------------------------------------------------------------------------------- 1 | //! Pointer to Block, unsynchronized. 2 | 3 | use core::{ 4 | cell::Cell, 5 | ptr::NonNull, 6 | }; 7 | 8 | /// BlockPtr 9 | /// 10 | /// A simple block over a potentially null pointer to T. 11 | pub(crate) struct BlockPtr(Cell>>); 12 | 13 | impl BlockPtr { 14 | /// Creates an instance. 15 | pub(crate) fn new(ptr: Option>) -> Self { Self(Cell::new(ptr)) } 16 | 17 | /// Returns the inner pointer, possibly null. 18 | pub(crate) fn get(&self) -> Option> { self.0.get() } 19 | 20 | /// Sets the inner pointer. 21 | pub(crate) fn set(&self, ptr: Option>) { self.0.set(ptr); } 22 | 23 | /// Sets the inner pointer to null and return the previous value, possibly null. 24 | pub(crate) fn replace_with_null(&self) -> Option> { self.0.replace(None) } 25 | } 26 | 27 | impl Default for BlockPtr { 28 | fn default() -> Self { Self::new(None) } 29 | } 30 | 31 | #[cfg(test)] 32 | mod tests { 33 | 34 | use super::*; 35 | 36 | #[test] 37 | fn block_ptr_new() { 38 | let a = 1u8; 39 | let a = Some(NonNull::from(&a)); 40 | 41 | let block = BlockPtr::::new(None); 42 | assert_eq!(None, block.get()); 43 | 44 | let block = BlockPtr::new(a); 45 | assert_eq!(a, block.get()); 46 | } 47 | 48 | #[test] 49 | fn block_ptr_get_set() { 50 | let (a, b) = (1u8, 2u8); 51 | let (a, b) = (Some(NonNull::from(&a)), Some(NonNull::from(&b))); 52 | 53 | let block = BlockPtr::new(None); 54 | 55 | block.set(a); 56 | assert_eq!(a, block.get()); 57 | 58 | block.set(b); 59 | assert_eq!(b, block.get()); 60 | } 61 | 62 | #[test] 63 | fn block_ptr_replace_with_null() { 64 | let a = 1u8; 65 | let a = Some(NonNull::from(&a)); 66 | 67 | let block = BlockPtr::new(None); 68 | 69 | assert_eq!(None, block.replace_with_null()); 70 | assert_eq!(None, block.get()); 71 | 72 | block.set(a); 73 | 74 | assert_eq!(a, block.replace_with_null()); 75 | assert_eq!(None, block.get()); 76 | } 77 | 78 | } // mod tests 79 | -------------------------------------------------------------------------------- /llmalloc-core/src/api/thread.rs: -------------------------------------------------------------------------------- 1 | //! Thread Handle 2 | //! 3 | //! The Thread Handle is a thread-local cache; the user is expected to allocate one for each of the threads they use, 4 | //! and on each allocation to refer to thread-local Thread Handle. 5 | //! 6 | //! The lack of `AtomicThreadHandle` reflects the fact that no two threads should ever contend for a single handle. 7 | //! 8 | //! # Safety 9 | //! 10 | //! A Thread Handle _assumes_ it is only used from a single thread, and makes no attempt at synchronizing memory 11 | //! accesses. It is Undefined Behavior to use a Thread Handle from multiple threads without external synchronization. 12 | 13 | use core::ptr::NonNull; 14 | 15 | use crate::{Configuration, SocketHandle}; 16 | use crate::internals::thread_local::ThreadLocal; 17 | 18 | /// Handle to thread-local cache. 19 | pub struct ThreadHandle(NonNull>); 20 | 21 | impl ThreadHandle 22 | where 23 | C: Configuration 24 | { 25 | /// Rematerialize from raw pointer. 26 | /// 27 | /// # Safety 28 | /// 29 | /// - Assumes `pointer` points to a valid instance of `ThreadHandle` 30 | pub unsafe fn from_pointer(pointer: NonNull) -> ThreadHandle { 31 | Self::new(pointer.cast()) 32 | } 33 | 34 | /// Turn into raw pointer. 35 | pub fn into_pointer(self) -> NonNull { self.0.cast() } 36 | 37 | /// Get associated SocketHandle. 38 | /// 39 | /// # Safety 40 | /// 41 | /// - Assumes that the lifetime and platform are correct. 42 | pub unsafe fn socket<'a, P>(&self) -> SocketHandle<'a, C, P> { 43 | let socket = self.0.as_ref().owner(); 44 | debug_assert!(!socket.is_null()); 45 | 46 | SocketHandle::from(NonNull::new_unchecked(socket).cast()) 47 | } 48 | 49 | /// Creates an instance. 50 | pub(crate) fn new(value: NonNull>) -> Self { Self(value) } 51 | 52 | /// Back to raw. 53 | pub(crate) fn into_raw(self) -> NonNull> { self.0 } 54 | 55 | /// Yield the underlying reference. 56 | pub(crate) unsafe fn as_ref(&self) -> &ThreadLocal { self.0.as_ref() } 57 | } 58 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/blocks/block_foreign.rs: -------------------------------------------------------------------------------- 1 | //! A Block of memory of a Foreign thread, only accessed by the local thread. 2 | 3 | use core::{ 4 | cell::Cell, 5 | ptr::{self, NonNull}, 6 | }; 7 | 8 | use crate::{PowerOf2, utils}; 9 | 10 | use super::BlockPtr; 11 | 12 | /// BlockForeign. 13 | /// 14 | /// A BlockForeign points to memory not local to the current ThreadLocal, but is still only manipulated by the current 15 | /// thread. 16 | #[repr(C)] 17 | #[derive(Default)] 18 | pub(crate) struct BlockForeign { 19 | // Pointer to the next cell, linked-list style, if any. 20 | pub(crate) next: BlockPtr, 21 | // Length of linked-list starting at the next cell in CellAtomicForeignPtr. 22 | // Only accurate for the head. 23 | pub(crate) length: Cell, 24 | // Tail of the list, only used by BlockForeignList. 25 | pub(crate) tail: BlockPtr, 26 | } 27 | 28 | impl BlockForeign { 29 | /// In-place constructs a `CellAtomicForeign`. 30 | /// 31 | /// # Safety 32 | /// 33 | /// - Assumes that access to the memory location is exclusive. 34 | /// - Assumes that there is sufficient memory available. 35 | /// - Assumes that the pointer is correctly aligned. 36 | #[allow(clippy::cast_ptr_alignment)] 37 | pub(crate) unsafe fn initialize(at: NonNull) -> NonNull { 38 | debug_assert!(utils::is_sufficiently_aligned_for(at, PowerOf2::align_of::())); 39 | 40 | // Safety: 41 | // - `at` is assumed to be sufficiently aligned. 42 | let ptr = at.as_ptr() as *mut Self; 43 | 44 | // Safety: 45 | // - Access to the memory location is exclusive. 46 | // - `at` is assumed to be sufficiently sized. 47 | ptr::write(ptr, Self::default()); 48 | 49 | at.cast() 50 | } 51 | } 52 | 53 | #[cfg(test)] 54 | mod tests { 55 | 56 | use core::mem::MaybeUninit; 57 | 58 | use super::*; 59 | 60 | #[test] 61 | fn block_foreign_initialize() { 62 | let mut block = MaybeUninit::::uninit(); 63 | 64 | // Safety: 65 | // - Access to the memory location is exclusive. 66 | unsafe { ptr::write_bytes(block.as_mut_ptr(), 0xfe, 1) }; 67 | 68 | // Safety: 69 | // - Access to the memory location is exclusive. 70 | // - The memory location is sufficiently sized and aligned for `BlockForeign`. 71 | unsafe { BlockForeign::initialize(NonNull::from(&block).cast()) }; 72 | 73 | // Safety: 74 | // - Initialized! 75 | let block = unsafe { block.assume_init() }; 76 | 77 | assert!(block.next.get().is_none()); 78 | assert_eq!(0, block.length.get()); 79 | assert!(block.tail.get().is_none()); 80 | } 81 | 82 | } // mod tests 83 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/large_page/test.rs: -------------------------------------------------------------------------------- 1 | //! A Helper for tests. 2 | 3 | use core::{ 4 | mem, 5 | ops::Range, 6 | ptr::NonNull, 7 | }; 8 | 9 | use crate::internals::blocks::{AtomicBlockForeign, AtomicBlockForeignList, BlockForeign, BlockForeignList}; 10 | 11 | use super::local::Local; 12 | 13 | pub(crate) const BLOCK_SIZE: usize = 32; 14 | 15 | pub(crate) struct BlockStore([usize; 256]); 16 | 17 | impl BlockStore { 18 | pub(crate) const CAPACITY: usize = 256; 19 | 20 | pub(crate) fn get(&self, index: usize) -> NonNull { NonNull::from(&self.0[index]).cast() } 21 | 22 | pub(crate) fn end(&self) -> NonNull { 23 | let pointer = unsafe { self.get(0).as_ptr().add(BLOCK_SIZE / 4 * BlockStore::CAPACITY) }; 24 | NonNull::new(pointer).unwrap() 25 | } 26 | 27 | /// Borrows self, outside of the compiler's overview. 28 | pub(crate) unsafe fn create_local(&self, block_size: usize) -> Local { 29 | assert!(block_size >= mem::size_of::()); 30 | 31 | let (begin, end) = self.begin_end(block_size); 32 | 33 | Local::new(block_size, begin, end) 34 | } 35 | 36 | /// Creates a `BlockForeignList` containing the specified range of cells. 37 | /// 38 | /// # Safety 39 | /// 40 | /// - `local` should have been created from this instance. 41 | /// - The cells should not _also_ be available through `local.next`. 42 | pub(crate) unsafe fn create_foreign_list(&self, local: &Local, blocks: Range) -> BlockForeignList { 43 | assert!(blocks.start <= blocks.end); 44 | 45 | let block_size = local.block_size(); 46 | 47 | let (begin, end) = self.begin_end(block_size); 48 | assert_eq!(end, local.end()); 49 | assert!(blocks.end <= (end.as_ptr() as usize - begin.as_ptr() as usize) / block_size); 50 | 51 | let list = BlockForeignList::default(); 52 | 53 | for index in blocks.rev() { 54 | let block_address = begin.as_ptr().add(index * block_size); 55 | let block = NonNull::new(block_address as *mut BlockForeign).unwrap(); 56 | list.push(block); 57 | } 58 | 59 | list 60 | } 61 | 62 | /// Creates a stack of `BlockForeign` containing the specified range of blocks. 63 | /// 64 | /// # Safety 65 | /// 66 | /// - `local` should have been created from this instance. 67 | /// - The blocks should not _also_ be available through `local.next`. 68 | pub(crate) unsafe fn create_foreign_stack(&self, local: &Local, blocks: Range) -> NonNull { 69 | let list = self.create_foreign_list(local, blocks); 70 | 71 | let block = AtomicBlockForeignList::default(); 72 | block.extend(&list); 73 | 74 | block.steal().unwrap() 75 | } 76 | 77 | // Internal: Compute begin and end for a given `block_size`. 78 | unsafe fn begin_end(&self, block_size: usize) -> (NonNull, NonNull) { 79 | let begin = self as *const Self as *mut Self as *mut u8; 80 | let end = begin.add(mem::size_of::()); 81 | 82 | let number_elements = (end as usize - begin as usize) / block_size; 83 | let begin = end.sub(number_elements * block_size); 84 | 85 | (NonNull::new(begin).unwrap(), NonNull::new(end).unwrap()) 86 | } 87 | } 88 | 89 | impl Default for BlockStore { 90 | fn default() -> Self { 91 | let result: Self = unsafe { mem::zeroed() }; 92 | assert_eq!(Self::CAPACITY, result.0.len()); 93 | 94 | result 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /llmalloc-c/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | #![deny(missing_docs)] 3 | 4 | //! Exposition of LLAllocator API via a C ABI. 5 | 6 | use core::{ 7 | alloc::Layout, 8 | ptr::{self, NonNull}, 9 | }; 10 | 11 | use llmalloc::LLAllocator; 12 | 13 | /// Prepares the socket-local and thread-local structures for allocation. 14 | /// 15 | /// Returns 0 on success, and a negative value otherwise. 16 | /// 17 | /// Failure to warm up the current thread may occur if: 18 | /// 19 | /// - The socket-local structure is not ready, and the underlying `Platform` cannot allocate one. 20 | /// - The socket-local structure cannot allocate a thread-local structure. 21 | #[cold] 22 | #[no_mangle] 23 | pub extern fn ll_warm_up() -> i32 { if ALLOCATOR.warm_up().is_ok() { 0 } else { -1 } } 24 | 25 | /// Ensures that at least `target` `HugePage` are allocated on the socket. 26 | /// 27 | /// Returns the minimum of the currently allocated number of pages and `target`. 28 | /// 29 | /// Failure to meet the `target` may occur if: 30 | /// 31 | /// - The maximum number of `HugePage` a `socket` may contain has been reached. 32 | /// - The underlying `Platform` is failing to allocate more `HugePage`. 33 | #[cold] 34 | #[no_mangle] 35 | pub extern fn ll_reserve(target: usize) -> usize { ALLOCATOR.reserve(target) } 36 | 37 | /// Allocates `size` bytes of memory, generally suitably aligned. 38 | /// 39 | /// If the allocation fails, the returned pointer may be NULL. 40 | /// 41 | /// If the allocation succeeds, the pointer is aligned on the greatest power of 2 which divides `size`, or 1 if `size` 42 | /// is 0; this guarantees that the pointer is suitably aligned: 43 | /// 44 | /// - The alignment of the type for which memory is allocated must be a power of 2. 45 | /// - The size of the type for which memory is allocated must be a multiple of its alignment. 46 | /// - Therefore, the greatest power of 2 which divides `size` is greater than the required alignment. 47 | pub extern fn ll_malloc(size: usize) -> *mut u8 { 48 | let shift = size.trailing_zeros(); 49 | let alignment = 1usize << shift; 50 | 51 | // Safety: 52 | // - `alignment` is non-zero. 53 | // - `alignment` is a power of 2. 54 | // - `size` is a multiple of `alignment`. 55 | let layout = unsafe { Layout::from_size_align_unchecked(size, alignment) }; 56 | 57 | ALLOCATOR.allocate(layout).map(|ptr| ptr.as_ptr()).unwrap_or(ptr::null_mut()) 58 | } 59 | 60 | /// Allocates `size` bytes of memory, aligned as specified. 61 | /// 62 | /// If the allocation fails, the returned pointer may be NULL. 63 | /// 64 | /// # Safety 65 | /// 66 | /// - Assumes that `alignment` is non-zero. 67 | /// - Assumes that `alignment` is a power of 2. 68 | /// - Assumes that `size` is a multiple of `alignment`. 69 | pub unsafe extern fn ll_aligned_malloc(size: usize, alignment: usize) -> *mut u8 { 70 | // Safety: 71 | // - `alignment` is non-zero. 72 | // - `alignment` is a power of 2. 73 | // - `size` is a multiple of `alignment`. 74 | let layout = Layout::from_size_align_unchecked(size, alignment); 75 | 76 | ALLOCATOR.allocate(layout).map(|ptr| ptr.as_ptr()).unwrap_or(ptr::null_mut()) 77 | } 78 | 79 | /// Deallocates the memory located at `pointer`. 80 | /// 81 | /// # Safety 82 | /// 83 | /// - Assumes `pointer` has been returned by a prior call to `allocate`. 84 | /// - Assumes `pointer` has not been deallocated since its allocation. 85 | /// - Assumes the memory pointed by `pointer` is no longer in use. 86 | pub unsafe extern fn ll_free(pointer: *mut u8) { 87 | if let Some(pointer) = NonNull::new(pointer) { 88 | ALLOCATOR.deallocate(pointer) 89 | } 90 | } 91 | 92 | // 93 | // Implementation 94 | // 95 | 96 | static ALLOCATOR: LLAllocator = LLAllocator::new(); 97 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/socket_local/test.rs: -------------------------------------------------------------------------------- 1 | //! Test helpers for socket_local. 2 | 3 | use core::{ 4 | alloc::Layout, 5 | cell::Cell, 6 | mem, 7 | ptr::NonNull, 8 | }; 9 | 10 | use crate::{Configuration, Platform, PowerOf2}; 11 | use super::{HugeAllocator, HugePagesManager}; 12 | 13 | /// Test Huge Allocator 14 | pub(crate) type TestHugeAllocator = HugeAllocator; 15 | 16 | // Test Huge Pages Manager 17 | pub(crate) type TestHugePagesManager = HugePagesManager; 18 | 19 | /// Test configuration 20 | pub(crate) struct TestConfiguration; 21 | 22 | impl Configuration for TestConfiguration { 23 | const LARGE_PAGE_SIZE: PowerOf2 = unsafe { PowerOf2::new_unchecked(4 * 1024) }; 24 | const HUGE_PAGE_SIZE: PowerOf2 = unsafe { PowerOf2::new_unchecked(8 * 1024) }; 25 | } 26 | 27 | /// Test Platform 28 | pub(crate) struct TestPlatform([Cell>>; 32]); 29 | 30 | impl TestPlatform { 31 | pub(crate) const HUGE_PAGE_SIZE: usize = TestConfiguration::HUGE_PAGE_SIZE.value(); 32 | 33 | pub(crate) unsafe fn new(store: &HugePageStore) -> TestPlatform { 34 | let stores: [Cell>>; 32] = Default::default(); 35 | 36 | for (i, cell) in stores.iter().enumerate() { 37 | cell.set(NonNull::new(store.as_ptr().add(i * Self::HUGE_PAGE_SIZE))); 38 | } 39 | 40 | TestPlatform(stores) 41 | } 42 | 43 | // Creates a TestHugeAllocator. 44 | pub(crate) unsafe fn allocator(store: &HugePageStore) -> TestHugeAllocator { 45 | TestHugeAllocator::new(Self::new(store)) 46 | } 47 | 48 | // Shrink the number of allocations to at most n. 49 | pub(crate) fn shrink(&self, n: usize) { 50 | for ptr in &self.0[n..] { 51 | ptr.set(None); 52 | } 53 | } 54 | 55 | // Exhausts the HugePagesManager. 56 | pub(crate) fn exhaust(&self, manager: &TestHugePagesManager) { 57 | let owner = 0x1234 as *mut (); 58 | let platform = TestPlatform::default(); 59 | 60 | loop { 61 | let large = unsafe { manager.allocate_large(LARGE_PAGE_LAYOUT, owner, &platform) }; 62 | 63 | if large.is_none() { break; } 64 | } 65 | } 66 | 67 | // Returns the number of allocated pages. 68 | pub(crate) fn allocated(&self) -> usize { self.0.len() - self.available() } 69 | 70 | // Returns the number of available pages. 71 | pub(crate) fn available(&self) -> usize { self.0.iter().filter(|p| p.get().is_some()).count() } 72 | } 73 | 74 | impl Platform for TestPlatform { 75 | unsafe fn allocate(&self, layout: Layout) -> Option> { 76 | assert_eq!(Self::HUGE_PAGE_SIZE, layout.size()); 77 | assert_eq!(Self::HUGE_PAGE_SIZE, layout.align()); 78 | 79 | for ptr in &self.0[..] { 80 | if ptr.get().is_none() { 81 | continue; 82 | } 83 | 84 | return ptr.replace(None); 85 | } 86 | 87 | None 88 | } 89 | 90 | unsafe fn deallocate(&self, pointer: NonNull, layout: Layout) { 91 | assert_eq!(Self::HUGE_PAGE_SIZE, layout.size()); 92 | assert_eq!(Self::HUGE_PAGE_SIZE, layout.align()); 93 | 94 | for ptr in &self.0[..] { 95 | if ptr.get().is_some() { 96 | continue; 97 | } 98 | 99 | ptr.set(Some(pointer)); 100 | return; 101 | } 102 | } 103 | } 104 | 105 | impl Default for TestPlatform { 106 | fn default() -> Self { unsafe { mem::zeroed() } } 107 | } 108 | 109 | /// Store of Huge Pages. 110 | pub(crate) struct HugePageStore(Vec); // 128K 111 | 112 | impl HugePageStore { 113 | pub(crate) fn as_ptr(&self) -> *mut u8 { self.0.as_ptr() as *const u8 as *mut _ } 114 | } 115 | 116 | impl Default for HugePageStore { 117 | fn default() -> Self { 118 | let mut vec = vec!(); 119 | // 256K worth of memory 120 | vec.resize(256 * 1024 / mem::size_of::(), HugePageCell::default()); 121 | 122 | Self(vec) 123 | } 124 | } 125 | 126 | // 127 | // Implementation 128 | // 129 | 130 | const LARGE_PAGE_SIZE: usize = TestConfiguration::LARGE_PAGE_SIZE.value(); 131 | const LARGE_PAGE_LAYOUT: Layout = unsafe { Layout::from_size_align_unchecked(LARGE_PAGE_SIZE, LARGE_PAGE_SIZE) }; 132 | 133 | #[repr(align(8192))] 134 | #[derive(Clone, Default)] 135 | struct HugePageCell(u8); 136 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/huge_page/foreign.rs: -------------------------------------------------------------------------------- 1 | use crate::PowerOf2; 2 | 3 | use super::{ 4 | NumberPages, 5 | PageIndex, 6 | page_sizes::PageSizes, 7 | page_tokens::PageTokens, 8 | }; 9 | 10 | // Foreign data. Accessible both from the local thread and foreign threads, at the cost of synchronization. 11 | #[repr(align(128))] 12 | pub(crate) struct Foreign { 13 | // Bitmap of pages. 14 | pages: PageTokens, 15 | // Sizes of allocations. 16 | sizes: PageSizes, 17 | // Actual number of available pages. 18 | number_pages: NumberPages, 19 | } 20 | 21 | impl Foreign { 22 | /// Creates a new instance of `Foreign`. 23 | pub(crate) fn new(number_pages: NumberPages) -> Self { 24 | let pages = PageTokens::new(number_pages); 25 | let sizes = PageSizes::default(); 26 | 27 | Self { pages, sizes, number_pages, } 28 | } 29 | 30 | /// Allocates `n` consecutive pages, returns their index. 31 | /// 32 | /// The index returned is a multiple of `align_pages`. 33 | /// 34 | /// Returns 0 if no allocation could be made. 35 | pub(crate) unsafe fn allocate(&self, number_pages: NumberPages, align_pages: PowerOf2) -> Option { 36 | if number_pages.0 == 1 { 37 | self.fast_allocate() 38 | } else { 39 | self.flexible_allocate(number_pages, align_pages) 40 | } 41 | } 42 | 43 | /// Deallocates all cells allocated at the given index. 44 | pub(crate) unsafe fn deallocate(&self, index: PageIndex) { 45 | debug_assert!(index.value() > 0); 46 | debug_assert!(index.value() <= self.number_pages.0); 47 | 48 | // Safety: 49 | // - `index` is assumed to be within bounds. 50 | let number_pages = self.sizes.get(index); 51 | 52 | if number_pages.0 == 1 { 53 | self.fast_deallocate(index); 54 | } else { 55 | self.flexible_deallocate(index, number_pages); 56 | } 57 | } 58 | 59 | // Internal: fast-allocate a single page. 60 | fn fast_allocate(&self) -> Option { self.pages.fast_allocate() } 61 | 62 | // Internal: allocate multiple pages aligned on `align_pages`, if possible. 63 | fn flexible_allocate(&self, number_pages: NumberPages, align_pages: PowerOf2) -> Option { 64 | let index = self.pages.flexible_allocate(number_pages, align_pages); 65 | 66 | if let Some(index) = index { 67 | // Safety: 68 | // - `index` is within bounds. 69 | unsafe { self.sizes.set(index, number_pages) }; 70 | } 71 | 72 | index 73 | } 74 | 75 | // Internal. 76 | unsafe fn fast_deallocate(&self, index: PageIndex) { self.pages.fast_deallocate(index) } 77 | 78 | // Internal. 79 | unsafe fn flexible_deallocate(&self, index: PageIndex, number_pages: NumberPages) { 80 | self.pages.flexible_deallocate(index, number_pages); 81 | self.sizes.unset(index, number_pages); 82 | } 83 | } 84 | 85 | #[cfg(test)] 86 | mod tests { 87 | 88 | use super::*; 89 | 90 | #[test] 91 | fn foreign_allocate_deallocate_fast() { 92 | fn allocate_fast(foreign: &Foreign) -> Option { 93 | unsafe { foreign.allocate(NumberPages(1), PowerOf2::ONE) }.map(|x| x.value()) 94 | } 95 | 96 | let foreign = Foreign::new(NumberPages(511)); 97 | 98 | for i in 0..95 { 99 | assert_eq!(Some(i + 1), allocate_fast(&foreign)); 100 | } 101 | 102 | let reused = PageIndex::new(75).unwrap(); 103 | 104 | unsafe { foreign.deallocate(reused) }; 105 | 106 | assert_eq!(Some(reused.value()), allocate_fast(&foreign)); 107 | } 108 | 109 | #[test] 110 | fn foreign_allocate_deallocate_flexible() { 111 | fn allocate_flexible(foreign: &Foreign, number_pages: usize) -> Option { 112 | unsafe { foreign.allocate(NumberPages(number_pages), PowerOf2::ONE) }.map(|x| x.value()) 113 | } 114 | 115 | let foreign = Foreign::new(NumberPages(511)); 116 | 117 | assert_eq!(Some(64 * 7 + 38), allocate_flexible(&foreign, 26)); 118 | assert_eq!(Some(64 * 7 + 13), allocate_flexible(&foreign, 25)); 119 | assert_eq!(Some(64 * 6 + 51), allocate_flexible(&foreign, 26)); 120 | assert_eq!(Some(64 * 6 + 26), allocate_flexible(&foreign, 25)); 121 | 122 | unsafe { foreign.deallocate(PageIndex::new(64 * 6 + 51).unwrap()) }; 123 | unsafe { foreign.deallocate(PageIndex::new(64 * 7 + 13).unwrap()) }; 124 | 125 | assert_eq!(Some(64 * 7 + 11), allocate_flexible(&foreign, 27)); 126 | assert_eq!(Some(64 * 6 + 51), allocate_flexible(&foreign, 24)); 127 | } 128 | 129 | } // mod tests 130 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/blocks/atomic_block_foreign.rs: -------------------------------------------------------------------------------- 1 | //! A Block of Foreign memory, accessed from multiple threads. 2 | 3 | use core::{ 4 | cell::Cell, 5 | mem, 6 | ptr::NonNull, 7 | }; 8 | 9 | use super::{AtomicLength, AtomicPtr, BlockForeign, BlockPtr}; 10 | 11 | /// AtomicBlockForeign. 12 | /// 13 | /// A AtomicBlockForeign points to memory not local to the current ThreadLocal. 14 | #[repr(C)] 15 | #[derive(Default)] 16 | pub(crate) struct AtomicBlockForeign { 17 | // Pointer to the next block, linked-list style, if any. 18 | pub(crate) next: AtomicPtr, 19 | // Length of linked-list starting at the next block in AtomicBlockForeignPtr. 20 | // Only accurate for the head. 21 | pub(crate) length: AtomicLength, 22 | } 23 | 24 | impl AtomicBlockForeign { 25 | /// In-place reinterpret a `BlockForeign` as a `AtomicBlockForeign`. 26 | /// 27 | /// # Safety 28 | /// 29 | /// - Assumes that access to the block, and all tail blocks, is exclusive. 30 | /// - Assumes that a Release atomic fence was called after the last write to the `BlockForeign` list. 31 | pub(crate) unsafe fn from(foreign: NonNull) -> NonNull { 32 | // Safety: 33 | // - The layout are checked to be compatible below. 34 | let atomic = foreign.cast(); 35 | 36 | debug_assert!(Self::are_layout_compatible(foreign, atomic)); 37 | 38 | atomic 39 | } 40 | 41 | // Returns whether the layout of BlockForeign and AtomicBlockForeign are compatible. 42 | // 43 | // The layout are compatible if: 44 | // - BlockPtr and AtomicBlockForeignPtr are both plain pointers, size-wise. 45 | // - Block and AtomicLength are both plain usize, size-wise. 46 | // - AtomicBlockForeign::next and BlockForeign::next are placed at the same offset. 47 | // - AtomicBlockForeign::length and BlockForeign::length are placed at the same offset. 48 | fn are_layout_compatible(foreign: NonNull, atomic: NonNull) -> bool { 49 | const PTR_SIZE: usize = mem::size_of::<*const u8>(); 50 | const USIZE_SIZE: usize = mem::size_of::(); 51 | 52 | if mem::size_of::>() != PTR_SIZE || mem::size_of::>() != PTR_SIZE { 53 | return false; 54 | } 55 | 56 | if mem::size_of::>() != USIZE_SIZE || mem::size_of::() != USIZE_SIZE { 57 | return false; 58 | } 59 | 60 | let (foreign_next_offset, foreign_length_offset) = { 61 | let address = foreign.as_ptr() as usize; 62 | // Safety: 63 | // - Bounded lifetime. 64 | let next_address = unsafe { &foreign.as_ref().next as *const _ as usize }; 65 | let length_address = unsafe { &foreign.as_ref().length as *const _ as usize }; 66 | (next_address - address, length_address - address) 67 | }; 68 | 69 | let (atomic_next_offset, atomic_length_offset) = { 70 | let address = atomic.as_ptr() as usize; 71 | // Safety: 72 | // - Bounded lifetime. 73 | let next_address = unsafe { &atomic.as_ref().next as *const _ as usize }; 74 | let length_address = unsafe { &atomic.as_ref().length as *const _ as usize }; 75 | (next_address - address, length_address - address) 76 | }; 77 | 78 | foreign_next_offset == atomic_next_offset && foreign_length_offset == atomic_length_offset 79 | } 80 | } 81 | 82 | #[cfg(test)] 83 | mod tests { 84 | 85 | use super::*; 86 | 87 | #[test] 88 | fn atomic_block_foreign_from() { 89 | let other = BlockForeign::default(); 90 | let other = NonNull::from(&other); 91 | 92 | { 93 | let block = BlockForeign::default(); 94 | let block = NonNull::from(&block); 95 | let atomic = unsafe { AtomicBlockForeign::from(block) }; 96 | let atomic = unsafe { atomic.as_ref() }; 97 | 98 | assert_eq!(None, atomic.next.load()); 99 | assert_eq!(0, atomic.length.load()); 100 | } 101 | 102 | { 103 | let block = BlockForeign::default(); 104 | block.next.set(Some(other)); 105 | block.length.set(1); 106 | 107 | let block = NonNull::from(&block); 108 | let atomic = unsafe { AtomicBlockForeign::from(block) }; 109 | let atomic = unsafe { atomic.as_ref() }; 110 | 111 | assert_eq!(Some(other.cast::()), atomic.next.load().map(NonNull::cast::)); 112 | assert_eq!(1, atomic.length.load()); 113 | } 114 | } 115 | 116 | } // mod tests 117 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/huge_page/page_sizes.rs: -------------------------------------------------------------------------------- 1 | //! A mapping of how many pages are allocated. 2 | 3 | use core::{ 4 | mem, 5 | sync::atomic::{AtomicU8, Ordering}, 6 | }; 7 | 8 | use super::{NumberPages, PageIndex}; 9 | 10 | // Page Sizes. 11 | // 12 | // For a given allocation spanning pages [M, N), the size of the allocation is N - M, which is registered at index 13 | // M in the array below. 14 | // 15 | // For performance reasons, the size is represented as with an implied +1, so that a value of 0 means a size of 1. 16 | // 17 | // This has two benefits: 18 | // - The fast case of single-page allocation need not register the size, as the array is initialized with 0s, 19 | // and deallocation restores 0. 20 | // - Very large allocation sizes of 257 or more can be represented with only 2 u8s: 256 + 255 == 511. 21 | pub(crate) struct PageSizes([AtomicU8; 512]); 22 | 23 | impl PageSizes { 24 | /// Returns the number of pages from a particular index. 25 | /// 26 | /// # Safety 27 | /// 28 | /// - Assumes that `index` is within bounds. 29 | pub(crate) unsafe fn get(&self, index: PageIndex) -> NumberPages { 30 | let index = index.value(); 31 | debug_assert!(index < self.0.len()); 32 | 33 | let number_pages = self.0.get_unchecked(index).load(Ordering::Acquire) as usize; 34 | 35 | if number_pages == 255 { 36 | debug_assert!(index + 1 < self.0.len()); 37 | // No implicit +1 on overflow size. 38 | NumberPages(256 + self.0.get_unchecked(index + 1).load(Ordering::Acquire) as usize) 39 | } else { 40 | // Implicit +1 41 | NumberPages(number_pages + 1) 42 | } 43 | } 44 | 45 | /// Sets the number of pages at a particular index. 46 | /// 47 | /// # Safety 48 | /// 49 | /// - Assumes that `index` is within bounds. 50 | /// - Assumes that `number_pages` is less than or equal to 511. 51 | pub(crate) unsafe fn set(&self, index: PageIndex, number_pages: NumberPages) { 52 | debug_assert!(number_pages.0 >= 1 && number_pages.0 <= (u8::MAX as usize) * 2 + 1, 53 | "index: {}, number_pages: {}", index.value(), number_pages.0); 54 | 55 | let index = index.value(); 56 | debug_assert!(index < self.0.len()); 57 | 58 | if number_pages.0 <= 256 { 59 | let number_pages = (number_pages.0 - 1) as u8; 60 | self.0.get_unchecked(index).store(number_pages, Ordering::Release); 61 | } else { 62 | let overflow = number_pages.0 - 256; 63 | debug_assert!(overflow <= (u8::MAX as usize)); 64 | 65 | self.0.get_unchecked(index).store(255, Ordering::Release); 66 | self.0.get_unchecked(index + 1).store(overflow as u8, Ordering::Release); 67 | } 68 | } 69 | 70 | /// Unsets the number of pages at a particular index. 71 | /// 72 | /// # Safety 73 | /// 74 | /// - Assumes that `index` is within bounds. 75 | /// - Assumes that `number_pages` is less than or equal to 511. 76 | pub(crate) unsafe fn unset(&self, index: PageIndex, number_pages: NumberPages) { 77 | let index = index.value(); 78 | debug_assert!(index < self.0.len()); 79 | 80 | self.0.get_unchecked(index).store(0, Ordering::Release); 81 | 82 | if number_pages.0 > 256 { 83 | debug_assert!(self.0[index + 1].load(Ordering::Acquire) > 0); 84 | 85 | self.0.get_unchecked(index + 1).store(0, Ordering::Release); 86 | } 87 | } 88 | } 89 | 90 | impl Default for PageSizes { 91 | fn default() -> Self { unsafe { mem::zeroed() } } 92 | } 93 | 94 | #[cfg(test)] 95 | mod tests { 96 | 97 | use super::*; 98 | 99 | #[test] 100 | fn page_sizes_get_set() { 101 | // Creates a PageSizes, call `set` then with `index` and `number`, call `get` with `index`, return its result. 102 | fn set_get(index: usize, number: usize) -> usize { 103 | let index = PageIndex::new(index).unwrap(); 104 | 105 | let page_sizes = PageSizes::default(); 106 | 107 | unsafe { page_sizes.set(index, NumberPages(number)) }; 108 | 109 | unsafe { page_sizes.get(index).0 } 110 | } 111 | 112 | for index in 1..512 { 113 | assert_eq!(1, set_get(index, 1)); 114 | } 115 | 116 | for number in 1..512 { 117 | assert_eq!(number, set_get(1, number)); 118 | } 119 | } 120 | 121 | #[test] 122 | fn page_sizes_unset() { 123 | // Creates a PageSizes, call `set` then `unset` with `index` and `number`, call `get` with `index`, return its result. 124 | fn unset(index: usize, number: usize) -> usize { 125 | let index = PageIndex::new(index).unwrap(); 126 | 127 | let page_sizes = PageSizes::default(); 128 | 129 | unsafe { page_sizes.set(index, NumberPages(number)) }; 130 | unsafe { page_sizes.unset(index, NumberPages(number)) }; 131 | 132 | unsafe { page_sizes.get(index).0 } 133 | } 134 | 135 | for index in 1..512 { 136 | assert_eq!(1, unset(index, 1)); 137 | } 138 | 139 | for number in 1..512 { 140 | assert_eq!(1, unset(1, number)); 141 | } 142 | } 143 | 144 | } // mod tests 145 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/large_page/adrift.rs: -------------------------------------------------------------------------------- 1 | //! The Adrift flag. 2 | 3 | use core::sync::atomic::{AtomicU64, Ordering}; 4 | 5 | // Adrift "boolean" 6 | // 7 | // A memory allocator needs to track pages that are empty, partially used, or completely used. 8 | // 9 | // Tracking the latter is really unfortunate, moving them and out of concurrent lists means contention with other 10 | // other threads, for pages that are completely unusable to fulfill allocation requests. 11 | // 12 | // Enters the Anchored/Adrift mechanism! 13 | // 14 | // Rather than keeping track of full pages within a special list, they are instead "cast adrift". 15 | // 16 | // The trick is to catch and anchor them back when the user deallocates memory, for unlike a leaked page, a page that 17 | // is cast adrift is still pointed to: by the user's allocations. 18 | // 19 | // Essentially, thus, the page is cast adrift when full, and caught back when "sufficiently" empty. 20 | // 21 | // To avoid ABA issues with a boolean, a counter is used instead: 22 | // - An even value means "anchored". 23 | // - An odd value means "adrift". 24 | pub(crate) struct Adrift(AtomicU64); 25 | 26 | impl Adrift { 27 | // Creates an anchored instance. 28 | pub(crate) fn new() -> Self { Self(AtomicU64::new(0)) } 29 | 30 | // Checks whether the value is adrift. 31 | // 32 | // If adrift, returns the current value, otherwise returns None. 33 | pub(crate) fn is_adrift(&self) -> Option { 34 | let current = self.load(); 35 | if current % 2 != 0 { Some(current) } else { None } 36 | } 37 | 38 | // Casts the value adrift, incrementing the counter. 39 | // 40 | // Returns the (new) current value. 41 | pub(crate) fn cast_adrift(&self) -> u64 { 42 | let before = self.0.fetch_add(1, Ordering::AcqRel); 43 | debug_assert!(before % 2 == 0, "before: {}", before); 44 | 45 | before + 1 46 | } 47 | 48 | // Attempts to catch the value, returns true if it succeeds. 49 | pub(crate) fn catch(&self, current: u64) -> bool { 50 | debug_assert!(current % 2 != 0); 51 | 52 | self.0.compare_exchange(current, current + 1, Ordering::AcqRel, Ordering::Relaxed).is_ok() 53 | } 54 | 55 | // Introspection for testing purposes. 56 | #[cfg(test)] 57 | pub(crate) fn value(&self) -> u64 { self.0.load(Ordering::Relaxed) } 58 | 59 | // Internal: load. 60 | fn load(&self) -> u64 { self.0.load(Ordering::Acquire) } 61 | } 62 | 63 | impl Default for Adrift { 64 | fn default() -> Self { Self::new() } 65 | } 66 | 67 | #[cfg(test)] 68 | mod tests { 69 | 70 | use std::sync::atomic::{AtomicU64, Ordering}; 71 | 72 | use llmalloc_test::BurstyBuilder; 73 | 74 | use super::*; 75 | 76 | #[test] 77 | fn adrift() { 78 | let adrift = Adrift::default(); 79 | assert_eq!(None, adrift.is_adrift()); 80 | 81 | assert_eq!(1, adrift.cast_adrift()); 82 | assert_eq!(Some(1), adrift.is_adrift()); 83 | 84 | assert!(!adrift.catch(3)); 85 | assert_eq!(Some(1), adrift.is_adrift()); 86 | 87 | assert!(adrift.catch(1)); 88 | assert_eq!(None, adrift.is_adrift()); 89 | 90 | assert_eq!(3, adrift.cast_adrift()); 91 | assert_eq!(Some(3), adrift.is_adrift()); 92 | } 93 | 94 | #[test] 95 | fn adrift_concurrent_catch_fuzzing() { 96 | // This test aims at testing that a single thread can catch an adrift page. 97 | // 98 | // To do so: 99 | // - Adrift is cast adrift. 100 | // - Each thread attempts to catch it, recording whether it did. 101 | // - A check is made that a single thread caught it. 102 | #[derive(Default)] 103 | struct Global { 104 | victim: Adrift, 105 | cast: AtomicU64, 106 | caught: [AtomicU64; 4], 107 | } 108 | 109 | impl Global { 110 | fn reset(&self, index: usize) { 111 | if index == 0 { 112 | if let Some(current) = self.victim.is_adrift() { 113 | assert!(self.victim.catch(current)); 114 | } 115 | self.cast.store(self.victim.cast_adrift(), Ordering::Relaxed); 116 | } 117 | 118 | self.caught[index].store(0, Ordering::Relaxed); 119 | } 120 | 121 | fn verify(&self) { 122 | let cast = self.cast.load(Ordering::Relaxed); 123 | 124 | let caught: Vec<_> = self.caught.iter() 125 | .map(|caught| caught.load(Ordering::Relaxed)) 126 | .filter(|caught| *caught != 0) 127 | .collect(); 128 | 129 | assert_eq!(vec!(cast), caught, "{:?}", self.caught); 130 | } 131 | } 132 | 133 | let mut builder = BurstyBuilder::new(Global::default(), vec!(0usize, 1, 2, 3)); 134 | 135 | // Step 1: reset. 136 | builder.add_simple_step(|| |global: &Global, local: &mut usize| { 137 | global.reset(*local); 138 | }); 139 | 140 | // Step 2: catch, if you can! 141 | builder.add_simple_step(|| |global: &Global, local: &mut usize| { 142 | let cast = global.cast.load(Ordering::Relaxed); 143 | 144 | if global.victim.catch(cast) { 145 | global.caught[*local].store(cast, Ordering::Relaxed); 146 | } 147 | }); 148 | 149 | // Step 3: verify one, and only one, thread caught it. 150 | builder.add_simple_step(|| |global: &Global, _: &mut usize| { 151 | global.verify(); 152 | }); 153 | 154 | builder.launch(100); 155 | } 156 | 157 | } // mod tests 158 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/blocks/block_foreign_list.rs: -------------------------------------------------------------------------------- 1 | //! List of Foreign Blocks of memory. 2 | 3 | use core::{ 4 | hint, 5 | ptr::NonNull, 6 | }; 7 | 8 | use crate::Configuration; 9 | 10 | use super::{BlockForeign, BlockPtr}; 11 | 12 | /// BlockForeignList. 13 | #[derive(Default)] 14 | pub(crate) struct BlockForeignList(BlockPtr); 15 | 16 | impl BlockForeignList { 17 | /// Returns whether the list is empty. 18 | pub(crate) fn is_empty(&self) -> bool { self.0.get().is_none() } 19 | 20 | /// Returns the length of the list. 21 | pub(crate) fn len(&self) -> usize { 22 | self.head() 23 | // Safety: 24 | // - `head` is valid. 25 | .map(|head| unsafe { head.as_ref().length.get() + 1 }) 26 | .unwrap_or(0) 27 | } 28 | 29 | /// Returns the head of the list, it may be None. 30 | pub(crate) fn head(&self) -> Option> { self.0.get() } 31 | 32 | /// Returns true if either the list is empty or it contains a Block within the same page. 33 | pub(crate) fn is_compatible(&self, block: NonNull) -> bool 34 | where 35 | C: Configuration 36 | { 37 | let head = if let Some(head) = self.head() { 38 | head 39 | } else { 40 | return true; 41 | }; 42 | 43 | let head = head.as_ptr() as usize; 44 | let block = block.as_ptr() as usize; 45 | 46 | let page_size = C::LARGE_PAGE_SIZE; 47 | 48 | page_size.round_down(head) == page_size.round_down(block) 49 | } 50 | 51 | /// Prepends the block to the head of the tail-list. 52 | /// 53 | /// Returns the length of the list after the operation. 54 | pub(crate) fn push(&self, block: NonNull) -> usize { 55 | match self.head() { 56 | None => { 57 | // Safety: 58 | // - Bounded lifetime. 59 | unsafe { 60 | block.as_ref().next.set(None); 61 | block.as_ref().length.set(0); 62 | block.as_ref().tail.set(Some(block)); 63 | } 64 | 65 | self.0.set(Some(block)); 66 | 67 | 1 68 | }, 69 | Some(head) => { 70 | // Safety: 71 | // - The pointer is valid. 72 | let length = unsafe { head.as_ref().length.get() }; 73 | let tail = unsafe { head.as_ref().tail.get() }; 74 | 75 | { 76 | // Safety: 77 | // - Bounded lifetime. 78 | let block = unsafe { block.as_ref() }; 79 | 80 | block.next.set(Some(head)); 81 | block.length.set(length + 1); 82 | block.tail.set(tail); 83 | } 84 | 85 | self.0.set(Some(block)); 86 | 87 | // +1 as the length incremented. 88 | // +1 as length is the length of the _tail_. 89 | length + 2 90 | }, 91 | } 92 | } 93 | 94 | // Steals the content of the list. 95 | // 96 | // Returns the head and tail, in this order. 97 | // 98 | // After the call, the list is empty. 99 | // 100 | // # Safety 101 | // 102 | // - Assumes the list is not empty. 103 | pub(crate) unsafe fn steal(&self) -> (NonNull, NonNull) { 104 | debug_assert!(!self.is_empty()); 105 | 106 | let head = force_unwrap(self.0.replace_with_null()); 107 | 108 | // Safety: 109 | // - `head` is not null, as the list is not empty. 110 | let tail = head.as_ref().tail.replace_with_null(); 111 | 112 | (head, force_unwrap(tail)) 113 | } 114 | } 115 | 116 | // 117 | // Implementation 118 | // 119 | 120 | #[inline(always)] 121 | unsafe fn force_unwrap(t: Option) -> T { 122 | match t { 123 | None => { 124 | debug_assert!(false, "Unexpectedly empty Option"); 125 | hint::unreachable_unchecked() 126 | }, 127 | Some(t) => t, 128 | } 129 | } 130 | 131 | #[cfg(test)] 132 | mod tests { 133 | 134 | use crate::PowerOf2; 135 | 136 | use super::*; 137 | use super::super::AlignedArray; 138 | 139 | #[test] 140 | fn block_foreign_list_default() { 141 | let list = BlockForeignList::default(); 142 | 143 | assert!(list.is_empty()); 144 | assert_eq!(0, list.len()); 145 | } 146 | 147 | #[test] 148 | fn block_foreign_list_is_compatible() { 149 | struct C; 150 | 151 | impl Configuration for C { 152 | const LARGE_PAGE_SIZE: PowerOf2 = unsafe { PowerOf2::new_unchecked(1 << 8) }; 153 | const HUGE_PAGE_SIZE: PowerOf2 = unsafe { PowerOf2::new_unchecked(1 << 16) }; 154 | } 155 | 156 | let array = AlignedArray::::default(); 157 | 158 | // The array is aligned on a 256 bytes boundaries, and contains 16-bytes aligned elements. 159 | // Hence the page break is at element 16. 160 | let (a, b, c) = (array.get(14), array.get(15), array.get(16)); 161 | 162 | let list = BlockForeignList::default(); 163 | 164 | assert!(list.is_compatible::(a)); 165 | assert!(list.is_compatible::(b)); 166 | assert!(list.is_compatible::(c)); 167 | 168 | list.push(a); 169 | 170 | assert!(list.is_compatible::(b)); 171 | assert!(!list.is_compatible::(c)); 172 | } 173 | 174 | #[test] 175 | fn block_foreign_list_push_steal() { 176 | let array = AlignedArray::::default(); 177 | let (a, b, c) = (array.get(0), array.get(1), array.get(2)); 178 | 179 | let list = BlockForeignList::default(); 180 | assert_eq!(0, list.len()); 181 | 182 | list.push(c); 183 | assert_eq!(1, list.len()); 184 | 185 | list.push(b); 186 | assert_eq!(2, list.len()); 187 | 188 | list.push(a); 189 | assert_eq!(3, list.len()); 190 | 191 | // Safety: 192 | // - `list` is not empty. 193 | let (head, tail) = unsafe { list.steal() }; 194 | 195 | assert_eq!(a, head); 196 | assert_eq!(c, tail); 197 | } 198 | 199 | } // mod tests 200 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # llmalloc - low-latency memory allocator 2 | 3 | llmalloc is an alternative to malloc for low-latency environments. 4 | 5 | If you are looking for a general-purpose replacement for malloc, prefer jemalloc or tcmalloc. The list of non-goals 6 | should convince you to. 7 | 8 | ## Goals 9 | 10 | The primary goal of this alternative to malloc is low-latency memory allocation and deallocation. 11 | 12 | Supplementary Goals: 13 | 14 | - Low cache footprint: llmalloc is a support library, it strives to minimize its cache footprint, both cache and data. 15 | - Wait-free allocation: llmalloc-core is itself wait-free, guaranteeing that even if all but one thread are blocked, 16 | the lone running thread can still allocate and deallocate on its own. The guarantee partially extends to llmalloc 17 | itself, with the exception of system calls. 18 | 19 | Non-goals: 20 | 21 | - High throughput: llmalloc will always favor latency, as usual it comes at the expense of pure throughput. 22 | - Low-latency system calls: system calls are out of the purview of llmalloc, so instead the API provides ways to 23 | reserve memory ahead of time, so that no further system call is necessary until shutdown. 24 | - Memory efficiency: on x64/linux, llmalloc will reserve memory by increment of 1GB at a time, using Huge Pages if 25 | available. 26 | 27 | Limitations: 28 | 29 | - Memory frugality: llmalloc cannot, by design, relinquish any allocated page of memory back to the OS until 30 | shutdown, it does not keep track of the necessary information. 31 | - Metrics: llmalloc does not provide any metric on actual memory usage, it does not keep track of such information. 32 | - Portability: llmalloc is only available on x64/linux platforms at the moment. 33 | 34 | While the limitations could, potentially, be lifted, there is currently no intent to do so. 35 | 36 | ## Structure of the repository 37 | 38 | This repository contains 3 libraries: 39 | 40 | - llmalloc-core: the unopinionated core library, which contains the building bricks. 41 | - llmalloc: an opinionated implementation. 42 | - llmalloc-c: C bindings for llmalloc. 43 | 44 | ## Maturity 45 | 46 | llmalloc is in _alpha_ state; of note: 47 | 48 | - llmalloc-core: 49 | - Audit: not audited. 50 | - Fuzzing: fuzzed. 51 | - Testing: good single-thread and multi-thread unit-test support. 52 | 53 | - llmalloc: 54 | - Audit: not audited. 55 | - Benchmarks: benchmarked, with performance on par or better than the system allocator. 56 | - Fuzzing: fuzzed. 57 | - Testing: one single-threaded integration test. 58 | 59 | The library is ready to be trialed in realistic setups, but is not mature enough to drop in in a production application 60 | without a qualification phase. 61 | 62 | ## Design Sketch 63 | 64 | A quick overview of the design of the library, to pique your interest. 65 | 66 | For a more extensive discussion of design decisions, see [doc/Design.md](doc/Design.md). 67 | 68 | ### Concepts 69 | 70 | llmalloc is organized around the concept of 3 allocation categories and 2 page sizes: 71 | 72 | - Large Pages: pages from which Normal allocations are served. 73 | - Huge Pages: pages from which Large allocations are served. 74 | 75 | Huge Pages are provided by Huge allocations, and Large Pages by Large allocations. 76 | 77 | ### Key Pieces 78 | 79 | At the root of llmalloc-core lies the _Domain_. Simply said, the _Domain_ defines the boundaries of memory, all 80 | allocations performed within a _Domain_ can be deallocated only with the same _Domain_. The _Domain_ is host to an 81 | instance of the `Platform` trait: a trait which defines how to allocate and deallocate Huge allocations, and therefore 82 | Huge Pages. 83 | 84 | A _Domain_ may have multiple _Sockets_. As the name implies, it is expected that each socket on the motherboard feature 85 | one _Socket_. Non-Uniform Memory Architecture places a tax on inter-socket cache-coherence traffic, and therefore it 86 | is best not to write to the same piece of memory from multiple sockets. Each _Socket_ will therefore keep track of its 87 | own set of Huge Pages, from which all its allocations will come from, as well as provide a cache of Large Pages. 88 | 89 | And finally, a _Socket_ can allocate multiple _Threads_. As the name implies, it is expected that each software thread 90 | feature one _Thread_. Each _Thread_ will keep one Large Page for each class size of Normal allocations that it may 91 | serve, in order to provide thread-local uncontended allocation and deallocation -- most of the time. 92 | 93 | ### Implementation of Deallocation 94 | 95 | Unfortunately, the interface of the C function `free`, and the C++ operator `delete`, are not friendly: they hide 96 | one critical piece of information, the size of the memory area to be returned to the memory allocator. 97 | 98 | llmalloc borrows a trick from [mimalloc](https://github.com/microsoft/mimalloc), and encodes the category of the 99 | allocation (Normal, Large, or Huge) in the _alignment_ of the pointer. This places some constraints on the allocation 100 | of the Huge Pages, and enables low-latency deallocation. 101 | 102 | On x64/linux, for example: 103 | 104 | - Normal allocations have an alignment strictly less than 2MB -- the alignment of Large Pages. 105 | - Large allocations have an alignment greater than or equal to 2MB, and strictly less than 1GB -- the alignment of 106 | Huge Pages. 107 | - Huge allocations have an alignment greater than or equal to 1GB. 108 | 109 | And while the exact alignments may vary from platform to platform, the principles remain the same on all. As a result 110 | finding the page to which an allocation belongs is as simple as masking its pointer lower bits by the appropriate mask, 111 | which is deduced from the alignment. 112 | 113 | ## Acknowledgements 114 | 115 | The spark came from reading about the design of [mimalloc](https://github.com/microsoft/mimalloc), in particular two 116 | key ideas resonated within me: 117 | 118 | - Alignment is information: the idea of encoding the category of an allocation (Normal, Large, or Huge) within the 119 | alignment of the pointer was an eye opener; it vastly simplifies unsized deallocation. 120 | - Cache footprint: the idea that a support library should vie for as low a cache footprint as possible, both an 121 | instruction cache footprint, and a data cache footprint, was also a light bulb moment. A key difference between 122 | micro-benchmarks and real-life applications is that in micro-benchmarks the library has all the cache to itself. 123 | -------------------------------------------------------------------------------- /llmalloc-core/src/utils/power_of_2.rs: -------------------------------------------------------------------------------- 1 | //! An integer guaranteed to be a PowerOf2. 2 | 3 | use core::{mem, num, ops}; 4 | 5 | /// PowerOf2 6 | /// 7 | /// An integral guaranteed to be non-zero and a power of 2. 8 | #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd)] 9 | pub struct PowerOf2(num::NonZeroUsize); 10 | 11 | impl PowerOf2 { 12 | /// 1 as a PowerOf2 instance. 13 | // Safety: 14 | // - 1 is a power of 2. 15 | pub const ONE: PowerOf2 = unsafe { PowerOf2::new_unchecked(1) }; 16 | 17 | /// Creates a new instance of PowerOf2. 18 | /// 19 | /// Or nothing if the value is not a power of 2. 20 | pub fn new(value: usize) -> Option { 21 | if value.count_ones() == 1 { 22 | // Safety: 23 | // - Value is a power of 2, as per the if check. 24 | Some(unsafe { PowerOf2::new_unchecked(value) }) 25 | } else { 26 | None 27 | } 28 | } 29 | 30 | /// Creates a new instance of PowerOf2. 31 | /// 32 | /// # Safety 33 | /// 34 | /// Assumes that the value is a power of 2. 35 | pub const unsafe fn new_unchecked(value: usize) -> PowerOf2 { 36 | // Safety: 37 | // - A power of 2 cannot be 0. 38 | PowerOf2(num::NonZeroUsize::new_unchecked(value)) 39 | } 40 | 41 | /// Creates a PowerOf2 matching the alignment of a type. 42 | pub const fn align_of() -> PowerOf2 { 43 | // Safety: 44 | // - Alignment is always a power of 2, and never 0. 45 | unsafe { PowerOf2::new_unchecked(mem::align_of::()) } 46 | } 47 | 48 | /// Returns the inner value. 49 | pub const fn value(&self) -> usize { self.0.get() } 50 | 51 | /// Rounds the value up to the nearest higher multiple of `self`. 52 | pub const fn round_up(&self, n: usize) -> usize { 53 | let mask = self.mask(); 54 | 55 | (n + mask) & !mask 56 | } 57 | 58 | /// Rounds the value down to the nearest lower multiple of `self`. 59 | pub const fn round_down(&self, n: usize) -> usize { n & !self.mask() } 60 | 61 | const fn bit_index(&self) -> usize { self.value().trailing_zeros() as usize } 62 | 63 | const fn mask(&self) -> usize { self.value() - 1 } 64 | } 65 | 66 | impl ops::Div for PowerOf2 { 67 | // Cannot be PowerOf2, because it could yield 0. 68 | type Output = usize; 69 | 70 | fn div(self, rhs: PowerOf2) -> usize { self.value() / rhs } 71 | } 72 | 73 | impl ops::Div for usize { 74 | type Output = usize; 75 | 76 | #[allow(clippy::suspicious_arithmetic_impl)] 77 | fn div(self, rhs: PowerOf2) -> usize { self >> rhs.bit_index() } 78 | } 79 | 80 | impl ops::Mul for PowerOf2 { 81 | type Output = PowerOf2; 82 | 83 | fn mul(self, rhs: PowerOf2) -> PowerOf2 { unsafe { PowerOf2::new_unchecked(self.value() * rhs) } } 84 | } 85 | 86 | impl ops::Mul for PowerOf2 { 87 | type Output = usize; 88 | 89 | #[allow(clippy::suspicious_arithmetic_impl)] 90 | fn mul(self, rhs: usize) -> usize { rhs << self.bit_index() } 91 | } 92 | 93 | impl ops::Mul for usize { 94 | type Output = usize; 95 | 96 | #[allow(clippy::suspicious_arithmetic_impl)] 97 | fn mul(self, rhs: PowerOf2) -> usize { self << rhs.bit_index() } 98 | } 99 | 100 | impl ops::Rem for usize { 101 | type Output = usize; 102 | 103 | #[allow(clippy::suspicious_arithmetic_impl)] 104 | fn rem(self, rhs: PowerOf2) -> usize { self & rhs.mask() } 105 | } 106 | 107 | #[cfg(test)] 108 | mod tests { 109 | 110 | use super::*; 111 | 112 | #[test] 113 | fn power_of_2_new() { 114 | fn new(value: usize) -> Option { 115 | PowerOf2::new(value).map(|p| p.value()) 116 | } 117 | 118 | assert_eq!(None, new(0)); 119 | assert_eq!(Some(1), new(1)); 120 | assert_eq!(Some(2), new(2)); 121 | assert_eq!(None, new(3)); 122 | assert_eq!(Some(4), new(4)); 123 | assert_eq!(None, new(5)); 124 | assert_eq!(None, new(6)); 125 | assert_eq!(None, new(7)); 126 | assert_eq!(Some(8), new(8)); 127 | assert_eq!(None, new(9)); 128 | } 129 | 130 | #[test] 131 | fn power_of_2_div() { 132 | fn div(pow2: usize, n: usize) -> usize { 133 | n / PowerOf2::new(pow2).expect("Power of 2") 134 | } 135 | 136 | assert_eq!(0, div(1, 0)); 137 | assert_eq!(1, div(1, 1)); 138 | assert_eq!(2, div(1, 2)); 139 | assert_eq!(3, div(1, 3)); 140 | 141 | assert_eq!(0, div(2, 1)); 142 | assert_eq!(1, div(2, 2)); 143 | assert_eq!(1, div(2, 3)); 144 | assert_eq!(2, div(2, 4)); 145 | 146 | assert_eq!(0, div(4, 3)); 147 | assert_eq!(1, div(4, 4)); 148 | assert_eq!(1, div(4, 7)); 149 | assert_eq!(2, div(4, 8)); 150 | } 151 | 152 | #[test] 153 | fn power_of_2_mul() { 154 | fn mul(pow2: usize, n: usize) -> usize { 155 | n * PowerOf2::new(pow2).expect("Power of 2") 156 | } 157 | 158 | assert_eq!(0, mul(1, 0)); 159 | assert_eq!(1, mul(1, 1)); 160 | assert_eq!(2, mul(1, 2)); 161 | assert_eq!(3, mul(1, 3)); 162 | 163 | assert_eq!(2, mul(2, 1)); 164 | assert_eq!(4, mul(2, 2)); 165 | assert_eq!(6, mul(2, 3)); 166 | assert_eq!(8, mul(2, 4)); 167 | 168 | assert_eq!(12, mul(4, 3)); 169 | assert_eq!(16, mul(4, 4)); 170 | assert_eq!(28, mul(4, 7)); 171 | assert_eq!(32, mul(4, 8)); 172 | } 173 | 174 | #[test] 175 | fn power_of_2_rem() { 176 | fn rem(pow2: usize, n: usize) -> usize { 177 | n % PowerOf2::new(pow2).expect("Power of 2") 178 | } 179 | 180 | assert_eq!(0, rem(1, 0)); 181 | assert_eq!(0, rem(1, 1)); 182 | assert_eq!(0, rem(1, 2)); 183 | assert_eq!(0, rem(1, 3)); 184 | 185 | assert_eq!(0, rem(2, 0)); 186 | assert_eq!(1, rem(2, 1)); 187 | assert_eq!(0, rem(2, 2)); 188 | assert_eq!(1, rem(2, 3)); 189 | 190 | assert_eq!(0, rem(4, 0)); 191 | assert_eq!(1, rem(4, 1)); 192 | assert_eq!(2, rem(4, 2)); 193 | assert_eq!(3, rem(4, 3)); 194 | assert_eq!(0, rem(4, 4)); 195 | assert_eq!(1, rem(4, 5)); 196 | assert_eq!(2, rem(4, 6)); 197 | assert_eq!(3, rem(4, 7)); 198 | assert_eq!(0, rem(4, 8)); 199 | } 200 | 201 | #[test] 202 | fn power_of_2_round_up() { 203 | fn round_up(pow2: usize, n: usize) -> usize { 204 | PowerOf2::new(pow2).expect("Power of 2").round_up(n) 205 | } 206 | 207 | assert_eq!(0, round_up(1, 0)); 208 | assert_eq!(1, round_up(1, 1)); 209 | assert_eq!(2, round_up(1, 2)); 210 | assert_eq!(3, round_up(1, 3)); 211 | 212 | assert_eq!(0, round_up(2, 0)); 213 | assert_eq!(2, round_up(2, 1)); 214 | assert_eq!(2, round_up(2, 2)); 215 | assert_eq!(4, round_up(2, 3)); 216 | assert_eq!(4, round_up(2, 4)); 217 | assert_eq!(6, round_up(2, 5)); 218 | 219 | assert_eq!(0, round_up(4, 0)); 220 | assert_eq!(4, round_up(4, 1)); 221 | assert_eq!(4, round_up(4, 4)); 222 | assert_eq!(8, round_up(4, 5)); 223 | assert_eq!(8, round_up(4, 8)); 224 | assert_eq!(12, round_up(4, 9)); 225 | } 226 | 227 | #[test] 228 | fn power_of_2_round_down() { 229 | fn round_down(pow2: usize, n: usize) -> usize { 230 | PowerOf2::new(pow2).expect("Power of 2").round_down(n) 231 | } 232 | 233 | assert_eq!(0, round_down(1, 0)); 234 | assert_eq!(1, round_down(1, 1)); 235 | assert_eq!(2, round_down(1, 2)); 236 | assert_eq!(3, round_down(1, 3)); 237 | 238 | assert_eq!(0, round_down(2, 1)); 239 | assert_eq!(2, round_down(2, 2)); 240 | assert_eq!(2, round_down(2, 3)); 241 | assert_eq!(4, round_down(2, 4)); 242 | assert_eq!(4, round_down(2, 5)); 243 | assert_eq!(6, round_down(2, 6)); 244 | 245 | assert_eq!(0, round_down(4, 3)); 246 | assert_eq!(4, round_down(4, 4)); 247 | assert_eq!(4, round_down(4, 7)); 248 | assert_eq!(8, round_down(4, 8)); 249 | assert_eq!(8, round_down(4, 11)); 250 | assert_eq!(12, round_down(4, 12)); 251 | } 252 | 253 | } 254 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/large_page/local.rs: -------------------------------------------------------------------------------- 1 | //! Local data, only accessible from the local thread. 2 | 3 | use core::{ 4 | cell::Cell, 5 | ptr::NonNull, 6 | }; 7 | 8 | use crate::{ 9 | PowerOf2, 10 | internals::blocks::{BlockForeignList, BlockLocal, BlockLocalStack}, 11 | utils, 12 | }; 13 | 14 | // Local data. Only accessible from the local thread. 15 | #[repr(align(128))] 16 | pub(crate) struct Local { 17 | // Stack of free blocks. 18 | next: BlockLocalStack, 19 | // Pointer to the beginning of the uncarved area of the page. 20 | // 21 | // Linking all the cells together when creating the page would take too long, so instead only the first block is 22 | // prepared, and the `watermark` and `end` are initialized to denote the area of the page which can freely be 23 | // carved into further cells. 24 | watermark: Cell>, 25 | // Pointer to the end of the page; when `watermark == end`, the entire page has been carved. 26 | // 27 | // When the entire page has been carved, acquiring new cells from the page is only possible through `foreign.freed`. 28 | end: NonNull, 29 | // Size, in bytes, of the cells. 30 | block_size: usize, 31 | } 32 | 33 | impl Local { 34 | /// Creates a new instance of `Local`. 35 | /// 36 | /// # Safety 37 | /// 38 | /// - `begin` and `end` are assumed to be correctly aligned and sized for a `BlockForeign` pointer. 39 | /// - `end - begin` is assumed to be a multiple of `block_size`. 40 | pub(crate) unsafe fn new(block_size: usize, begin: NonNull, end: NonNull) -> Self { 41 | debug_assert!(block_size >= 1); 42 | debug_assert!((end.as_ptr() as usize - begin.as_ptr() as usize) % block_size == 0, 43 | "block_size: {}, begin: {:x}, end: {:x}", block_size, begin.as_ptr() as usize, end.as_ptr() as usize); 44 | 45 | let next = BlockLocalStack::from_raw(begin); 46 | let watermark = Cell::new(NonNull::new_unchecked(begin.as_ptr().add(block_size))); 47 | 48 | Self { next, watermark, end, block_size, } 49 | } 50 | 51 | /// Allocates one cell from the page, if any. 52 | /// 53 | /// Returns a null pointer is no cell is available. 54 | pub(crate) fn allocate(&self) -> Option> { 55 | // Fast Path. 56 | if let Some(block) = self.next.pop() { 57 | return Some(block.cast()); 58 | } 59 | 60 | // Cruise path. 61 | if self.watermark.get() == self.end { 62 | return None; 63 | } 64 | 65 | // Expansion path. 66 | let result = self.watermark.get(); 67 | 68 | // Safety: 69 | // - `self.block_size` matches the size of the cells. 70 | // - `self.watermark` is still within bounds. 71 | unsafe { self.watermark.set(NonNull::new_unchecked(result.as_ptr().add(self.block_size))) }; 72 | 73 | Some(result.cast()) 74 | } 75 | 76 | /// Deallocates one cell from the page. 77 | /// 78 | /// # Safety 79 | /// 80 | /// - Assumes that `ptr` is sufficiently sized. 81 | /// - Assumes that `ptr` is sufficiently aligned. 82 | pub(crate) unsafe fn deallocate(&self, ptr: NonNull) { 83 | debug_assert!(utils::is_sufficiently_aligned_for(ptr, PowerOf2::align_of::())); 84 | 85 | // Safety: 86 | // - `ptr` is assumed to be sufficiently sized. 87 | // - `ptr` is assumed to be sufficiently aligned. 88 | let block = ptr.cast(); 89 | 90 | self.next.push(block); 91 | } 92 | 93 | /// Extends the local list from a foreign list. 94 | /// 95 | /// # Safety 96 | /// 97 | /// - Assumes that the access to the linked cells, is exclusive. 98 | pub(crate) unsafe fn extend(&self, list: &BlockForeignList) { 99 | debug_assert!(!list.is_empty()); 100 | 101 | // Safety: 102 | // - It is assumed that access to the cell, and all linked cells, is exclusive. 103 | self.next.extend(list); 104 | } 105 | 106 | /// Refills the local list from a foreign list. 107 | /// 108 | /// # Safety 109 | /// 110 | /// - Assumes that access to the cell, and all linked cells, is exclusive. 111 | pub(crate) unsafe fn refill(&self, list: NonNull) { 112 | // Safety: 113 | // - It is assumed that access to the cell, and all linked cells, is exclusive. 114 | self.next.refill(list); 115 | } 116 | 117 | /// Returns the size of the blocks. 118 | #[cfg(test)] 119 | pub(crate) fn block_size(&self) -> usize { self.block_size } 120 | 121 | /// Returns the end of the watermark area. 122 | #[cfg(test)] 123 | pub(crate) fn end(&self) -> NonNull { self.end } 124 | } 125 | 126 | #[cfg(test)] 127 | mod tests { 128 | 129 | use super::*; 130 | use super::super::test::{BlockStore, BLOCK_SIZE}; 131 | 132 | #[test] 133 | fn local_new() { 134 | // This test actually tests `block_store.create_local` more than anything. 135 | // Since further tests will depend on it correctly initializing `Local`, it is better to validate it early. 136 | let block_store = BlockStore::default(); 137 | let end_store = block_store.end(); 138 | 139 | { 140 | let local = unsafe { block_store.create_local(BLOCK_SIZE * 2) }; 141 | assert_eq!(block_store.get(0), local.next.peek().unwrap().cast()); 142 | assert_eq!(block_store.get(8), local.watermark.get()); 143 | assert_eq!(end_store, local.end); 144 | } 145 | 146 | { 147 | let local = unsafe { block_store.create_local(BLOCK_SIZE * 2 + BLOCK_SIZE / 2) }; 148 | assert_eq!(block_store.get(6), local.next.peek().unwrap().cast()); 149 | assert_eq!(block_store.get(16), local.watermark.get()); 150 | assert_eq!(end_store, local.end); 151 | } 152 | } 153 | 154 | #[test] 155 | fn local_allocate_expansion() { 156 | let block_store = BlockStore::default(); 157 | let local = unsafe { block_store.create_local(BLOCK_SIZE) }; 158 | 159 | // Bump watermark until it is no longer possible. 160 | for i in 0..64 { 161 | assert_eq!(block_store.get(4 * i), local.allocate().unwrap()); 162 | } 163 | 164 | assert_eq!(local.end, local.watermark.get()); 165 | 166 | assert_eq!(None, local.allocate()); 167 | } 168 | 169 | #[test] 170 | fn local_allocate_deallocate_ping_pong() { 171 | let block_store = BlockStore::default(); 172 | let local = unsafe { block_store.create_local(BLOCK_SIZE) }; 173 | 174 | let ptr = local.allocate(); 175 | 176 | for _ in 0..10 { 177 | unsafe { local.deallocate(ptr.unwrap()) }; 178 | assert_eq!(ptr, local.allocate()); 179 | } 180 | 181 | assert_eq!(block_store.get(4), local.watermark.get()); 182 | } 183 | 184 | #[test] 185 | fn local_extend() { 186 | let block_store = BlockStore::default(); 187 | let local = unsafe { block_store.create_local(BLOCK_SIZE) }; 188 | 189 | // Allocate all. 190 | while let Some(_) = local.allocate() {} 191 | 192 | let foreign_list = unsafe { block_store.create_foreign_list(&local, 3..7) }; 193 | 194 | unsafe { local.extend(&foreign_list) }; 195 | assert!(foreign_list.is_empty()); 196 | 197 | for i in 3..7 { 198 | assert_eq!(block_store.get(4 * i), local.allocate().unwrap()); 199 | } 200 | } 201 | 202 | #[test] 203 | fn local_refill() { 204 | let block_store = BlockStore::default(); 205 | let local = unsafe { block_store.create_local(BLOCK_SIZE) }; 206 | 207 | // Allocate all. 208 | while let Some(_) = local.allocate() {} 209 | 210 | let foreign = unsafe { block_store.create_foreign_stack(&local, 3..7) }; 211 | 212 | unsafe { local.refill(BlockLocal::from_atomic(foreign)) }; 213 | 214 | for i in 3..7 { 215 | assert_eq!(block_store.get(4 * i), local.allocate().unwrap()); 216 | } 217 | } 218 | 219 | } // mod tests 220 | -------------------------------------------------------------------------------- /llmalloc-core/src/api/configuration.rs: -------------------------------------------------------------------------------- 1 | //! The configuration of llmalloc-core. 2 | //! 3 | //! A single Configuration instance should be shared between all related SocketLocals and ThreadLocals. 4 | //! 5 | //! llmalloc features 3 allocation categories: 6 | //! 7 | //! - Normal: fulfilled using slabs, cached in ThreadLocal, and lazily refilled. 8 | //! - Large: fulfilled using bitmaps, stored in SocketLocal, and eagerly refilled. 9 | //! - Huge: fullfilled directly by the Platform trait. 10 | //! 11 | //! The Configuration instance allows adjusting the thresholds of those categories to better match the underlying 12 | //! platform native page sizes. 13 | 14 | use core::{ 15 | num, 16 | ptr::NonNull, 17 | }; 18 | 19 | use super::{AllocationSize, Category, ClassSize, Layout, PowerOf2}; 20 | 21 | /// Configuration 22 | /// 23 | /// The Configuration instance allows adjusting the thresholds of the allocation categories. 24 | pub trait Configuration { 25 | /// The size of Large Pages, from which Normal allocations are produced. 26 | /// 27 | /// The minimum this size can be is 4096, as the header of 512 bytes cannot exceed 1/8th of the page. 28 | const LARGE_PAGE_SIZE: PowerOf2; 29 | 30 | /// The size of Huge Pages, from which Large allocations are produced. 31 | /// 32 | /// Allocations from the Platform are always requested as multiple of this page size. 33 | const HUGE_PAGE_SIZE: PowerOf2; 34 | } 35 | 36 | /// Properties 37 | /// 38 | /// Properties of a given Configuration. 39 | /// 40 | /// Work-around for the inability to implement static methods directly on a trait. 41 | pub struct Properties(C); 42 | 43 | impl Properties 44 | where 45 | C: Configuration 46 | { 47 | /// Returns the minimum allocation size. 48 | pub fn minimum_allocation_size() -> AllocationSize { ClassSize::minimum_allocation_size() } 49 | 50 | /// Returns the threshold of Normal allocations. 51 | /// 52 | /// Allocations for a size less than or equal to the threshold are of the Normal category. 53 | pub fn normal_threshold() -> AllocationSize { AllocationSize::new(C::LARGE_PAGE_SIZE.value() / 16 * 7) } 54 | 55 | /// Returns the threshold of Large allocations. 56 | /// 57 | /// Allocations for a size less than or equal to the threshold, yet too large to be of the Normal category, are of 58 | /// the Large category. 59 | pub fn large_threshold() -> AllocationSize { 60 | AllocationSize::new(C::HUGE_PAGE_SIZE.value() - C::LARGE_PAGE_SIZE.value()) 61 | } 62 | 63 | /// Returns the category of a pointer, based on its alignment. 64 | pub fn category_of_pointer(ptr: NonNull) -> Category { 65 | let ptr = ptr.as_ptr() as usize; 66 | 67 | if ptr % C::LARGE_PAGE_SIZE != 0 { 68 | Category::Normal 69 | } else if ptr % C::HUGE_PAGE_SIZE != 0 { 70 | Category::Large 71 | } else { 72 | Category::Huge 73 | } 74 | } 75 | 76 | /// Returns the category of an allocation, based on its size. 77 | pub fn category_of_size(size: usize) -> Category { 78 | debug_assert!(size > 0); 79 | 80 | if size <= Self::normal_threshold().value() { 81 | Category::Normal 82 | } else if size <= Self::large_threshold().value() { 83 | Category::Large 84 | } else { 85 | Category::Huge 86 | } 87 | } 88 | 89 | /// Returns the class size of an allocation, if Normal. 90 | pub fn class_size_of_size(size: usize) -> Option { 91 | if size == 0 || size > Self::normal_threshold().value() { 92 | return None; 93 | } 94 | 95 | // Safety: 96 | // - Not 0. 97 | let size = unsafe { num::NonZeroUsize::new_unchecked(size) }; 98 | 99 | Some(ClassSize::from_size(size)) 100 | } 101 | 102 | /// Returns the allocation size and alignment of an allocation, based on its size. 103 | pub fn layout_of_size(size: usize) -> Layout { 104 | match Self::category_of_size(size) { 105 | Category::Normal => Self::class_size_of_size(size).expect("Normal").layout(), 106 | Category::Large => Self::page_layout(C::LARGE_PAGE_SIZE, size), 107 | Category::Huge => Self::page_layout(C::HUGE_PAGE_SIZE, size), 108 | } 109 | } 110 | 111 | fn page_layout(page_size: PowerOf2, size: usize) -> Layout { 112 | let size = page_size.round_up(size); 113 | let align = page_size.value(); 114 | 115 | // Safety: 116 | // - `size` is a multiple of `align`. 117 | // - `align` is a power of 2. 118 | unsafe { Layout::from_size_align_unchecked(size, align) } 119 | } 120 | } 121 | 122 | #[cfg(test)] 123 | mod tests { 124 | 125 | use super::*; 126 | 127 | struct TestConfiguration; 128 | 129 | impl Configuration for TestConfiguration { 130 | const LARGE_PAGE_SIZE: PowerOf2 = unsafe { PowerOf2::new_unchecked(1 << 11) }; 131 | const HUGE_PAGE_SIZE: PowerOf2 = unsafe { PowerOf2::new_unchecked(1 << 20) }; 132 | } 133 | 134 | type TestProperties = Properties; 135 | 136 | #[test] 137 | fn properties_normal_threshold() { 138 | fn threshold() -> usize { 139 | TestProperties::normal_threshold().value() 140 | } 141 | 142 | assert_eq!(896, threshold()); 143 | } 144 | 145 | #[test] 146 | fn properties_large_threshold() { 147 | fn threshold() -> usize { 148 | TestProperties::large_threshold().value() 149 | } 150 | 151 | assert_eq!(1_046_528, threshold()); 152 | } 153 | 154 | #[test] 155 | fn properties_category_of_pointer() { 156 | fn category(ptr: usize) -> Category { 157 | TestProperties::category_of_pointer(NonNull::new(ptr as *mut u8).unwrap()) 158 | } 159 | 160 | assert_eq!(Category::Normal, category(1 << 0)); 161 | assert_eq!(Category::Normal, category(1 << 1)); 162 | assert_eq!(Category::Normal, category(1 << 2)); 163 | assert_eq!(Category::Normal, category(1 << 9)); 164 | assert_eq!(Category::Normal, category(1 << 10)); 165 | 166 | assert_eq!(Category::Large, category(1 << 11)); 167 | assert_eq!(Category::Large, category(1 << 12)); 168 | assert_eq!(Category::Large, category(1 << 18)); 169 | assert_eq!(Category::Large, category(1 << 19)); 170 | 171 | assert_eq!(Category::Huge, category(1 << 20)); 172 | assert_eq!(Category::Huge, category(1 << 21)); 173 | } 174 | 175 | #[test] 176 | fn properties_category_of_size() { 177 | fn category(size: usize) -> Category { 178 | TestProperties::category_of_size(size) 179 | } 180 | 181 | assert_eq!(Category::Normal, category(1)); 182 | assert_eq!(Category::Normal, category(2)); 183 | assert_eq!(Category::Normal, category(895)); 184 | assert_eq!(Category::Normal, category(896)); 185 | 186 | assert_eq!(Category::Large, category(897)); 187 | assert_eq!(Category::Large, category(898)); 188 | assert_eq!(Category::Large, category(1_046_527)); 189 | assert_eq!(Category::Large, category(1_046_528)); 190 | 191 | assert_eq!(Category::Huge, category(1_046_529)); 192 | assert_eq!(Category::Huge, category(1 << 20)); 193 | assert_eq!(Category::Huge, category(1 << 21)); 194 | assert_eq!(Category::Huge, category(1 << 22)); 195 | } 196 | 197 | #[test] 198 | fn properties_class_size_of_size() { 199 | fn class_size(size: usize) -> Option { 200 | TestProperties::class_size_of_size(size).map(|c| c.value()) 201 | } 202 | 203 | let minimum_allocation_size = TestProperties::minimum_allocation_size().value(); 204 | 205 | assert_eq!(None, class_size(0)); 206 | 207 | assert_eq!(Some(0), class_size(1)); 208 | assert_eq!(Some(0), class_size(minimum_allocation_size)); 209 | assert_eq!(Some(1), class_size(minimum_allocation_size + 1)); 210 | assert_eq!(Some(19), class_size(895)); 211 | assert_eq!(Some(19), class_size(896)); 212 | 213 | assert_eq!(None, class_size(897)); 214 | assert_eq!(None, class_size(898)); 215 | } 216 | 217 | #[test] 218 | fn properties_layout_of_size() { 219 | fn layout(size: usize) -> (usize, usize) { 220 | let layout = TestProperties::layout_of_size(size); 221 | (layout.size(), layout.align()) 222 | } 223 | 224 | let min = TestProperties::minimum_allocation_size().value(); 225 | 226 | assert_eq!((min, min), layout(1)); 227 | assert_eq!((min, min), layout(2)); 228 | assert_eq!((768, 256), layout(768)); 229 | assert_eq!((896, 128), layout(896)); 230 | 231 | assert_eq!((2048, 2048), layout(897)); 232 | assert_eq!((2048, 2048), layout(2048)); 233 | assert_eq!((4096, 2048), layout(2049)); 234 | assert_eq!((4096, 2048), layout(4096)); 235 | assert_eq!((1_046_528, 2048), layout(1_046_527)); 236 | assert_eq!((1_046_528, 2048), layout(1_046_528)); 237 | 238 | assert_eq!((1 << 20, 1 << 20), layout(1_046_529)); 239 | assert_eq!((1 << 20, 1 << 20), layout(1 << 20)); 240 | assert_eq!((1 << 21, 1 << 20), layout(1 << 21)); 241 | assert_eq!((1 << 22, 1 << 20), layout(1 << 22)); 242 | } 243 | 244 | } 245 | -------------------------------------------------------------------------------- /llmalloc-core/src/api/socket.rs: -------------------------------------------------------------------------------- 1 | //! Socket Handle. 2 | //! 3 | //! All Thread Handles instances sharing a given Socket Handle instance will exchange memory between themselves. 4 | //! 5 | //! For a simple allocator, a simple Socket Handle instance is sufficient. 6 | //! 7 | //! Using multiple Socket Handle instances allows: 8 | //! 9 | //! - Better performance when a given Socket Handle instance is local to a NUMA node. 10 | //! - Less contention, by reducing the number of ThreadLocal instances contending over it. 11 | //! 12 | //! The name comes from the socket in which a CPU is plugged in, as a recommendation to use one instance of Socket Handle 13 | //! for each socket. 14 | 15 | use core::{ 16 | alloc::Layout, 17 | ptr::{self, NonNull}, 18 | sync::atomic::{AtomicPtr, Ordering}, 19 | }; 20 | 21 | use crate::{Configuration, DomainHandle, Platform, ThreadHandle}; 22 | use crate::internals::socket_local::SocketLocal; 23 | 24 | /// A handle to socket-local memory structures. 25 | /// 26 | /// The socket-local memory structures are thread-safe, but the handle itself is not. For a thread-safe handle, please 27 | /// see `AtomicSocketHandle`. 28 | pub struct SocketHandle<'a, C, P>(NonNull>); 29 | 30 | impl<'a, C, P> SocketHandle<'a, C, P> 31 | where 32 | C: Configuration, 33 | P: Platform, 34 | { 35 | /// Creates a new instance of SocketHandle. 36 | pub fn new(domain: &'a DomainHandle) -> Option { 37 | SocketLocal::bootstrap(domain.as_raw()).map(SocketHandle) 38 | } 39 | 40 | /// Returns whether the layout is valid, or not, for use with `SocketLocal`. 41 | pub fn is_valid_layout(layout: Layout) -> bool { SocketLocal::::is_valid_layout(layout) } 42 | 43 | /// Attempts to acquire a `ThreadHandle` from within the buffer area of the first HugePage. 44 | /// 45 | /// Returns a valid pointer to `ThreadHandle` if successful, and None otherwise. 46 | pub fn acquire_thread_handle(&self) -> Option> { 47 | // Safety: 48 | // - Local lifetime. 49 | let socket_local = unsafe { self.0.as_ref() }; 50 | 51 | socket_local.acquire_thread_local().map(ThreadHandle::new) 52 | } 53 | 54 | /// Releases a `ThreadHandle`. 55 | /// 56 | /// # Safety 57 | /// 58 | /// - Assumes that the `ThreadHandle` came from `self`. 59 | pub unsafe fn release_thread_handle(&self, handle: ThreadHandle) { 60 | // Safety: 61 | // - Local lifetime. 62 | let socket_local = self.0.as_ref(); 63 | 64 | // Safety: 65 | // - `handle` is assumed to come from `socket`. 66 | socket_local.release_thread_local(handle.into_raw()); 67 | } 68 | 69 | /// Attempts to ensure that at least `target` `HugePage` are allocated on the socket. 70 | /// 71 | /// Returns the minimum of the currently allocated number of pages and `target`. 72 | /// 73 | /// Failure to meet the `target` may occur if: 74 | /// 75 | /// - The maximum number of `HugePage` a `socket` may contain has been reached. 76 | /// - The underlying `Platform` is failing to allocate more `HugePage`. 77 | pub fn reserve(&self, target: usize) -> usize { 78 | // Safety: 79 | // - Local lifetime. 80 | let socket_local = unsafe { self.0.as_ref() }; 81 | 82 | socket_local.reserve(target) 83 | } 84 | 85 | /// Deallocates all HugePages allocated by the socket. 86 | /// 87 | /// This may involve deallocating the memory used by the socket itself, after which it can no longer be used. 88 | /// 89 | /// # Safety 90 | /// 91 | /// - Assumes that none of the memory allocated by the socket is still in use, with the possible exception of the 92 | /// memory used by `self`. 93 | pub unsafe fn close(self) { 94 | // Safety: 95 | // - Local lifetime. 96 | let socket_local = self.0.as_ref(); 97 | 98 | // Safety: 99 | // - Assumes that none of the memory allocated by the socket is still in use. 100 | // - Assumes that `P` can deallocate itself. 101 | socket_local.close() 102 | } 103 | 104 | /// Allocates a fresh block of memory as per the specified layout. 105 | /// 106 | /// May return a null pointer if the allocation request cannot be satisfied. 107 | /// 108 | /// # Safety 109 | /// 110 | /// The caller may assume that if the returned pointer is not null then: 111 | /// - The number of usable bytes is _greater than or equal_ to `layout.size()`. 112 | /// - The pointer is _at least_ aligned to `layout.align()`. 113 | /// 114 | /// `allocate` assumes that: 115 | /// - `thread_handle` is not concurrently accessed by another thread. 116 | /// - `thread_handle` belongs to this socket. 117 | /// - `layout` is valid, as per `Self::is_valid_layout`. 118 | #[inline(always)] 119 | pub unsafe fn allocate(&self, thread_handle: &ThreadHandle, layout: Layout) -> Option> { 120 | // Safety: 121 | // - Local lifetime. 122 | let socket_local = self.0.as_ref(); 123 | let thread_local = thread_handle.as_ref(); 124 | 125 | socket_local.allocate(thread_local, layout) 126 | } 127 | 128 | /// Deallocates the supplied block of memory. 129 | /// 130 | /// # Safety 131 | /// 132 | /// The caller should no longer reference the memory after calling this function. 133 | /// 134 | /// `deallocate` assumes that: 135 | /// - `thread_handle` is not concurrently accessed by another thread. 136 | /// - `thread_handle` belongs to this socket. 137 | /// - `ptr` is a value allocated by an instance of `Self`, and the same underlying `Platform`. 138 | #[inline(always)] 139 | pub unsafe fn deallocate(&self, thread_handle: &ThreadHandle, ptr: NonNull) { 140 | // Safety: 141 | // - Local lifetime. 142 | let socket_local = self.0.as_ref(); 143 | let thread_local = thread_handle.as_ref(); 144 | 145 | socket_local.deallocate(thread_local, ptr) 146 | } 147 | 148 | /// Deallocates the supplied block of memory. 149 | /// 150 | /// Unlike `deallocate`, the pointer is not cached for reuse on the local thread; as a result, this call may be 151 | /// slightly more costly. 152 | /// 153 | /// # Safety 154 | /// 155 | /// The caller should no longer reference the memory after calling this function. 156 | /// 157 | /// `deallocate` assumes that: 158 | /// - `thread_handle` is not concurrently accessed by another thread. 159 | /// - `thread_handle` belongs to this socket. 160 | /// - `ptr` is a value allocated by an instance of `Self`, and the same underlying `Platform`. 161 | #[inline(always)] 162 | pub unsafe fn deallocate_uncached(&self, ptr: NonNull) { 163 | // Safety: 164 | // - Local lifetime. 165 | let socket_local = self.0.as_ref(); 166 | 167 | socket_local.deallocate_uncached(ptr) 168 | } 169 | } 170 | 171 | impl<'a, C, P> SocketHandle<'a, C, P> { 172 | /// Creates a new instance from its content. 173 | pub(crate) fn from(socket: NonNull>) -> Self { SocketHandle(socket) } 174 | } 175 | 176 | impl<'a, C, P> Clone for SocketHandle<'a, C, P> { 177 | fn clone(&self) -> Self { *self } 178 | } 179 | 180 | impl<'a, C, P> Copy for SocketHandle<'a, C, P> {} 181 | 182 | /// A thread-safe handle to socket-local memory structures. 183 | /// 184 | /// # Recommendation 185 | /// 186 | /// There is a slight potential cost to using `AtomicSocketHandle` instead of `SocketHandle`, hence it is recommended 187 | /// to keep a global array of `AtomicSocketHandle` indexed by socket _and_ a thread-local `SocketHandle` on each 188 | /// thread: 189 | /// 190 | /// - The global array to avoid allocating more than one `SocketHandle` per socket. 191 | /// - The thread-local `SocketHandle` is used to speed-up allocation and deallocation. 192 | pub struct AtomicSocketHandle<'a, C, P>(AtomicPtr>); 193 | 194 | impl <'a, C, P> AtomicSocketHandle<'a, C, P> { 195 | /// Creates a null instance. 196 | pub const fn new() -> Self { Self(AtomicPtr::new(ptr::null_mut())) } 197 | 198 | /// Initializes the instance with the given handle. 199 | /// 200 | /// If `self` is NOT currently None, then the initialization fails and the `handle` is returned. 201 | pub fn initialize(&self, handle: SocketHandle<'a, C, P>) -> Result<(), SocketHandle<'a, C, P>> { 202 | self.0.compare_exchange(ptr::null_mut(), handle.0.as_ptr(), Ordering::Relaxed, Ordering::Relaxed) 203 | .and(Ok(())) 204 | .or(Err(handle)) 205 | } 206 | 207 | /// Loads the value of the handle, it may be None. 208 | pub fn load(&self) -> Option> { 209 | NonNull::new(self.0.load(Ordering::Relaxed)).map(SocketHandle) 210 | } 211 | 212 | /// Stores a handle, discards the previous handle if any. 213 | /// 214 | /// This method should generally be used only at start-up, to initialize the instance. 215 | pub fn store(&self, handle: SocketHandle<'a, C, P>) { 216 | self.0.store(handle.0.as_ptr(), Ordering::Relaxed); 217 | } 218 | } 219 | 220 | impl<'a, C, P> Default for AtomicSocketHandle<'a, C, P> { 221 | fn default() -> Self { Self(AtomicPtr::new(ptr::null_mut())) } 222 | } 223 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/huge_page.rs: -------------------------------------------------------------------------------- 1 | //! Huge Page 2 | //! 3 | //! A Huge Page is a polymorphic allocator of `Configuration::HUGE_PAGE_SIZE` bytes, which fulfills allocations of the 4 | //! Large category. 5 | //! 6 | //! A `SocketLocal` may own multiple 7 | 8 | mod atomic_bit_mask; 9 | mod foreign; 10 | mod number_pages; 11 | mod page_index; 12 | mod page_sizes; 13 | mod page_tokens; 14 | 15 | use core::{ 16 | alloc::Layout, 17 | cmp, 18 | mem, 19 | ptr::{self, NonNull}, 20 | slice, 21 | sync::atomic::{self, Ordering}, 22 | }; 23 | 24 | use crate::{Configuration, PowerOf2}; 25 | use crate::utils; 26 | 27 | use atomic_bit_mask::AtomicBitMask; 28 | use foreign::Foreign; 29 | use number_pages::NumberPages; 30 | use page_index::PageIndex; 31 | 32 | #[repr(C)] 33 | pub(crate) struct HugePage { 34 | // Guard against pre-fetching on previous page. 35 | _prefetch: utils::PrefetchGuard, 36 | // Common elements, immutable. 37 | common: Common, 38 | // Foreign elements, mutable and contended. 39 | foreign: Foreign, 40 | // Guard against pre-fetching on start of buffer zone. 41 | _postfetch: utils::PrefetchGuard, 42 | } 43 | 44 | impl HugePage { 45 | /// In-place constructs a `HugePage`. 46 | /// 47 | /// # Safety 48 | /// 49 | /// - Assumes that there is sufficient memory available. 50 | /// - Assumes that the pointer is correctly aligned. 51 | pub(crate) unsafe fn initialize(place: &mut [u8], owner: *mut ()) -> NonNull 52 | where 53 | C: Configuration, 54 | { 55 | debug_assert!(place.len() >= C::HUGE_PAGE_SIZE.value()); 56 | 57 | // Safety: 58 | // - `place` is not a null size. 59 | let at = NonNull::new_unchecked(place.as_mut_ptr()); 60 | 61 | debug_assert!(utils::is_sufficiently_aligned_for(at, C::HUGE_PAGE_SIZE)); 62 | debug_assert!(mem::size_of::() <= C::LARGE_PAGE_SIZE.value()); 63 | 64 | // Safety: 65 | // - `at` is assumed to be sufficiently sized. 66 | // - `at` is assumed to be sufficiently aligned. 67 | #[allow(clippy::cast_ptr_alignment)] 68 | let huge_page = at.as_ptr() as *mut Self; 69 | 70 | ptr::write(huge_page, HugePage::new::(owner)); 71 | 72 | // Enforce memory ordering, later Acquire need to see those 0s and 1s. 73 | atomic::fence(Ordering::Release); 74 | 75 | at.cast() 76 | } 77 | 78 | /// Obtain the huge page associated to a given allocation. 79 | /// 80 | /// # Safety 81 | /// 82 | /// - Assumes that the pointer is pointing strictly _inside_ a HugePage. 83 | pub(crate) unsafe fn from_raw(ptr: NonNull) -> NonNull 84 | where 85 | C: Configuration, 86 | { 87 | debug_assert!(!utils::is_sufficiently_aligned_for(ptr, C::HUGE_PAGE_SIZE)); 88 | 89 | let address = ptr.as_ptr() as usize; 90 | let huge_page = C::HUGE_PAGE_SIZE.round_down(address); 91 | 92 | // Safety: 93 | // - `ptr` was not null 94 | NonNull::new_unchecked(huge_page as *mut HugePage) 95 | } 96 | 97 | /// Allocate one or more LargePages from this page, if any. 98 | /// 99 | /// Returns a null pointer if the allocation cannot be fulfilled. 100 | pub(crate) unsafe fn allocate(&self, layout: Layout) -> Option> { 101 | debug_assert!(layout.align().count_ones() == 1, "{} is not a power of 2", layout.align()); 102 | 103 | let large_page_size = self.common.page_size; 104 | 105 | let number_pages = large_page_size.round_up(layout.size()) / large_page_size; 106 | debug_assert!(number_pages > 0); 107 | debug_assert!(number_pages <= self.common.number_pages.0); 108 | 109 | // Safety: 110 | // - `layout.align()` is a power of 2. 111 | let align_pages = PowerOf2::new_unchecked(cmp::max(layout.align() / large_page_size, 1)); 112 | debug_assert!(align_pages.value() > 0); 113 | 114 | if let Some(index) = self.foreign.allocate(NumberPages(number_pages), align_pages) { 115 | NonNull::new(self.address().add(index.value() * large_page_size)) 116 | } else { 117 | None 118 | } 119 | } 120 | 121 | /// Deallocates one or multiple pages from this page. 122 | /// 123 | /// # Safety 124 | /// 125 | /// - Assumes that the pointer is pointing to a `LargePage` inside _this_ `HugePage`. 126 | /// - Assumes that the pointed page is no longer in use. 127 | pub(crate) unsafe fn deallocate(&self, ptr: NonNull) { 128 | debug_assert!(utils::is_sufficiently_aligned_for(ptr, self.common.page_size)); 129 | 130 | let index = (ptr.as_ptr() as usize - self.address() as usize) / self.common.page_size; 131 | debug_assert!(index > 0 && index <= self.common.number_pages.0); 132 | 133 | // Safety: 134 | // - `index` is assumed not to be 0. 135 | // - `index` is assumed to point to pages no longer in use. 136 | self.foreign.deallocate(PageIndex::new_unchecked(index)); 137 | } 138 | 139 | /// Returns the owner of the page. 140 | pub(crate) fn owner(&self) -> *mut () { self.common.owner } 141 | 142 | /// Sets the owner of the page. 143 | pub(crate) fn set_owner(&mut self, owner: *mut ()) { 144 | debug_assert!(self.common.owner.is_null()); 145 | self.common.owner = owner; 146 | } 147 | 148 | /// Returns a mutable slice to the free-space area between the end of the `HugePage` header and the first 149 | /// `LargePage`. 150 | /// 151 | /// Note that this buffer area may be empty, if the HugePage header just fits within a Large page size. 152 | pub(crate) fn buffer_mut(&mut self) -> &mut [u8] { 153 | // Safety: 154 | // - The pointer is non-null and suitably aligned. 155 | // - The length correctly represents the available memory, and fits within `isize`. 156 | unsafe { slice::from_raw_parts_mut(self.buffer_ptr(), self.buffer_len()) } 157 | } 158 | 159 | fn new(owner: *mut ()) -> Self 160 | where 161 | C: Configuration, 162 | { 163 | let large_page_size = C::LARGE_PAGE_SIZE; 164 | let huge_page_size = C::HUGE_PAGE_SIZE; 165 | 166 | let number_pages = NumberPages(huge_page_size / large_page_size - 1); 167 | 168 | let _prefetch = utils::PrefetchGuard::default(); 169 | let common = Common::new(owner, large_page_size, number_pages); 170 | let foreign = Foreign::new(number_pages); 171 | let _postfetch = utils::PrefetchGuard::default(); 172 | 173 | Self { _prefetch, common, foreign, _postfetch, } 174 | } 175 | 176 | fn address(&self) -> *mut u8 { self as *const _ as *const u8 as *mut u8 } 177 | 178 | fn buffer_ptr(&self) -> *mut u8 { 179 | // Safety: 180 | // - The size of Self is small enough that the resulting pointer fits within the same block of memory. 181 | unsafe { self.address().add(mem::size_of::()) } 182 | } 183 | 184 | fn buffer_len(&self) -> usize { 185 | // Account for a pre-fetch guard at end of buffer area. 186 | let reserved = mem::size_of::() + 128; 187 | 188 | self.common.page_size.value().saturating_sub(reserved) 189 | } 190 | } 191 | 192 | // 193 | // Implementation Details 194 | // 195 | // A 128-bytes alignment is used as Intel CPUs prefetch data 2 cache lines (64 bytes) at a time, which to the best of 196 | // my knowledge is the greatest prefetching among mainstream CPUs. 197 | // 198 | 199 | // Common data. Read-only, accessible from both the local thread and foreign threads without synchronization. 200 | #[repr(align(128))] 201 | struct Common { 202 | // A pointer to the owner of the LargePage. 203 | // 204 | // Outside tests, this should point to the SocketLocal from which the page was allocated. 205 | owner: *mut (), 206 | // The size of an individual Large Page. 207 | page_size: PowerOf2, 208 | // The number of Large Pages. 209 | number_pages: NumberPages, 210 | } 211 | 212 | impl Common { 213 | /// Creates a new instance of `Common`. 214 | fn new(owner: *mut (), page_size: PowerOf2, number_pages: NumberPages) -> Self { 215 | debug_assert!(number_pages.0 >= 1); 216 | 217 | Self { owner, page_size, number_pages, } 218 | } 219 | } 220 | 221 | #[cfg(test)] 222 | mod tests { 223 | 224 | use super::*; 225 | 226 | #[test] 227 | fn huge_page_smoke_test() { 228 | const HUGE_PAGE_SIZE: usize = 128 * 1024; 229 | const LARGE_PAGE_SIZE: usize = 8 * 1024; 230 | const HUGE_HEADER_SIZE: usize = mem::size_of::(); 231 | 232 | struct TestConfiguration; 233 | 234 | impl Configuration for TestConfiguration { 235 | const LARGE_PAGE_SIZE: PowerOf2 = unsafe { PowerOf2::new_unchecked(LARGE_PAGE_SIZE) }; 236 | const HUGE_PAGE_SIZE: PowerOf2 = unsafe { PowerOf2::new_unchecked(HUGE_PAGE_SIZE) }; 237 | } 238 | 239 | #[repr(align(131072))] 240 | struct AlignedPage(u8); 241 | 242 | let owner = 1234usize as *mut (); 243 | 244 | // Use PrefetchGuard to guarantee correct alignment. 245 | let mut raw: mem::MaybeUninit = mem::MaybeUninit::uninit(); 246 | let slice = unsafe { slice::from_raw_parts_mut(raw.as_mut_ptr() as *mut u8, mem::size_of::()) }; 247 | 248 | let mut huge_page = unsafe { HugePage::initialize::(slice, ptr::null_mut()) }; 249 | let huge_page_ptr = huge_page.as_ptr() as *mut u8; 250 | assert_eq!(slice.as_mut_ptr(), huge_page_ptr); 251 | 252 | let huge_page = unsafe { huge_page.as_mut() }; 253 | assert_eq!(huge_page_ptr as usize, huge_page.address() as usize); 254 | 255 | assert_eq!(ptr::null_mut(), huge_page.owner()); 256 | 257 | huge_page.set_owner(owner); 258 | assert_eq!(owner, huge_page.owner()); 259 | 260 | { 261 | let buffer = huge_page.buffer_mut(); 262 | 263 | let start_ptr = huge_page_ptr as usize; 264 | let buffer_ptr = buffer.as_mut_ptr() as usize; 265 | 266 | assert_eq!(HUGE_HEADER_SIZE, buffer_ptr - start_ptr); 267 | assert_eq!(LARGE_PAGE_SIZE - HUGE_HEADER_SIZE - 128, buffer.len()); 268 | } 269 | 270 | let layout = Layout::from_size_align(LARGE_PAGE_SIZE + 1, 1).expect("Proper layout"); 271 | let allocated = unsafe { huge_page.allocate(layout) }; 272 | assert_ne!(None, allocated); 273 | 274 | let retrieved = unsafe { HugePage::from_raw::(allocated.unwrap()) }; 275 | assert_eq!(huge_page_ptr, retrieved.as_ptr() as *mut u8); 276 | 277 | let layout = Layout::from_size_align(LARGE_PAGE_SIZE * 14, 1).expect("Proper layout"); 278 | let failed = unsafe { huge_page.allocate(layout) }; 279 | assert_eq!(None, failed); 280 | 281 | unsafe { huge_page.deallocate(allocated.unwrap()) }; 282 | } 283 | 284 | } // mod tests 285 | -------------------------------------------------------------------------------- /llmalloc/src/allocator.rs: -------------------------------------------------------------------------------- 1 | //! Allocator 2 | 3 | use core::{ 4 | alloc::GlobalAlloc, 5 | ptr::{self, NonNull}, 6 | }; 7 | 8 | use llmalloc_core::{self, Configuration, Layout, PowerOf2}; 9 | 10 | use crate::{LLConfiguration, Platform, LLPlatform, ThreadLocal, LLThreadLocal}; 11 | 12 | /// Low-Latency Allocator. 13 | #[derive(Default)] 14 | pub struct LLAllocator; 15 | 16 | impl LLAllocator { 17 | /// Creates an instance. 18 | pub const fn new() -> Self { Self } 19 | 20 | /// Prepares the socket-local and thread-local structures for allocation. 21 | /// 22 | /// Returns Ok if the attempt succeeded, Err otherwise. 23 | /// 24 | /// Failure to warm up the current thread may occur if: 25 | /// 26 | /// - The socket-local structure is not ready, and the underlying `Platform` cannot allocate one. 27 | /// - The socket-local structure cannot allocate a thread-local structure. 28 | #[cold] 29 | pub fn warm_up(&self) -> Result<(), ()> { 30 | Thread::get().or_else(Thread::initialize).map(|_| ()).ok_or(()) 31 | } 32 | 33 | /// Ensures that at least `target` `HugePage` are allocated on the socket. 34 | /// 35 | /// Returns the minimum of the currently allocated number of pages and `target`. 36 | /// 37 | /// Failure to meet the `target` may occur if: 38 | /// 39 | /// - The maximum number of `HugePage` a `socket` may contain has been reached. 40 | /// - The underlying `Platform` is failing to allocate more `HugePage`. 41 | #[cold] 42 | pub fn reserve(&self, target: usize) -> usize { 43 | if let Some(socket) = Sockets::socket_handle() { 44 | socket.reserve(target) 45 | } else { 46 | 0 47 | } 48 | } 49 | 50 | /// Allocates `size` bytes of memory, aligned on at least an `alignment` boundary. 51 | /// 52 | /// If allocation fails, the returned pointer may be NULL. 53 | pub fn allocate(&self, layout: Layout) -> Option> { 54 | debug_assert!(layout.align().count_ones() == 1); 55 | 56 | if layout.align() > LLConfiguration::HUGE_PAGE_SIZE.value() { 57 | return None; 58 | } 59 | 60 | // Safety: 61 | // - `layout.align()` is a power of 2. 62 | let align = unsafe { PowerOf2::new_unchecked(layout.align()) }; 63 | 64 | // Round up size to a multiple of `align`, if not already. 65 | let layout = if layout.size() % align == 0 { 66 | layout 67 | } else { 68 | let size = align.round_up(layout.size()); 69 | 70 | // Safety: 71 | // - `align` is not 0. 72 | // - `align` is a power of 2. 73 | // - `size` is rounded up to a multiple of `align`, without overflow. 74 | unsafe { Layout::from_size_align_unchecked(size, align.value()) } 75 | }; 76 | 77 | if let Some(thread_local) = Thread::get().or_else(Thread::initialize) { 78 | return thread_local.allocate(layout); 79 | } 80 | 81 | None 82 | } 83 | 84 | /// Deallocates the memory located at `pointer`. 85 | /// 86 | /// # Safety 87 | /// 88 | /// - Assumes `pointer` has been returned by a prior call to `allocate`. 89 | /// - Assumes `pointer` has not been deallocated since its allocation. 90 | /// - Assumes the memory pointed by `pointer` is no longer in use. 91 | pub unsafe fn deallocate(&self, pointer: NonNull) { 92 | if let Some(thread_local) = Thread::get().or_else(Thread::initialize) { 93 | return thread_local.deallocate(pointer); 94 | } 95 | 96 | // If a non-null pointer exists, it _must_ have been allocated, and therefore there should be at least one 97 | // non-null socket-handle, somewhere, through which the memory can be returned. 98 | Sockets::any_socket_handle().deallocate_uncached(pointer); 99 | } 100 | } 101 | 102 | unsafe impl GlobalAlloc for LLAllocator { 103 | unsafe fn alloc(&self, layout: Layout) -> *mut u8 { 104 | self.allocate(layout).map(|ptr| ptr.as_ptr()).unwrap_or(ptr::null_mut()) 105 | } 106 | 107 | unsafe fn dealloc(&self, ptr: *mut u8, _: Layout) { 108 | if let Some(ptr) = NonNull::new(ptr) { 109 | self.deallocate(ptr); 110 | } 111 | } 112 | } 113 | 114 | // 115 | // Integration test backdoors. 116 | // 117 | // Unfortunately the backdoors have to be exposed as part of the public API for use in integration tests. 118 | // 119 | 120 | impl LLAllocator { 121 | /// Exposes the index of the SocketHandle. 122 | #[cold] 123 | #[doc(hidden)] 124 | pub fn socket_index(&self) -> usize { Sockets::current_node() } 125 | 126 | /// Exposes the index of the ThreadHandle. 127 | #[cold] 128 | #[doc(hidden)] 129 | pub fn thread_index(&self) -> usize { 130 | THREAD_LOCAL.get().map(|ptr| ptr.as_ptr() as usize).unwrap_or(0) 131 | } 132 | } 133 | 134 | // 135 | // Implementation 136 | // 137 | 138 | type AtomicSocketHandle = llmalloc_core::AtomicSocketHandle<'static, LLConfiguration, LLPlatform>; 139 | type DomainHandle = llmalloc_core::DomainHandle; 140 | type SocketHandle = llmalloc_core::SocketHandle<'static, LLConfiguration, LLPlatform>; 141 | type ThreadHandle = llmalloc_core::ThreadHandle; 142 | 143 | // Domain Handle. 144 | static DOMAIN: DomainHandle = DomainHandle::new(LLPlatform::new()); 145 | 146 | // Storage for up to 64 NUMA nodes; it should be vastly overkill. 147 | static SOCKETS: Sockets = Sockets::new(); 148 | 149 | // Thread-local. 150 | // 151 | // Safety: 152 | // - `drop_handle` points to an `unsafe extern "C" fn(*mut u8)`. 153 | static THREAD_LOCAL: LLThreadLocal = unsafe { LLThreadLocal::new(drop_handle as *const u8) }; 154 | 155 | #[cold] 156 | unsafe extern "C" fn drop_handle(handle: *mut u8) { 157 | let handle = NonNull::new(handle).expect("Non-null handle"); 158 | 159 | let thread = ThreadHandle::from_pointer(handle); 160 | let socket: SocketHandle = thread.socket(); 161 | socket.release_thread_handle(thread); 162 | } 163 | 164 | struct Thread(ThreadHandle); 165 | 166 | impl Thread { 167 | // Returns a pointer to the thread-local instance, if initialized. 168 | #[inline(always)] 169 | fn get() -> Option { 170 | THREAD_LOCAL.get() 171 | .map(|pointer| unsafe { Self(ThreadHandle::from_pointer(pointer)) }) 172 | } 173 | 174 | // Initializes the thread-local instance and attempts to return a reference to it. 175 | // 176 | // Initialization may fail for any reason, in which case None is returned. 177 | #[cold] 178 | #[inline(never)] 179 | fn initialize() -> Option { 180 | // Get the handles, can't do anything without both! 181 | let socket = Sockets::socket_handle()?; 182 | let thread = socket.acquire_thread_handle()?; 183 | 184 | THREAD_LOCAL.set(thread.into_pointer()); 185 | 186 | Self::get() 187 | } 188 | 189 | // Allocates `size` bytes of memory, aligned on at least an `alignment` boundary. 190 | // 191 | // If allocation fails, the returned pointer may be NULL. 192 | #[inline(always)] 193 | fn allocate(&self, layout: Layout) -> Option> { 194 | // Safety: 195 | // - Only uses SocketHandle type. 196 | let socket: SocketHandle = unsafe { self.0.socket() }; 197 | 198 | // Safety: 199 | // - `layout` is valid. 200 | // - `self.0` belongs `socket`. 201 | // - `self.0` is exclusively accessed from this thread. 202 | unsafe { socket.allocate(&self.0, layout) } 203 | } 204 | 205 | // Deallocates the memory located at `pointer`. 206 | // 207 | // # Safety 208 | // 209 | // - Assumes `pointer` has been returned by a prior call to `allocate`. 210 | // - Assumes `pointer` has not been deallocated since its allocation. 211 | // - Assumes the memory pointed by `pointer` is no longer in use. 212 | #[inline(always)] 213 | unsafe fn deallocate(&self, pointer: NonNull) { 214 | // Safety: 215 | // - Only uses SocketHandle type. 216 | let socket: SocketHandle = self.0.socket(); 217 | 218 | // Safety: TODO 219 | socket.deallocate(&self.0, pointer) 220 | } 221 | } 222 | 223 | struct Sockets([AtomicSocketHandle; 64]); 224 | 225 | impl Sockets { 226 | // Creates an instance. 227 | #[cold] 228 | const fn new() -> Self { 229 | const fn ash() -> AtomicSocketHandle { AtomicSocketHandle::new() } 230 | 231 | Self([ 232 | // Line 0: up to 16 instances. 233 | ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), 234 | // Line 1: up to 32 instances. 235 | ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), 236 | // Line 2: up to 48 instances. 237 | ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), 238 | // Line 3: up to 64 instances. 239 | ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), ash(), 240 | ]) 241 | } 242 | 243 | // Returns a SocketHandle for this particular NUMA Node. 244 | #[cold] 245 | #[inline(never)] 246 | fn socket_handle() -> Option { SOCKETS.socket_handle_impl() } 247 | 248 | // Returns the first SocketHandle it finds. 249 | // 250 | // # Panics 251 | // 252 | // If no handle has been allocated. 253 | #[cold] 254 | #[inline(never)] 255 | fn any_socket_handle() -> SocketHandle { SOCKETS.any_socket_handle_impl() } 256 | 257 | // Internal; returns a SocketHandle, initialized if need be. 258 | #[cold] 259 | fn socket_handle_impl(&self) -> Option { 260 | let index = Self::current_node(); 261 | let atomic_handle = &self.0[index]; 262 | 263 | if let Some(socket_handle) = atomic_handle.load() { 264 | return Some(socket_handle); 265 | } 266 | 267 | // There may not be enough memory to allocate a new handle. 268 | let socket_handle = SocketHandle::new(&DOMAIN)?; 269 | 270 | // Let's race to see who gets to initialize the handle. 271 | // 272 | // If this thread loses, free the superfluous handle. 273 | if let Err(socket_handle) = atomic_handle.initialize(socket_handle) { 274 | // Safety: 275 | // - No alias exists, the handle has never been shared yet. 276 | unsafe { socket_handle.close() }; 277 | } 278 | 279 | // If the race was won, it's initialized, otherwise, it's initialized! 280 | atomic_handle.load() 281 | } 282 | 283 | // Internal; returns the first SocketHandle it finds, or panics if it finds none. 284 | #[cold] 285 | fn any_socket_handle_impl(&self) -> SocketHandle { 286 | for atomic_handle in &self.0[..] { 287 | if let Some(socket_handle) = atomic_handle.load() { 288 | return socket_handle; 289 | } 290 | } 291 | 292 | unreachable!("How can memory need be deallocated, if no socket handle was ever allocated?"); 293 | } 294 | 295 | #[cold] 296 | fn current_node() -> usize { DOMAIN.platform().current_node().value() as usize } 297 | } 298 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2020 matthieu-m 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/blocks/block_local.rs: -------------------------------------------------------------------------------- 1 | //! A Block of memory from the Local thread, only accessed by the local thread. 2 | 3 | use core::{ 4 | mem, 5 | ptr::{self, NonNull}, 6 | }; 7 | 8 | use crate::{PowerOf2, utils}; 9 | 10 | use super::{AtomicBlockForeign, AtomicBlockForeignList, BlockForeign, BlockForeignList, BlockPtr}; 11 | 12 | /// BlockLocal. 13 | /// 14 | /// A BlockLocal points to memory local to the current ThreadLocal. 15 | #[repr(C)] 16 | #[derive(Default)] 17 | pub(crate) struct BlockLocal { 18 | next: BlockLocalStack, 19 | } 20 | 21 | impl BlockLocal { 22 | /// In-place constructs a `BlockLocal`. 23 | /// 24 | /// # Safety 25 | /// 26 | /// - Assumes that access to the memory location is exclusive. 27 | /// - Assumes that there is sufficient memory available. 28 | /// - Assumes that the pointer is correctly aligned. 29 | #[allow(clippy::cast_ptr_alignment)] 30 | pub(crate) unsafe fn initialize(at: NonNull) -> NonNull { 31 | debug_assert!(utils::is_sufficiently_aligned_for(at, PowerOf2::align_of::())); 32 | 33 | // Safety: 34 | // - `at` is assumed to be sufficiently aligned. 35 | let ptr = at.as_ptr() as *mut BlockLocal; 36 | 37 | // Safety: 38 | // - Access to the memory location is exclusive. 39 | // - `ptr` is assumed to be sufficiently sized. 40 | ptr::write(ptr, BlockLocal::default()); 41 | 42 | at.cast() 43 | } 44 | 45 | /// In-place reinterpret a `AtomicBlockForeign` as a `BlockLocal`. 46 | /// 47 | /// # Safety 48 | /// 49 | /// - Assumes that access to the block, and all tail blocks, is exclusive. 50 | pub(crate) unsafe fn from_atomic(foreign: NonNull) -> NonNull { 51 | // Safety: 52 | // - The layout are checked to be compatible below. 53 | let local = foreign.cast(); 54 | 55 | debug_assert!(Self::are_layout_compatible(foreign, local)); 56 | 57 | local 58 | } 59 | 60 | /// In-place reinterpret a `BlockForeign` as a `BlockLocal`. 61 | /// 62 | /// # Safety 63 | /// 64 | /// - Assumes that access to the block, and all tail blocks, is exclusive. 65 | pub(crate) unsafe fn from_foreign(foreign: NonNull) -> NonNull { 66 | // Safety: 67 | // - The layout are checked to be compatible below. 68 | let atomic: NonNull = foreign.cast(); 69 | 70 | // Safety: 71 | // - The layout are checked to be compatible below. 72 | let local = atomic.cast(); 73 | 74 | debug_assert!(Self::are_layout_compatible(atomic, local)); 75 | 76 | local 77 | } 78 | 79 | // Returns whether the layout of AtomicBlockForeign and BlockLocal are compatible. 80 | // 81 | // The layout are compatible if: 82 | // - BlockLocalStack and AtomicBlockForeignList are both plain pointers, size-wise. 83 | // - BlockLocal::next and AtomicBlockForeign::next are placed at the same offset. 84 | fn are_layout_compatible(foreign: NonNull, local: NonNull) -> bool { 85 | const PTR_SIZE: usize = mem::size_of::<*const u8>(); 86 | 87 | if mem::size_of::() != PTR_SIZE || mem::size_of::() != PTR_SIZE { 88 | return false; 89 | } 90 | 91 | let foreign_offset = { 92 | let address = foreign.as_ptr() as usize; 93 | // Safety: 94 | // - Bounded lifetime. 95 | let next_address = unsafe { &foreign.as_ref().next as *const _ as usize }; 96 | next_address - address 97 | }; 98 | 99 | let local_offset = { 100 | let address = local.as_ptr() as usize; 101 | // Safety: 102 | // - Bounded lifetime. 103 | let next_address = unsafe { &local.as_ref().next as *const _ as usize }; 104 | next_address - address 105 | }; 106 | 107 | foreign_offset == local_offset 108 | } 109 | } 110 | 111 | /// BlockLocalStack. 112 | #[derive(Default)] 113 | pub(crate) struct BlockLocalStack(BlockPtr); 114 | 115 | impl BlockLocalStack { 116 | /// Creates an instance. 117 | pub(crate) fn new(ptr: Option>) -> Self { Self(BlockPtr::new(ptr)) } 118 | 119 | /// Creates an instance from a raw pointer. 120 | /// 121 | /// # Safety 122 | /// 123 | /// - Assumes that access to the memory location is exclusive. 124 | /// - Assumes that there is sufficient memory available. 125 | /// - Assumes that the pointer is correctly aligned. 126 | pub(crate) unsafe fn from_raw(ptr: NonNull) -> Self { Self::new(Some(BlockLocal::initialize(ptr))) } 127 | 128 | /// Returns whether the stack is empty, or not. 129 | pub(crate) fn is_empty(&self) -> bool { self.get().is_none() } 130 | 131 | /// Pops the head of the tail-list, if any. 132 | pub(crate) fn pop(&self) -> Option> { 133 | let result = self.get()?; 134 | 135 | // Safety: 136 | // - Non-null, and valid instance. 137 | let next = unsafe { result.as_ref().next.get() }; 138 | self.set(next); 139 | 140 | Some(result) 141 | } 142 | 143 | /// Prepends the block to the head of the tail-list. 144 | pub(crate) fn push(&self, block: NonNull) { 145 | unsafe { 146 | // Safety: 147 | // - Bounded lifetime. 148 | block.as_ref().next.set(self.get()); 149 | } 150 | 151 | self.set(Some(block)); 152 | } 153 | 154 | /// Refills the list from a BlockForeign. 155 | /// 156 | /// # Safety 157 | /// 158 | /// - Assumes that access to the memory location, and any tail location, is exclusive. 159 | pub(crate) unsafe fn refill(&self, block: NonNull) { 160 | debug_assert!(self.is_empty()); 161 | 162 | self.set(Some(block)) 163 | } 164 | 165 | /// Extends the tail-list pointed to by prepending `list`. 166 | /// 167 | /// # Safety 168 | /// 169 | /// - Assumes that the access to the tail blocks, is exclusive. 170 | /// - Assumes that the list is not empty. 171 | pub(crate) unsafe fn extend(&self, list: &BlockForeignList) { 172 | debug_assert!(!list.is_empty()); 173 | 174 | // Safety: 175 | // - `list` is assumed not to be empty. 176 | let (head, tail) = list.steal(); 177 | 178 | // Link the tail. 179 | let tail = BlockLocal::from_foreign(tail); 180 | 181 | // Safety: 182 | // - Boundded lifetime. 183 | tail.as_ref().next.set(self.get()); 184 | 185 | // Set the head. 186 | let head = BlockLocal::from_foreign(head); 187 | self.set(Some(head)); 188 | } 189 | 190 | /// Returns the pointer, possibly null. 191 | #[cfg(test)] 192 | pub(crate) fn peek(&self) -> Option> { self.get() } 193 | 194 | fn get(&self) -> Option> { self.0.get() } 195 | 196 | fn set(&self, value: Option>) { self.0.set(value); } 197 | } 198 | 199 | #[cfg(test)] 200 | mod tests { 201 | 202 | use core::mem::MaybeUninit; 203 | 204 | use super::*; 205 | use super::super::AlignedArray; 206 | 207 | #[test] 208 | fn block_local_initialize() { 209 | let mut block = MaybeUninit::::uninit(); 210 | 211 | // Safety: 212 | // - Access to the memory location is exclusive. 213 | unsafe { ptr::write_bytes(block.as_mut_ptr(), 0xfe, 1) }; 214 | 215 | // Safety: 216 | // - Access to the memory location is exclusive. 217 | // - The memory location is sufficiently sized and aligned for `BlockLocal`. 218 | unsafe { BlockLocal::initialize(NonNull::from(&block).cast()) }; 219 | 220 | // Safety: 221 | // - Initialized! 222 | let block = unsafe { block.assume_init() }; 223 | 224 | assert!(block.next.is_empty()); 225 | } 226 | 227 | #[test] 228 | fn block_local_from() { 229 | let array = AlignedArray::::default(); 230 | 231 | let (head, tail) = (array.get(1), array.get(2)); 232 | 233 | // Safety: 234 | // - Bounded lifetime. 235 | unsafe { 236 | head.as_ref().next.set(Some(tail)); 237 | head.as_ref().length.set(1); 238 | } 239 | 240 | // Safety: 241 | // - Access to the blocks is exclusive. 242 | let block = unsafe { BlockLocal::from_foreign(head) }; 243 | 244 | // Safety: 245 | // - Bounded lifetime. 246 | let next = unsafe { block.as_ref().next.peek() }; 247 | 248 | assert_eq!(Some(tail.cast()), next); 249 | } 250 | 251 | #[test] 252 | fn block_local_stack_is_empty() { 253 | let array = AlignedArray::::default(); 254 | let block = array.get(1); 255 | 256 | let stack = BlockLocalStack::new(None); 257 | assert!(stack.is_empty()); 258 | 259 | let stack = BlockLocalStack::new(Some(block)); 260 | assert!(!stack.is_empty()); 261 | } 262 | 263 | #[test] 264 | fn block_local_stack_peek() { 265 | let array = AlignedArray::::default(); 266 | let block = array.get(1); 267 | 268 | let stack = BlockLocalStack::new(None); 269 | assert_eq!(None, stack.peek()); 270 | 271 | let stack = BlockLocalStack::new(Some(block)); 272 | assert_eq!(Some(block), stack.peek()); 273 | } 274 | 275 | #[test] 276 | fn block_local_stack_pop_push() { 277 | let array = AlignedArray::::default(); 278 | let (a, b) = (array.get(1), array.get(2)); 279 | 280 | let stack = BlockLocalStack::default(); 281 | assert_eq!(None, stack.peek()); 282 | assert_eq!(None, stack.pop()); 283 | 284 | stack.push(a); 285 | 286 | assert_eq!(Some(a), stack.peek()); 287 | assert_eq!(Some(a), stack.pop()); 288 | 289 | assert_eq!(None, stack.peek()); 290 | assert_eq!(None, stack.pop()); 291 | 292 | stack.push(b); 293 | stack.push(a); 294 | 295 | assert_eq!(Some(a), stack.peek()); 296 | assert_eq!(Some(a), stack.pop()); 297 | 298 | assert_eq!(Some(b), stack.peek()); 299 | assert_eq!(Some(b), stack.pop()); 300 | 301 | assert_eq!(None, stack.peek()); 302 | assert_eq!(None, stack.pop()); 303 | } 304 | 305 | #[test] 306 | fn block_local_stack_refill() { 307 | let array = AlignedArray::::default(); 308 | let (head, tail) = (array.get(1), array.get(2)); 309 | 310 | // Safety: 311 | // - Bounded lifetime. 312 | unsafe { 313 | head.as_ref().next.set(Some(tail)); 314 | } 315 | 316 | let stack = BlockLocalStack::default(); 317 | 318 | unsafe { stack.refill(head) }; 319 | 320 | assert_eq!(Some(head.cast()), stack.pop()); 321 | assert_eq!(Some(tail.cast()), stack.pop()); 322 | assert_eq!(None, stack.pop()); 323 | } 324 | 325 | #[test] 326 | fn block_local_stack_extend_empty() { 327 | let array = AlignedArray::::default(); 328 | let (head, tail) = (array.get(1), array.get(2)); 329 | 330 | let list = BlockForeignList::default(); 331 | list.push(tail); 332 | list.push(head); 333 | 334 | let stack = BlockLocalStack::default(); 335 | 336 | unsafe { stack.extend(&list) }; 337 | 338 | assert_eq!(Some(head.cast()), stack.pop()); 339 | assert_eq!(Some(tail.cast()), stack.pop()); 340 | assert_eq!(None, stack.pop()); 341 | } 342 | 343 | #[test] 344 | fn block_local_stack_extend_existing() { 345 | let array = AlignedArray::::default(); 346 | let (head, tail) = (array.get(1), array.get(2)); 347 | 348 | let local = BlockLocal::default(); 349 | let local = NonNull::from(&local); 350 | 351 | let list = BlockForeignList::default(); 352 | list.push(tail); 353 | list.push(head); 354 | 355 | let stack = BlockLocalStack::new(Some(local)); 356 | 357 | unsafe { stack.extend(&list) }; 358 | 359 | assert_eq!(Some(head.cast()), stack.pop()); 360 | assert_eq!(Some(tail.cast()), stack.pop()); 361 | assert_eq!(Some(local), stack.pop()); 362 | assert_eq!(None, stack.pop()); 363 | } 364 | 365 | } // mod tests 366 | -------------------------------------------------------------------------------- /llmalloc/src/platform/linux.rs: -------------------------------------------------------------------------------- 1 | //! Implementation of Linux specific calls. 2 | 3 | use core::{ 4 | alloc::Layout, 5 | marker::PhantomData, 6 | mem, 7 | ptr::{self, NonNull}, 8 | sync::atomic, 9 | }; 10 | 11 | use llmalloc_core::{self, PowerOf2}; 12 | 13 | use super::{NumaNodeIndex, Configuration, Platform, ThreadLocal}; 14 | 15 | /// Implementation of the Configuration trait, for Linux. 16 | #[derive(Default)] 17 | pub(crate) struct LLConfiguration; 18 | 19 | impl Configuration for LLConfiguration { 20 | // 2 MB 21 | const LARGE_PAGE_SIZE: PowerOf2 = unsafe { PowerOf2::new_unchecked(2 * 1024 * 1024) }; 22 | 23 | // 1 GB 24 | const HUGE_PAGE_SIZE: PowerOf2 = unsafe { PowerOf2::new_unchecked(1024 * 1024 * 1024) }; 25 | } 26 | 27 | /// Implementation of the Platform trait, for Linux. 28 | #[derive(Default)] 29 | pub(crate) struct LLPlatform; 30 | 31 | impl LLPlatform { 32 | /// Creates an instance. 33 | pub(crate) const fn new() -> Self { Self } 34 | } 35 | 36 | impl llmalloc_core::Platform for LLPlatform { 37 | unsafe fn allocate(&self, layout: Layout) -> Option> { 38 | const HUGE_PAGE_SIZE: PowerOf2 = LLConfiguration::HUGE_PAGE_SIZE; 39 | 40 | assert!(layout.size() % HUGE_PAGE_SIZE == 0, 41 | "Incorrect size: {} % {} != 0", layout.size(), HUGE_PAGE_SIZE.value()); 42 | assert!(layout.align() <= HUGE_PAGE_SIZE.value(), 43 | "Incorrect alignment: {} > {}", layout.align(), HUGE_PAGE_SIZE.value()); 44 | 45 | let candidate = mmap_huge(layout.size()) 46 | .or_else(|| mmap_exact(layout.size())) 47 | .or_else(|| mmap_over(layout.size()))?; 48 | 49 | debug_assert!(candidate.as_ptr() as usize % HUGE_PAGE_SIZE == 0, 50 | "Incorrect alignment of allocation: {:x} % {:x} != 0", candidate.as_ptr() as usize, HUGE_PAGE_SIZE.value()); 51 | 52 | Some(candidate) 53 | } 54 | 55 | unsafe fn deallocate(&self, pointer: NonNull, layout: Layout) { 56 | munmap_deallocate(pointer.as_ptr(), layout.size()); 57 | } 58 | } 59 | 60 | impl Platform for LLPlatform { 61 | #[cold] 62 | #[inline(never)] 63 | fn current_node(&self) -> NumaNodeIndex { 64 | let cpu = unsafe { libc::sched_getcpu() }; 65 | assert!(cpu >= 0, "Expected CPU index, got {}", cpu); 66 | 67 | let node = unsafe { numa_node_of_cpu(cpu) }; 68 | 69 | // If libnuma cannot find the appropriate node (such as under WSL), then use 0 as fallback. 70 | if node < 0 { 71 | return NumaNodeIndex::new(0); 72 | } 73 | 74 | select_node(NumaNodeIndex::new(node as u32)) 75 | } 76 | } 77 | 78 | /// Implementation of the ThreadLocal trait, for Linux. 79 | pub(crate) struct LLThreadLocal { 80 | key: atomic::AtomicI64, 81 | destructor: *const u8, 82 | _marker: PhantomData<*const T>, 83 | } 84 | 85 | impl LLThreadLocal { 86 | const UNINITIALIZED: i64 = -1; 87 | const UNDER_INITIALIZATION: i64 = -2; 88 | 89 | /// Creates an uninitialized instance. 90 | /// 91 | /// # Safety 92 | /// 93 | /// - Assumes that `destructor` points to an `unsafe extern "C" fn(*mut c_void)` function, or compatible. 94 | pub(crate) const unsafe fn new(destructor: *const u8) -> Self { 95 | let key = atomic::AtomicI64::new(-1); 96 | let _marker = PhantomData; 97 | 98 | LLThreadLocal { key, destructor, _marker } 99 | } 100 | 101 | #[inline(always)] 102 | fn get_key(&self) -> libc::pthread_key_t { 103 | let key = self.key.load(atomic::Ordering::Relaxed); 104 | if key >= 0 { key as libc::pthread_key_t} else { unsafe { self.initialize() } } 105 | } 106 | 107 | #[cold] 108 | #[inline(never)] 109 | unsafe fn initialize(&self) -> libc::pthread_key_t { 110 | const RELAXED: atomic::Ordering = atomic::Ordering::Relaxed; 111 | 112 | let mut key = self.key.load(RELAXED); 113 | 114 | if let Ok(_) = self.key.compare_exchange(Self::UNINITIALIZED, Self::UNDER_INITIALIZATION, RELAXED, RELAXED) 115 | { 116 | key = self.create_key(); 117 | self.key.store(key, RELAXED); 118 | } 119 | 120 | while key < 0 { 121 | libc::sched_yield(); 122 | key = self.key.load(RELAXED); 123 | } 124 | 125 | key as libc::pthread_key_t 126 | } 127 | 128 | #[cold] 129 | unsafe fn create_key(&self) -> i64 { 130 | let mut key: libc::pthread_key_t = 0; 131 | 132 | // Safety: 133 | // - fn pointers are just pointers. 134 | let destructor = mem::transmute::<_, Destructor>(self.destructor); 135 | let result = libc::pthread_key_create(&mut key as *mut _, Some(destructor)); 136 | assert!(result == 0, "Could not create thread-local key: {}", result); 137 | 138 | key as i64 139 | } 140 | } 141 | 142 | impl ThreadLocal for LLThreadLocal { 143 | fn get(&self) -> Option> { 144 | let key = self.key.load(atomic::Ordering::Relaxed); 145 | 146 | // If key is not initialized, then a null pointer is returned. 147 | NonNull::new(unsafe { libc::pthread_getspecific(key as libc::pthread_key_t) as *mut T }) 148 | } 149 | 150 | #[cold] 151 | #[inline(never)] 152 | fn set(&self, value: NonNull) { 153 | let key = self.get_key(); 154 | 155 | let result = unsafe { libc::pthread_setspecific(key, value.as_ptr() as *mut libc::c_void) }; 156 | assert!(result == 0, "Could not set thread-local value for {}: {}", key, result); 157 | } 158 | } 159 | 160 | unsafe impl Sync for LLThreadLocal {} 161 | 162 | type Destructor = unsafe extern "C" fn(*mut libc::c_void); 163 | 164 | // Selects the "best" node. 165 | // 166 | // The Linux kernel sometimes distinguishes nodes even though their distance is 11, when the distance to self is 10. 167 | // This may lead to over-allocation, hence it is judged best to "cluster" the nodes together. 168 | // 169 | // This function will therefore return the smallest node number whose distance to the `original` is less than or 170 | // equal to 11. 171 | fn select_node(original: NumaNodeIndex) -> NumaNodeIndex { 172 | let original = original.value() as i32; 173 | 174 | for current in 0..original { 175 | if unsafe { numa_distance(current, original) } <= 11 { 176 | return NumaNodeIndex::new(current as u32); 177 | } 178 | } 179 | 180 | NumaNodeIndex::new(original as u32) 181 | } 182 | 183 | // Attempts to allocate the required size in Huge Pages. 184 | // 185 | // If non-null, the result is aligned on `HUGE_PAGE_SIZE`. 186 | fn mmap_huge(size: usize) -> Option> { 187 | const MAP_HUGE_SHIFT: u8 = 26; 188 | 189 | const MAP_HUGE_1GB: libc::c_int = 30 << MAP_HUGE_SHIFT; 190 | 191 | mmap_allocate(size, libc::MAP_HUGETLB | MAP_HUGE_1GB) 192 | .and_then(|pointer| unsafe { mmap_check(pointer, size) }) 193 | } 194 | 195 | // Attempts to allocate the required size in Normal (or Large) Pages. 196 | // 197 | // If non-null, the result is aligned on `HUGE_PAGE_SIZE`. 198 | fn mmap_exact(size: usize) -> Option> { 199 | mmap_allocate(size, 0) 200 | .and_then(|pointer| unsafe { mmap_check(pointer, size) }) 201 | } 202 | 203 | // Attempts to allocate the required size in Normal (or Large) Pages. 204 | // 205 | // Ensures the alignment is met by over-allocated then trimming front and back. 206 | fn mmap_over(size: usize) -> Option> { 207 | const ALIGNMENT: PowerOf2 = LLConfiguration::HUGE_PAGE_SIZE; 208 | 209 | let over_size = size + ALIGNMENT.value(); 210 | let front_pointer = mmap_allocate(over_size, 0)?; 211 | 212 | let back_size = (front_pointer.as_ptr() as usize) % ALIGNMENT; 213 | let front_size = ALIGNMENT.value() - back_size; 214 | 215 | debug_assert!(front_size <= ALIGNMENT.value(), "{} > {}", front_size, ALIGNMENT.value()); 216 | debug_assert!(back_size < ALIGNMENT.value(), "{} >= {}", back_size, ALIGNMENT.value()); 217 | debug_assert!(front_size + size + back_size == over_size, 218 | "{} + {} + {} != {}", front_size, size, back_size, over_size); 219 | 220 | // Safety: 221 | // - `front_size` is less than `over_size`, hence the result is within the allocated block. 222 | let aligned_pointer = unsafe { front_pointer.as_ptr().add(front_size) }; 223 | 224 | debug_assert!(aligned_pointer as usize % ALIGNMENT == 0, 225 | "{:x} not {:x}-aligned!", aligned_pointer as usize, ALIGNMENT.value()); 226 | 227 | // Safety: 228 | // - `front_size + size` is less than `over_size`, hence the result is within the allocated block, 229 | // or pointing to its end. 230 | let back_pointer = unsafe { aligned_pointer.add(size) }; 231 | 232 | if front_size > 0 { 233 | // Safety: 234 | // - `front_pointer` points to a `mmap`ed area of at least `front_size` bytes. 235 | // - `[front_pointer, front_pointer + front_size)` is no longer in use. 236 | unsafe { munmap_deallocate(front_pointer.as_ptr(), front_size) }; 237 | } 238 | 239 | if back_size > 0 { 240 | // Safety: 241 | // - `back_pointer` points to a `mmap`ed area of at least `back_size` bytes. 242 | // - `[back_pointer, back_pointer + back_size)` is no longer in use. 243 | unsafe { munmap_deallocate(back_pointer, back_size) }; 244 | } 245 | 246 | NonNull::new(aligned_pointer) 247 | } 248 | 249 | // `mmap` alignment checker. 250 | // 251 | // Returns a non-null pointer if suitably aligned, and None otherwise. 252 | // If none is returned, the memory has been unmapped. 253 | // 254 | // # Safety 255 | // 256 | // - Assumes that `pointer` points to a `mmap`ed area of at least `size` bytes. 257 | // - Assumes that `pointer` is no longer in use, unless returned. 258 | unsafe fn mmap_check(pointer: NonNull, size: usize) -> Option> { 259 | const ALIGNMENT: PowerOf2 = LLConfiguration::HUGE_PAGE_SIZE; 260 | 261 | if pointer.as_ptr() as usize % ALIGNMENT == 0 { 262 | Some(pointer) 263 | } else { 264 | // Safety: 265 | // - `pointer` points to a `mmap`ed area of at least `size` bytes. 266 | // - `[pointer, pointer + size)` is no longer in use. 267 | munmap_deallocate(pointer.as_ptr(), size); 268 | None 269 | } 270 | } 271 | 272 | // Wrapper around `mmap`. 273 | // 274 | // Returns a pointer to `size` bytes of memory; does not guarantee any alignment. 275 | fn mmap_allocate(size: usize, extra_flags: i32) -> Option> { 276 | let length = size; 277 | let prot = libc::PROT_READ | libc::PROT_WRITE; 278 | let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | extra_flags; 279 | 280 | // No specific address hint. 281 | let addr = ptr::null_mut(); 282 | // When used in conjunction with MAP_ANONYMOUS, fd is mandated to be -1 on some implementations. 283 | let fd = -1; 284 | // When used in conjunction with MAP_ANONYMOUS, offset is mandated to be 0 on some implementations. 285 | let offset = 0; 286 | 287 | // Safety: 288 | // - `addr`, `fd`, and `offset` are suitable for MAP_ANONYMOUS. 289 | let result = unsafe { libc::mmap(addr, length, prot, flags, fd, offset) }; 290 | 291 | let result = if result != libc::MAP_FAILED { result as *mut u8 } else { ptr::null_mut() }; 292 | NonNull::new(result) 293 | } 294 | 295 | // Wrapper around `munmap`. 296 | // 297 | // # Panics 298 | // 299 | // If `munmap` returns a non-0 result. 300 | // 301 | // # Safety 302 | // 303 | // - Assumes that `addr` points to a `mmap`ed area of at least `size` bytes. 304 | // - Assumes that the range `[addr, addr + size)` is no longer in use. 305 | unsafe fn munmap_deallocate(addr: *mut u8, size: usize) { 306 | let result = libc::munmap(addr as *mut libc::c_void, size); 307 | assert!(result == 0, "Could not munmap {:x}, {}: {}", addr as usize, size, result); 308 | } 309 | 310 | #[link(name = "numa")] 311 | extern "C" { 312 | // Returns the NUMA node corresponding to a CPU, or -1 if the CPU is invalid. 313 | fn numa_node_of_cpu(cpu: i32) -> i32; 314 | 315 | // Returns the distance between two NUMA nodes. 316 | // 317 | // A node has a distance 10 to itself; factors should be multiples of 10, although 11 and 21 has been observed. 318 | fn numa_distance(left: i32, right: i32) -> i32; 319 | } 320 | -------------------------------------------------------------------------------- /doc/Design.md: -------------------------------------------------------------------------------- 1 | # Design 2 | 3 | This document presents some of the design decisions that went into llmalloc, and why. 4 | 5 | ## Why would anyone allocate, or deallocate, during a latency-sensitive operation? 6 | 7 | Truth to be told? Ideally, they should _not_. 8 | 9 | _In theory_, careful use of appropriately sized object pools, and other such strategies, can allow entire application 10 | to completely eschew memory allocations. 11 | 12 | _In practice_, such strategies are unwieldy, and not as well supported by languages and debugging tools, which makes 13 | them _costly_ in terms of developer time. 14 | 15 | In contrast, a low-latency memory allocator works _much better_: 16 | 17 | - Careful design allows avoiding allocation/deallocation in _most_ cases, so its raw performance is not as critical, 18 | though see the following point on _what low-latency means_. 19 | - Allocation/deallocation is well known, and well supported: standard smart pointers, standard containers, etc... 20 | - Allocation/deallocation is well supported. 21 | 22 | The latter point is critical, a compile-time switch to disable the low-latency memory allocator immediately enables the 23 | use of tools such as the various _saniziters_, or _valgrind_, to debug the various memory bugs -- as long as the 24 | low-latency allocator itself is bug free, of course. 25 | 26 | 27 | ## What does low-latency means? 28 | 29 | Low-latency is often misconstrued. Obviously, low-latency means focusing on latency metrics, and not throughput metrics, 30 | but which metrics? 31 | 32 | In my line of work, the median, modes, or average are of course of interest, but the focus is on _tail_ latencies. 33 | Indeed, often times, it is considered a good trade-off to see the median rise up by 10%, if it means improving the 90th, 34 | 95th, or 99th percentiles. Ideally, having a perfectly constant latency would be the bees knees. 35 | 36 | The reason is that tail latencies compound: if there are 2 components A and B, with A having a 90th percentile latency 37 | of 500ns, and B having a 90th percentile latency of 500ns, then in over 1% of cases the compound latency will be above 38 | 1us. 39 | 40 | A low-latency application developer will therefore focus on the _tail_ cases to decide whether a particular operation 41 | is suitable, or not, in the hot path. 42 | 43 | 44 | ## Hardware 45 | 46 | ### Non-Uniform Memory Architectures 47 | 48 | RAM stands for Random-Access Memory. Traditionally, Random-Access meant that access time was constant. 49 | 50 | Nowadays, memory accesses are _far_ from constant time, for multiple reasons: 51 | 52 | - RAM accesses have not actually been constant time for a long time. 53 | - CPU cache and Out Of Order execution may or may not compensate. 54 | - Cache Coherency Protocols may or may not result in inter-core traffic, based on cache state. 55 | 56 | NUMA further compounds the above issues: 57 | 58 | - There is a latency penalty in accessing another socket's or core's memory bank. 59 | - There is a latency penalty in coordinating cache coherency between sockets, compared to only between cores of a 60 | single socket. 61 | 62 | llmalloc has been design with Mechanical Sympathy in mind: 63 | 64 | - _Sockets_ are designed to map to NUMA nodes, guaranteeing that memory allocations are drawn from the current NUMA 65 | node memory banks. 66 | - Inter-_Socket_ communications are limited; and notably deallocations are buffered locally and returned in batches. 67 | 68 | 69 | ### Cache misses 70 | 71 | A cache miss is a latency penalty. llmalloc has no direct control over cache misses, but can still be cache friendly. 72 | 73 | Being friendly to the instruction cache: 74 | 75 | - Avoid bloat: llmalloc's code contains a single instance of the generic llmalloc-core structures and functions, 76 | avoiding monomorphization bloat. 77 | - Avoid eviction: llmalloc vies to clearly separate hot code from cold code, to avoid dragging in seldom used 78 | code into the instruction cache... and evict application code. 79 | 80 | Being friendly to the data cache: 81 | 82 | - Isolate local and foreign data: llmalloc's data layout clearly separates data that is read-only, thread-local, and 83 | global in order to avoid needless chatter between cores (and sockets). 84 | - Batch foreign accesses: in a producer-consumer scenario, memory is inevitably deallocated on a different thread, and 85 | possibly socket, than it was allocated. llmalloc batches the return of deallocated memory. This does not affect 86 | worst case latency, but does improve throughput. 87 | 88 | Being friendly to the Translation Look-aside Buffer (TLB): 89 | 90 | - Huge Pages: llmalloc's core organization mirrors Linux' concept of Huge Pages and Large Pages to play to the TLB 91 | strengths and minimize TLB misses. On a suitably configured server, all memory allocated by llmalloc will be located 92 | in a handful of Huge Pages which will permanently reside in the TLB. 93 | 94 | 95 | ### Atomic Read-Modify-Write 96 | 97 | Atomic operations are not a silver bullet: 98 | 99 | - Atomic operations _still_ require obtaining read/write access to the cache line, which may involve Cache Coherency 100 | communication across cores and sockets. 101 | - Atomic Read-Modify-Write operations tend to be expensive _even_ in the absence of contention. 102 | 103 | Careful design will attempt to reduce the number of atomic operations, and in particular to avoid Read-Modify-Write 104 | operations unless strictly necessary. Guarding a Read-Modify-Write operation by a simply load and a check can be worth 105 | it. 106 | 107 | 108 | ## Operating System 109 | 110 | ### System calls 111 | 112 | System calls are the anti-thesis of low-latency calls. _Some_ system calls are accelerated on Linux with VDSO, `mmap` 113 | and `munmap` are not of them. 114 | 115 | Kernel calls are expensive and unbounded, beyond the switch to (and fro) kernel mode, the kernel may also decide to 116 | perform arbitrary operations. For example, a request to allocate more memory may trigger the OOM killer. 117 | 118 | As a result, low-latency applications will prefer to perform kernel calls only during the _startup_ and _shutdown_ 119 | sequences, where latency does not matter yet, and _no other call_ during normal operations. 120 | 121 | llmalloc is design with such requirements in mind, allowing the application to pre-allocate Huge Pages ahead of time. 122 | 123 | This decision has a functional consequence: unlike traditional malloc libraries, memory is _never_ relinquished to the 124 | Operating System during normal operations. 125 | 126 | 127 | ## Library 128 | 129 | ### Adrift 130 | 131 | Casting pages adrift is llmalloc perhaps surprising solution to handling pages that are too full, and therefore must 132 | wait for memory blocks to be freed before being able to handle allocations again. 133 | 134 | The text-book solution would be to create a linked-list of such pages. When a page is too full, prepend or append it to 135 | the list, then periodically walk the list searching for "empty enough" pages that can be used again for allocation. The 136 | text-book solution is rather unsatisfactory as it involves an O(N) walk and contention between unrelated threads. 137 | 138 | Solving the O(N) walk is simple. Whenever a memory-block is freed within the page, the thread which frees the block can 139 | assess whether the page is now "empty enough" and if so remove it from the list. More interesting, though, is the new 140 | question this brings forth: if there is no reason to ever walk the list, why is there a list in the first place? 141 | 142 | The answer is that the list is _pointless_! 143 | 144 | Instead, llmalloc opts for casting pages adrift when they are too full, and catching them when they are ready to handle 145 | allocations again: 146 | 147 | - Each page contains an atomic boolean, indicating whether the page is adrift or not. 148 | - Upon exhausting the page, the allocating thread toggles the boolean and forgets about the page. 149 | - Any thread which returns memory blocks to the page checks if the page is adrift, and if so and enough memory blocks 150 | have been returned, catches the page and inserts it into the list of pages that are ready to handle allocations. 151 | 152 | There are some race conditions to solve, both between casting the page adrift and catching it, and between potential 153 | catchers, but the algorithm is still simple, and the principle is dead simple. 154 | 155 | And it solves _both_ problems compared to the text-book solution: 156 | 157 | - There is no O(N) behavior, both casting and catching are O(1). 158 | - Contention is localized, only threads actively allocating and deallocating on this very page are potentially 159 | contending. Multiple threads casting multiple pages adrift do not interact in any way. 160 | 161 | Only one source of contention is left: inserting the page back into the list of pages ready to handle allocations. It 162 | can be limited by maintaining multiple lists -- one for each class size. It is unclear whether it can be entirely 163 | eliminated; it may not be worth trying harder though. 164 | 165 | 166 | ### Batches 167 | 168 | In general, a simple way to improve throughput, at the cost of increasing the variance of latency, is to batch 169 | operations. Batches tend to play to a modern CPU strengths, and notably tend to be cache friendly. 170 | 171 | llmalloc is a low-latency memory allocator, and is therefore primarily concerned about _latency_ rather than throughput. 172 | As a result, llmalloc does _not_ use "amortized O(1)" algorithms. 173 | 174 | llmalloc only uses _one_ batch operation, to reduce contention on deallocation. Deallocations are locally buffered, 175 | and returned in a batch. 176 | 177 | This particular operation, however, consists in pre-pending a singly-linked list to another singly-linked list, and 178 | careful care has been taken for this operation to be O(1) with regard to the number of elements in either linked list 179 | by caching the tail pointer of the list to be pre-pended. 180 | 181 | 182 | ### Contention 183 | 184 | llmalloc leans heavily on thread-local data, and a thread-local cache of memory, to avoid contention. 185 | 186 | _In theory_, doing so does not improve tail latencies, as it is always possible that _all_ cores could come to contend 187 | at the same time. 188 | 189 | _In practice_, sufficiently large thread-local caches do wonders to reduce the number of cores which actually come to 190 | contend at any given time, and the more "random" the allocation and deallocation patterns of the application are, the 191 | less contention is observed. 192 | 193 | 194 | ### Deallocation 195 | 196 | malloc implementations tend to favor the performance of memory allocation at the expense of memory deallocation. 197 | Expensive operations, such as consolidation of memory, is typically performed only during deallocation, leading to 198 | a much higher latency variance for deallocation than for allocation. 199 | 200 | llmalloc, instead, strives to treat both allocation and deallocation equally. Indeed, the very design of llmalloc has 201 | been geared towards efficient deallocation: by making use of pointer alignment to efficiently process them. 202 | 203 | 204 | ### Lazy Initialization 205 | 206 | Normal allocations in llmalloc are catered to by Large Pages. A single Large Page is initialized with a class size, 207 | which determines the size of the memory blocks it will return, and the page will maintain a _stack_ of intrusively 208 | linked memory blocks. 209 | 210 | On x64/linux, a Large Page is 2MB and the smallest class size is 32 bytes. Given a header of about 1KB, this 211 | translates into 65,504 memory blocks; which means that building the stack from scratch, even at an astounding 1ns per 212 | element, would require about 65 _micro_ seconds. 65 times [an eternity](https://www.youtube.com/watch?v=NH1Tta7purM). 213 | 214 | There are 2 solutions to the problem: 215 | 216 | - Preparing the Large Pages in advance. 217 | - Not building the entire stack at once. 218 | 219 | Preparing the Large Pages in advance would be ideal, I suppose. My experience has been that it is impractical. 220 | 221 | Instead, llmalloc uses a _watermark_ to indicate up to which points memory blocks have been allocated: 222 | 223 | - Memory before the watermark is managed by the stack. 224 | - Memory after the watermark has never been touched. 225 | 226 | Then, anytime the stack is exhausted, it checks whether the watermark has reached the end of the page, and if not it 227 | carves one memory block out of the raw memory, and bumps the watermark for next time. 228 | 229 | 230 | ### Wait Freedom 231 | 232 | In general, lock freedom and wait freedom are not strictly necessary to guarantee low-latency in the context of 233 | llmalloc: no application code is executed within the library itself. 234 | 235 | There is one wrinkle there: the operating system is still free to interrupt an application thread at any point. 236 | Disabling interrupts is one possibility, but it has its own cost. 237 | 238 | llmalloc uses wait free algorithms instead, guaranteeing that even if _all but one_ threads are interrupted allocations 239 | and deallocations on the non-interrupted thread can still proceed. At least as long as allocating, or deallocating, a 240 | Huge Page is not necessary. 241 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/blocks/atomic_block_foreign_list.rs: -------------------------------------------------------------------------------- 1 | //! Intrusive Linked List of AtomicBlockForeign. 2 | 3 | use core::{ 4 | ops::Deref, 5 | ptr::NonNull, 6 | sync::atomic::{self, Ordering}, 7 | }; 8 | 9 | use super::{AtomicPtr, AtomicBlockForeign, BlockForeignList}; 10 | 11 | /// AtomicBlockForeignList. 12 | #[derive(Default)] 13 | pub(crate) struct AtomicBlockForeignList(AtomicPtr); 14 | 15 | impl AtomicBlockForeignList { 16 | /// Returns the length of the tail list. 17 | pub(crate) fn len(&self) -> usize { 18 | let head = self.0.load(); 19 | 20 | head.map(|head| unsafe { head.as_ref() }.length.load() + 1) 21 | .unwrap_or(0) 22 | } 23 | 24 | /// Steals the content of the list. 25 | pub(crate) fn steal(&self) -> Option> { self.0.exchange(None) } 26 | 27 | /// Extends the tail-list pointed to by prepending `list`, atomically. 28 | /// 29 | /// Returns the size of the new list. 30 | /// 31 | /// # Safety 32 | /// 33 | /// - Assumes the list is not empty. 34 | pub(crate) unsafe fn extend(&self, list: &BlockForeignList) -> usize { 35 | debug_assert!(!list.is_empty()); 36 | 37 | let additional_length = list.len(); 38 | 39 | // Safety: 40 | // - The list is assumed not to be empty. 41 | let (head, tail) = list.steal(); 42 | 43 | atomic::fence(Ordering::Release); 44 | 45 | // Safety: 46 | // - Access to the list blocks is exclusive. 47 | // - A Release atomic fence was called after the last write to the `BlockForeign` list. 48 | let (head, tail) = (AtomicBlockForeign::from(head), AtomicBlockForeign::from(tail)); 49 | 50 | let mut current = self.0.load(); 51 | 52 | loop { 53 | let current_length = if let Some(current_ptr) = current { 54 | // Safety: 55 | // - `current` is not null. 56 | current_ptr.as_ref().length.load() + 1 57 | } else { 58 | 0 59 | }; 60 | 61 | // Safety: 62 | // - Bounded lifetime. 63 | tail.as_ref().next.store(current); 64 | tail.as_ref().length.store(current_length); 65 | 66 | // Safety: 67 | // - Bounded lifetime. 68 | head.as_ref().length.store(current_length + additional_length - 1); 69 | 70 | // Technical ABA. 71 | // 72 | // This CAS exhibits an ABA issue: 73 | // 1. Thread A reads `current` above, sets it as the tail, pauses. 74 | // 2. Thread B steals `current`, then reintroduces it. 75 | // 3. Thread A performs the CAS successfully, but returns the wrong length. 76 | // 77 | // Practically speaking, though, due to AtomicBlockForeignList only being used in LargePage::Foreign, this 78 | // is rather implausible. 79 | // 80 | // Specifically, step (2) actually requires thread B, the owner of the LargePage, to recycle the blocks, 81 | // allocate it, transfer it to another thread C, which deallocates it, overflows its own local collection, 82 | // and returns it to thread B's LargePage::Foreign. 83 | // 84 | // Furthermore, even if it _did_ happen, the list itself wouldn't be incorrect. Only its _length_ would be. 85 | match self.0.compare_exchange(current, Some(head)) { 86 | Ok(_) => return current_length + additional_length, 87 | Err(new_current) => current = new_current, 88 | } 89 | } 90 | } 91 | } 92 | 93 | impl Deref for AtomicBlockForeignList { 94 | type Target = AtomicPtr; 95 | 96 | fn deref(&self) -> &Self::Target { &self.0 } 97 | } 98 | 99 | #[cfg(test)] 100 | mod tests { 101 | 102 | use std::ops::Range; 103 | 104 | use llmalloc_test::BurstyBuilder; 105 | 106 | use super::*; 107 | use super::super::{AlignedArray, BlockForeign, BlockForeignList, BlockLocal, BlockLocalStack}; 108 | 109 | fn create_list(array: &AlignedArray, range: Range) -> BlockForeignList { 110 | let list = BlockForeignList::default(); 111 | 112 | for index in range.rev() { 113 | list.push(array.get(index)); 114 | } 115 | 116 | list 117 | } 118 | 119 | #[test] 120 | fn block_foreign_ptr_extend_steal() { 121 | let array = AlignedArray::::default(); 122 | let (a, b, c) = (array.get(0), array.get(1), array.get(2)); 123 | let (x, y, z) = (array.get(10), array.get(11), array.get(12)); 124 | 125 | let list = BlockForeignList::default(); 126 | 127 | list.push(c); 128 | list.push(b); 129 | list.push(a); 130 | 131 | let foreign = AtomicBlockForeignList::default(); 132 | assert_eq!(0, foreign.len()); 133 | 134 | assert_eq!(3, unsafe { foreign.extend(&list) }); 135 | assert_eq!(3, foreign.len()); 136 | 137 | list.push(z); 138 | list.push(y); 139 | list.push(x); 140 | 141 | assert_eq!(6, unsafe { foreign.extend(&list) }); 142 | assert_eq!(6, foreign.len()); 143 | 144 | let head = foreign.steal(); 145 | assert_eq!(0, foreign.len()); 146 | 147 | // Double-check the list! 148 | let local = BlockLocalStack::default(); 149 | unsafe { local.refill(BlockLocal::from_atomic(head.unwrap())) }; 150 | 151 | assert_eq!(Some(x.cast()), local.pop()); 152 | assert_eq!(Some(y.cast()), local.pop()); 153 | assert_eq!(Some(z.cast()), local.pop()); 154 | assert_eq!(Some(a.cast()), local.pop()); 155 | assert_eq!(Some(b.cast()), local.pop()); 156 | assert_eq!(Some(c.cast()), local.pop()); 157 | assert_eq!(None, local.pop()); 158 | } 159 | 160 | #[test] 161 | fn atomic_block_foreign_list_concurrent_steal_fuzzing() { 162 | // This test aims at validating that multiple threads can steal concurrently. 163 | // 164 | // To do so: 165 | // - The list is prepared with a list. 166 | // - Each thread attempts to steal from it, the successful one signals it succeeded. 167 | // - The fact that one, and only one, thread succeeded is checked. 168 | struct Global { 169 | victim: AtomicBlockForeignList, 170 | pointer: NonNull, 171 | thieves: atomic::AtomicUsize, 172 | } 173 | 174 | unsafe impl Send for Global {} 175 | unsafe impl Sync for Global {} 176 | 177 | let array = AlignedArray::::default(); 178 | 179 | let pointer = { 180 | let list = create_list(&array, 0..3); 181 | 182 | let head = unsafe { list.steal().0 }; 183 | 184 | atomic::fence(Ordering::Release); 185 | 186 | unsafe { AtomicBlockForeign::from(head) } 187 | }; 188 | 189 | let global = Global { victim: AtomicBlockForeignList::default(), pointer, thieves: atomic::AtomicUsize::default() }; 190 | 191 | let mut builder = BurstyBuilder::new(global, vec!(true, false, false, false)); 192 | 193 | // Step 1: setup the list. 194 | builder.add_simple_step(|| |global: &Global, local: &mut bool| { 195 | if *local { 196 | global.victim.store(Some(global.pointer)); 197 | global.thieves.store(0, Ordering::Relaxed); 198 | } 199 | }); 200 | 201 | // Step 2: attempt to steal from all threads. 202 | builder.add_simple_step(|| |global: &Global, _: &mut bool| { 203 | if let Some(stolen) = global.victim.steal() { 204 | assert_eq!(global.pointer, stolen); 205 | 206 | global.thieves.fetch_add(1, Ordering::Relaxed); 207 | } 208 | }); 209 | 210 | // Step 3: validate one, and only one, would-be-thief succeeded. 211 | builder.add_simple_step(|| |global: &Global, local: &mut bool| { 212 | if *local { 213 | global.victim.store(Some(global.pointer)); 214 | global.thieves.store(0, Ordering::Relaxed); 215 | } 216 | }); 217 | 218 | builder.launch(100); 219 | } 220 | 221 | #[test] 222 | fn atomic_block_foreign_list_concurrent_extend_fuzzing() { 223 | // This test aims at validating that multiple threads can extend concurrently. 224 | // 225 | // To do so: 226 | // - Each thread creates a BlockForeignList, from a dedicated range. 227 | // - Each thread attempts at extending the AtomicBlockForeignList, storing the maximum length globally. 228 | // - The maximum length is checked. 229 | #[derive(Default)] 230 | struct Global { 231 | victim: AtomicBlockForeignList, 232 | maximum_length: atomic::AtomicUsize, 233 | } 234 | 235 | struct Local { 236 | array: NonNull>, 237 | range: Range, 238 | } 239 | 240 | impl Local { 241 | fn new(array: &AlignedArray, range: Range) -> Self { 242 | Local { array: NonNull::from(array), range, } 243 | } 244 | } 245 | 246 | // Safety: 247 | // - We are going to be very careful with what we do. Promise. 248 | unsafe impl Send for Local {} 249 | 250 | let array = AlignedArray::::default(); 251 | 252 | let mut builder = BurstyBuilder::new(Global::default(), 253 | vec!(Local::new(&array, 0..3), Local::new(&array, 3..6), Local::new(&array, 6..9), Local::new(&array, 9..12))); 254 | 255 | // Step 1: Reset. 256 | builder.add_simple_step(|| |global: &Global, _: &mut Local| { 257 | global.victim.store(None); 258 | global.maximum_length.store(0, Ordering::Relaxed); 259 | }); 260 | 261 | // Step 2: Prepare list and extend. 262 | builder.add_complex_step(|| { 263 | let prep = |_: &Global, local: &mut Local| { 264 | create_list(unsafe { local.array.as_ref() }, local.range.clone()) 265 | }; 266 | let step = |global: &Global, _: &mut Local, list: BlockForeignList| { 267 | let length = unsafe { global.victim.extend(&list) }; 268 | global.maximum_length.fetch_max(length, Ordering::Relaxed); 269 | }; 270 | (prep, step) 271 | }); 272 | 273 | // Step 3: Check maximum length. 274 | builder.add_simple_step(|| |global: &Global, _: &mut Local| { 275 | assert_eq!(12, global.maximum_length.load(Ordering::Relaxed)); 276 | }); 277 | 278 | builder.launch(100); 279 | } 280 | 281 | #[test] 282 | fn atomic_block_foreign_list_concurrent_extend_steal_fuzzing() { 283 | // This test aims at validating that multiple threads can extend _and_ steal concurrently. 284 | // 285 | // To do so: 286 | // - Each thread is designed as either producer, or consumer, based on whether the range they are given is empty. 287 | // - Each producer thread attempts to extend the list, whilst each consumer thread attempts to steal from it. 288 | // Additionally, consumer threads total how much they stole. 289 | // - The total of list-length and stolen is checked. 290 | #[derive(Default)] 291 | struct Global { 292 | victim: AtomicBlockForeignList, 293 | stolen: atomic::AtomicUsize, 294 | } 295 | 296 | struct Local { 297 | array: NonNull>, 298 | range: Range, 299 | } 300 | 301 | impl Local { 302 | fn new(array: &AlignedArray, range: Range) -> Self { 303 | Local { array: NonNull::from(array), range, } 304 | } 305 | } 306 | 307 | // Safety: 308 | // - We are going to be very careful with what we do. Promise. 309 | unsafe impl Send for Local {} 310 | 311 | let array = AlignedArray::::default(); 312 | 313 | let mut builder = BurstyBuilder::new(Global::default(), 314 | vec!(Local::new(&array, 0..6), Local::new(&array, 6..6), Local::new(&array, 6..12), Local::new(&array, 12..12))); 315 | 316 | // Step 1: Reset. 317 | builder.add_simple_step(|| |global: &Global, _: &mut Local| { 318 | global.victim.store(None); 319 | global.stolen.store(0, Ordering::Relaxed); 320 | }); 321 | 322 | // Step 2: Prepare list and extend. 323 | builder.add_complex_step(|| { 324 | let prep = |_: &Global, local: &mut Local| { 325 | create_list(unsafe { local.array.as_ref() }, local.range.clone()) 326 | }; 327 | let step = |global: &Global, _: &mut Local, list: BlockForeignList| { 328 | if list.is_empty() { 329 | // Consumer 330 | if let Some(stolen) = global.victim.steal() { 331 | let total = unsafe { stolen.as_ref().length.load() + 1 }; 332 | global.stolen.fetch_add(total, Ordering::Relaxed); 333 | } 334 | } else { 335 | // Producer 336 | unsafe { global.victim.extend(&list) }; 337 | } 338 | }; 339 | (prep, step) 340 | }); 341 | 342 | // Step 3: Check maximum length. 343 | builder.add_simple_step(|| |global: &Global, _: &mut Local| { 344 | let remaining = global.victim.len(); 345 | let stolen = global.stolen.load(Ordering::Relaxed); 346 | 347 | assert_eq!(12, remaining + stolen, 348 | "remaining: {}, stolen: {}", remaining, stolen); 349 | }); 350 | 351 | builder.launch(100); 352 | } 353 | 354 | } // mod tests 355 | -------------------------------------------------------------------------------- /llmalloc-core/src/internals/atomic_stack.rs: -------------------------------------------------------------------------------- 1 | //! A wait-free intrusively-linked stack. 2 | //! 3 | //! # Warning 4 | //! 5 | //! Writing a proper wait-free stack is actually far more difficult than one would expect in the absence of 6 | //! [Load-Link/Store-Conditional](https://en.wikipedia.org/wiki/Load-link/store-conditional) or Double-CAS due to 7 | //! ABA issues around manipulation of the top pointer. 8 | //! 9 | //! This stack attempts to circumvent the issue using a regular CAS by using a Merkle-Chain, however this is definitely 10 | //! not foolproof, and the quality of it depends on the platform and the alignment of the linked elements. 11 | //! 12 | //! For best protection: 13 | //! 14 | //! - Use on platforms with as many "free" top-bits as possible; for example x64 has 17 free top bits in user-space. 15 | //! - Use the highest alignment possible, to gain even more bits. 16 | //! - Use only in situations when popping-pushing the same element in quick sequence is unlikely. 17 | //! 18 | //! # Safety 19 | //! 20 | //! The stack assumes that the elements stashed within are: 21 | //! 22 | //! - Exclusively accessible through the stack; ie, the stack _owns_ them. 23 | //! - Will outlive the stack. 24 | //! 25 | //! See the signature of `push`. 26 | 27 | use core::{ 28 | marker::PhantomData, 29 | ptr::NonNull, 30 | sync::atomic::{AtomicUsize, Ordering}, 31 | }; 32 | 33 | use crate::PowerOf2; 34 | 35 | /// AtomicStackElement 36 | pub(crate) trait AtomicStackElement : Sized { 37 | /// Alignment of the element. 38 | const ALIGNMENT: PowerOf2 = PowerOf2::align_of::(); 39 | 40 | /// Next link. 41 | fn next(&self) -> &AtomicStackLink; 42 | } 43 | 44 | /// AtomicStack 45 | pub(crate) struct AtomicStack(AtomicStackLink); 46 | 47 | impl AtomicStack { 48 | /// Creates an empty instance of the AtomicStack. 49 | pub(crate) fn new() -> Self { Self(AtomicStackLink::default()) } 50 | 51 | /// Checks whether the stack is empty, or not. 52 | #[cfg(test)] 53 | pub(crate) fn is_empty(&self) -> bool { self.0.is_null() } 54 | } 55 | 56 | impl AtomicStack { 57 | /// Pops the top element, if any. 58 | /// 59 | /// The returned `NonNull`, if any, is guaranteed to have exclusive access to its pointee. 60 | pub(crate) fn pop(&self) -> Option> { 61 | // WARNING: 62 | // 63 | // Due to concurrency, another thread may start using the data pointed to be `head` prior to this call 64 | // terminating. 65 | // 66 | // DO NOT WRITE to `head`/`current` before having exclusive ownership of it. 67 | let mut current = self.0.load(); 68 | 69 | loop { 70 | // Safety: 71 | // - `current` was obtained from `Self::pack` or is 0. 72 | let head = unsafe { Self::unpack(current) }?; 73 | 74 | // Safety: 75 | // - `head` is a valid pointer, to a valid `T`. 76 | // 77 | // There is a degree of uncertainy as to whether `head.as_ref()` is fine, in isolation. There is no 78 | // guarantee that a mutable reference to `*head` doesn't exist at this point, so technically it may 79 | // be invalid to form a `&T`, even though in practice we only access an atomic field. 80 | let next = unsafe { head.as_ref().next().load() }; 81 | 82 | if let Err(new_current) = self.0.compare_exchange_weak(current, next) { 83 | current = new_current; 84 | continue; 85 | } 86 | 87 | // Safety: 88 | // - `head` is a valid pointer, to a valid `T`. 89 | unsafe { head.as_ref().next().store(0) }; 90 | 91 | return Some(head); 92 | } 93 | } 94 | 95 | /// Pushes an element at the top. 96 | pub(crate) fn push<'a, 'b>(&'a self, element: &'b mut T) 97 | where 98 | 'b: 'a, 99 | { 100 | // WARNING: 101 | // 102 | // Due to concurrency, another thread may start using the data pointed to be `head` prior to this call 103 | // terminating. 104 | // 105 | // DO NOT WRITE to `head`/`current`. 106 | let element = NonNull::from(element); 107 | 108 | let mut current = self.0.load(); 109 | 110 | loop { 111 | // Safety: 112 | // - `element` is valid. 113 | // - Lifetime is bound. 114 | unsafe { element.as_ref() }.next().store(current); 115 | 116 | let hash = fnv(current); 117 | let packed = Self::pack(element, hash); 118 | 119 | if let Err(new_current) = self.0.compare_exchange_weak(current, packed) { 120 | current = new_current; 121 | continue; 122 | } 123 | 124 | break; 125 | } 126 | } 127 | 128 | // Internal. 129 | // FIXME: `const` once rustc grows up. 130 | fn merkle_bits() -> usize { 131 | T::ALIGNMENT.value().trailing_zeros() as usize + TOP_POINTER_BITS 132 | } 133 | 134 | // Internal. 135 | fn pack(pointer: NonNull, next: usize) -> usize { 136 | // FIXME: `const` once rustc grows up. 137 | let merkle_mask: usize = (1usize << Self::merkle_bits()) - 1; 138 | 139 | let pointer = (pointer.as_ptr() as usize) << TOP_POINTER_BITS; 140 | 141 | pointer | (next & merkle_mask) 142 | } 143 | 144 | // Internal. 145 | // 146 | // # Safety: 147 | // 148 | // - Assumes that `element` was obtained by `pack`, or is 0. 149 | unsafe fn unpack(element: usize) -> Option> { 150 | // FIXME: `const` once rustc grows up. 151 | let merkle_mask: usize = (1usize << Self::merkle_bits()) - 1; 152 | 153 | if element == 0 { 154 | return None; 155 | } 156 | 157 | let pointer = (element & !merkle_mask) >> TOP_POINTER_BITS; 158 | debug_assert_ne!(0, pointer, "element: {}, merkle: {}, top: {}", element, Self::merkle_bits(), TOP_POINTER_BITS); 159 | 160 | // Safety: 161 | // - `element` was obtained by packing a pointer. 162 | NonNull::new(pointer as *mut T) 163 | } 164 | } 165 | 166 | impl Default for AtomicStack { 167 | fn default() -> Self { Self::new() } 168 | } 169 | 170 | /// AtomicStackLink 171 | pub(crate) struct AtomicStackLink(AtomicUsize, PhantomData); 172 | 173 | impl AtomicStackLink { 174 | fn new() -> Self { Self(AtomicUsize::new(0), PhantomData) } 175 | 176 | #[cfg(test)] 177 | fn is_null(&self) -> bool { self.0.load(Ordering::Relaxed) == 0 } 178 | 179 | fn load(&self) -> usize { self.0.load(Ordering::Acquire) } 180 | 181 | fn store(&self, payload: usize) { self.0.store(payload, Ordering::Release) } 182 | 183 | fn compare_exchange_weak(&self, current: usize, value: usize) -> Result { 184 | self.0.compare_exchange_weak(current, value, Ordering::AcqRel, Ordering::Acquire) 185 | } 186 | } 187 | 188 | impl Default for AtomicStackLink { 189 | fn default() -> Self { Self::new() } 190 | } 191 | 192 | // 193 | // Implementation 194 | // 195 | 196 | #[cfg(target_arch = "x86_64")] 197 | const TOP_POINTER_BITS: usize = 17; 198 | 199 | #[cfg(not(target_arch = "x86_64"))] 200 | const TOP_POINTER_BITS: usize = 0; 201 | 202 | fn fnv(data: usize) -> usize { 203 | let mut hash: usize = 0xcbf29ce484222325; 204 | 205 | for i in 0..7 { 206 | hash = hash.wrapping_mul(0x100000001b3); 207 | hash ^= (data >> (8 * i)) & 0xFF; 208 | } 209 | 210 | hash 211 | } 212 | 213 | #[cfg(test)] 214 | mod tests { 215 | 216 | use std::mem; 217 | 218 | use llmalloc_test::BurstyBuilder; 219 | 220 | use super::*; 221 | 222 | type Stack = AtomicStack; 223 | 224 | #[repr(align(128))] 225 | #[derive(Default)] 226 | struct Element { 227 | next: AtomicStackLink, 228 | } 229 | 230 | impl AtomicStackElement for Element { 231 | fn next(&self) -> &AtomicStackLink { &self.next } 232 | } 233 | 234 | #[test] 235 | fn atomic_stack_send_sync() { 236 | fn ensure_send() {} 237 | fn ensure_sync() {} 238 | 239 | ensure_send::(); 240 | ensure_sync::(); 241 | } 242 | 243 | #[test] 244 | fn atomic_stack_pop_push() { 245 | let array = [Element::default(), Element::default()]; 246 | let (a, b) = (NonNull::from(&array[0]), NonNull::from(&array[1])); 247 | 248 | let stack = Stack::default(); 249 | 250 | assert_eq!(None, stack.pop()); 251 | assert!(stack.is_empty()); 252 | 253 | unsafe { 254 | stack.push(&mut *a.as_ptr()); 255 | stack.push(&mut *b.as_ptr()); 256 | } 257 | 258 | assert!(!stack.is_empty()); 259 | 260 | assert_eq!(Some(b), stack.pop()); 261 | assert_eq!(Some(a), stack.pop()); 262 | assert_eq!(None, stack.pop()); 263 | assert!(stack.is_empty()); 264 | } 265 | 266 | #[test] 267 | fn atomic_stack_concurrent_push_concurrent_pop_fuzzing() { 268 | // The test aims at validating that: 269 | // - Multiple threads can push concurrently. 270 | // - Multiple threads can pop concurrently. 271 | // 272 | // To do so: 273 | // - Each thread is given one element. 274 | // - Each thread will repeatedly push this element on the stack, then pop a random element. 275 | // - Each thread will assert that they did manage to pop an element. 276 | // - At the end of the test, the stack should be empty. 277 | #[derive(Default)] 278 | struct Local(Element); 279 | 280 | // Safety: 281 | // - Guaranteed to have exclusive access to its `element`. 282 | unsafe impl Send for Local {} 283 | 284 | let elements = vec!(Local::default(), Local::default(), Local::default(), Local::default()); 285 | 286 | let mut builder = BurstyBuilder::new(Stack::default(), elements); 287 | 288 | // Step 1: Push. 289 | builder.add_simple_step(|| |stack: &Stack, local: &mut Local| { 290 | // Safety: 291 | // - Let's not access stack elements after Local dies, eh? 292 | stack.push(unsafe { mem::transmute(&mut local.0) }); 293 | }); 294 | 295 | // Step 2: Pop one of the pushed elements. 296 | builder.add_simple_step(|| |stack: &Stack, _: &mut Local| { 297 | let element = stack.pop(); 298 | assert_ne!(None, element); 299 | }); 300 | 301 | // Step 3: There should be nothing to pop. 302 | builder.add_simple_step(|| |stack: &Stack, _: &mut Local| { 303 | let element = stack.pop(); 304 | assert_eq!(None, element); 305 | }); 306 | 307 | builder.launch(100); 308 | } 309 | 310 | #[test] 311 | fn atomic_stack_concurrent_push_pop_fuzzing() { 312 | // The test aims at validating that multiple threads can push _and_ pop concurrently. 313 | // 314 | // To do so: 315 | // - Each thread is given an element. 316 | // - In the first step, even threads attempt to push, whilst odd threads attempt to pop. 317 | // - In the second step, the roles are reversed. 318 | // - Each thread assert that they did manage to pop an element. 319 | // - The stack must therefore be primed with the elements from the odd threads, as they pop first. 320 | #[derive(Default)] 321 | struct Local { 322 | index: usize, 323 | element: Option>, 324 | } 325 | 326 | impl Local { 327 | fn is_pop_then_push(&self) -> bool { self.index % 2 != 0 } 328 | } 329 | 330 | // Safety: 331 | // - Guaranteed to have exclusive access to its `element`. 332 | unsafe impl Send for Local {} 333 | 334 | let store = vec!(Element::default(), Element::default(), Element::default(), Element::default()); 335 | 336 | let elements: Vec<_> = store.iter() 337 | .enumerate() 338 | .map(|(index, element)| { 339 | Local { index, element: Some(NonNull::from(element)), } 340 | }) 341 | .collect(); 342 | 343 | let number_threads = elements.len(); 344 | let half_threads = number_threads / 2; 345 | assert_eq!(number_threads, half_threads * 2); 346 | 347 | let mut builder = BurstyBuilder::new(Stack::default(), elements); 348 | 349 | // Step 1: Push elements from half of the threads, ready to pop. 350 | builder.add_simple_step(|| |stack: &Stack, local: &mut Local| { 351 | if local.is_pop_then_push() { 352 | // Safety: 353 | // - I promise to try not to access stack elements after `local` has died... 354 | stack.push(unsafe { &mut *local.element.unwrap().as_ptr() }); 355 | } 356 | }); 357 | 358 | // Step 2: Pop/Push from half of the threads. 359 | builder.add_simple_step(|| |stack: &Stack, local: &mut Local| { 360 | if local.is_pop_then_push() { 361 | local.element = stack.pop(); 362 | assert_ne!(None, local.element); 363 | } else { 364 | // Safety: 365 | // - I promise to try not to access stack elements after `local` has died... 366 | stack.push(unsafe { &mut *local.element.unwrap().as_ptr() }); 367 | } 368 | }); 369 | 370 | // Step 3: Pop/Push from the other half of the threads. 371 | builder.add_simple_step(|| |stack: &Stack, local: &mut Local| { 372 | if local.is_pop_then_push() { 373 | // Safety: 374 | // - I promise to try not to access stack elements after `local` has died... 375 | stack.push(unsafe { &mut *local.element.unwrap().as_ptr() }); 376 | } else { 377 | local.element = stack.pop(); 378 | assert_ne!(None, local.element); 379 | } 380 | }); 381 | 382 | // Step 4: Drain from the early pushers. 383 | builder.add_simple_step(|| |stack: &Stack, local: &mut Local| { 384 | if local.is_pop_then_push() { 385 | local.element = stack.pop(); 386 | assert_ne!(None, local.element); 387 | } 388 | }); 389 | 390 | // Step 5: Ensure the stack is drained. 391 | builder.add_simple_step(|| |stack: &Stack, _: &mut Local| { 392 | let element = stack.pop(); 393 | assert_eq!(None, element); 394 | 395 | assert!(stack.is_empty()); 396 | }); 397 | 398 | builder.launch(100); 399 | } 400 | 401 | } // mod tests 402 | -------------------------------------------------------------------------------- /llmalloc/tests/multi.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | alloc::Layout, 3 | collections::BTreeSet, 4 | env, 5 | mem, 6 | ops::{Deref, DerefMut}, 7 | ptr::{self, NonNull}, 8 | sync::{Arc, Mutex, atomic::{AtomicUsize, Ordering}}, 9 | thread::{self, JoinHandle}, 10 | }; 11 | 12 | use serial_test::serial; 13 | 14 | use llmalloc::LLAllocator; 15 | 16 | static LL_ALLOCATOR: LLAllocator = LLAllocator::new(); 17 | 18 | // 19 | // Tests 20 | // 21 | 22 | #[serial] 23 | #[test] 24 | fn acquire_release_thread_handles() { 25 | // Test that acquiring/releasing thread handles from multiple threads concurrently works as expected. 26 | 27 | let number_iterations = number_iterations(); 28 | let number_threads = number_threads(); 29 | 30 | let mut thread_handles = BTreeSet::new(); 31 | 32 | for _ in 0..number_iterations { 33 | let start = RendezVous::new("start", number_threads); 34 | let end = RendezVous::new("end", number_threads); 35 | 36 | let pool = Pool::new(number_threads, |i| { 37 | let start = start.clone(); 38 | let end = end.clone(); 39 | 40 | move || { 41 | start.wait_until_all_ready(); 42 | 43 | let pointer = Pointer::new(i); 44 | 45 | end.wait_until_all_ready(); 46 | 47 | // Sanity check, to ensure no other thread allocate the same pointer. 48 | assert_eq!(i, *pointer); 49 | 50 | LL_ALLOCATOR.thread_index() 51 | } 52 | }); 53 | 54 | let results = pool.join(); 55 | 56 | // Ensure that no 2 threads obtained the same thread-local handle. 57 | let local_handles: BTreeSet<_> = results.iter().cloned().collect(); 58 | 59 | assert_eq!(number_threads, local_handles.len()); 60 | 61 | // Ensure that thread-local handles are reused across invocations: 62 | // - This ensures they are freed on thread destruction. 63 | // - This ensures they are reused, to avoid running out. 64 | if thread_handles.is_empty() { 65 | let _ = std::mem::replace(&mut thread_handles, local_handles); 66 | continue; 67 | } 68 | 69 | assert_eq!(thread_handles, local_handles); 70 | } 71 | } 72 | 73 | #[serial] 74 | #[test] 75 | fn producer_consumer_ring() { 76 | // Test that blocks can be concurrenty allocated and deallocated, including deallocated on a separate thread. 77 | // 78 | // The test is slightly convoluted, so a high-level overview is provided: 79 | // 80 | // 1. "Victims" are prepared, those are numbers 0 to N made into `String`. 81 | // 2. Concurrently (synchronized) those victims are moved into `Pointer`, each requiring an allocation. 82 | // 3. The allocations are "shuffled", so that each vector contains pointers from alternating allocating threads. 83 | // 4. Concurrently (synchronized) those shuffled pointers are freed, and their values recovered. 84 | // 5. Check the recovered values match the originals, to ensure no corruption occurred. 85 | // 6. The rendez_vous are reset, for the next iteration. 86 | 87 | fn create_victims(number: usize) -> Vec { (0..number).map(|i| i.to_string()).collect() } 88 | 89 | fn push_victims(victims: Vec, sink: &mut Vec>) { 90 | debug_assert!(sink.is_empty()); 91 | 92 | for victim in victims { 93 | sink.push(Pointer::new(victim)); 94 | } 95 | } 96 | 97 | fn pop_victims(stream: &mut Vec>, sink: &mut Vec) { 98 | debug_assert!(stream.len() <= sink.capacity()); 99 | 100 | stream.drain(..) 101 | .for_each(|mut pointer| sink.push(pointer.replace_with_default())); 102 | } 103 | 104 | #[inline(never)] 105 | fn shuffle_ring(ring: &[Mutex>>]) { 106 | fn swap_head(vec: &mut Vec<&mut Pointer>, index: usize) { 107 | debug_assert!(index > 0); 108 | 109 | let (head, tail) = vec.split_at_mut(index); 110 | let head: &mut Pointer = head[0]; 111 | let tail: &mut Pointer = tail[0]; 112 | 113 | std::mem::swap(head, tail); 114 | } 115 | 116 | let number_threads = ring.len(); 117 | assert!(number_threads >= 2, "number_threads: {} < 2", number_threads); 118 | 119 | let number_victims = ring[0].try_lock().unwrap().len(); 120 | 121 | let mut guards: Vec<_> = ring.iter().map(|mutex| mutex.try_lock().unwrap()).collect(); 122 | 123 | for i in 0..number_victims { 124 | let shift = i % number_threads; 125 | 126 | if shift == 0 { 127 | continue; 128 | } 129 | 130 | let mut layer: Vec<&mut Pointer> = guards.iter_mut().map(|guard| &mut guard[i]).collect(); 131 | 132 | for _ in 0..shift { 133 | // A single shift goes from [A, B, C, D] to [D, A, B, C]. 134 | // By iterating `shift` times, A shifts by that many places to the right. 135 | for target in 1..number_threads { 136 | swap_head(&mut layer, target); 137 | } 138 | } 139 | } 140 | 141 | } 142 | 143 | let number_iterations = number_iterations(); 144 | let number_threads = number_threads(); 145 | let number_victims = 256; 146 | 147 | let allocation = RendezVous::new("allocation", number_threads); 148 | let deallocation = RendezVous::new("deallocation", number_threads); 149 | let shuffle_start = RendezVous::new("shuffle-start", number_threads); 150 | let shuffle_end = RendezVous::new("shuffle-end", number_threads); 151 | let next = RendezVous::new("next", 0); 152 | 153 | let ring = { 154 | let mut ring = Vec::with_capacity(number_threads); 155 | ring.resize_with(number_threads, || Mutex::new(vec!())); 156 | 157 | Arc::new(ring) 158 | }; 159 | 160 | let pool = Pool::new(number_threads, |thread_index| { 161 | let allocation = allocation.clone(); 162 | let shuffle_start = shuffle_start.clone(); 163 | let shuffle_end = shuffle_end.clone(); 164 | let deallocation = deallocation.clone(); 165 | let next = next.clone(); 166 | let ring = ring.clone(); 167 | 168 | move || { 169 | LL_ALLOCATOR.warm_up().expect("Warmed up"); 170 | 171 | let custodian = thread_index == 0; 172 | 173 | for iteration in 0..number_iterations { 174 | // Prepare batch of victims 175 | let victims = create_victims(number_victims); 176 | 177 | // Move victims to the vector of Pointers, which requires allocation. 178 | { 179 | // Pre-acquire guard to avoid delaying the start for unrelated reasons. 180 | let mut sink = ring[thread_index].try_lock().unwrap(); 181 | 182 | allocation.wait_until_all_ready(); 183 | 184 | push_victims(victims, &mut *sink); 185 | } 186 | 187 | // Rearm next iteration. 188 | if custodian { 189 | next.reset(number_threads); 190 | } 191 | 192 | shuffle_start.wait_until_all_ready(); 193 | 194 | if custodian { 195 | allocation.reset(number_threads); 196 | } 197 | 198 | // Shuffle the pointers. 199 | if custodian { 200 | shuffle_ring(&*ring); 201 | } 202 | 203 | shuffle_end.wait_until_all_ready(); 204 | 205 | if custodian { 206 | shuffle_start.reset(number_threads); 207 | } 208 | 209 | // Deallocate the pointers, recover the victims. 210 | let victims = { 211 | // Pre-acquire guard to avoid delaying the start for unrelated reasons. 212 | let mut stream = ring[thread_index].try_lock().unwrap(); 213 | let mut victims = Vec::with_capacity(stream.len()); 214 | 215 | deallocation.wait_until_all_ready(); 216 | 217 | pop_victims(&mut *stream, &mut victims); 218 | 219 | victims 220 | }; 221 | 222 | for (index, victim) in victims.into_iter().enumerate() { 223 | assert_eq!(Ok(index), victim.parse(), 224 | "thread {}, iteration {}, index {}, victim {:?}", thread_index, iteration, index, victim); 225 | } 226 | 227 | if custodian { 228 | shuffle_end.reset(number_threads); 229 | } 230 | 231 | next.wait_until_all_ready(); 232 | 233 | if custodian { 234 | deallocation.reset(number_threads); 235 | } 236 | } 237 | } 238 | }); 239 | 240 | pool.join(); 241 | } 242 | 243 | // 244 | // Multi-threaded helpers 245 | // 246 | 247 | struct Pool(Vec>); 248 | 249 | impl Pool { 250 | fn new(count: usize, mut factory: F) -> Self 251 | where 252 | F: FnMut(usize) -> G, 253 | G: FnOnce() -> T + Send + 'static, 254 | T: Send + 'static 255 | { 256 | let threads : Vec<_> = (0..count) 257 | .map(|i| { 258 | thread::spawn(factory(i)) 259 | }) 260 | .collect(); 261 | 262 | Self(threads) 263 | } 264 | 265 | fn join(mut self) -> Vec { 266 | let thread_handles = std::mem::replace(&mut self.0, vec!()); 267 | Self::join_handles(thread_handles) 268 | } 269 | 270 | fn join_handles(thread_handles: Vec>) -> Vec { 271 | // First join _all_ threads. 272 | let results: Vec<_> = thread_handles.into_iter() 273 | .map(|handle| handle.join()) 274 | .collect(); 275 | // Then collect the results. 276 | results.into_iter() 277 | .map(|value| value.unwrap()) 278 | .collect() 279 | } 280 | } 281 | 282 | impl Drop for Pool { 283 | fn drop(&mut self) { 284 | let thread_handles = std::mem::replace(&mut self.0, vec!()); 285 | Self::join_handles(thread_handles); 286 | } 287 | } 288 | 289 | // # Warning 290 | // 291 | // Rearming a RendezVous is complicated: 292 | // 293 | // 1. An instance should only be rearmed by 1 thread. 294 | // 2. An instance cannot be rearmed right before a `wait_until_all_ready` on the same instance. 295 | // 3. An instance cannot be rearmed right after a `wait_until_all_ready` on the same instance. 296 | // 297 | // Rearming before is incorrect: 298 | // 299 | // ```rust 300 | // if custodian { 301 | // barrier.reset(x); 302 | // } 303 | // 304 | // barrier.wait_until_all_ready(); // Some threads may already have decremented the counter prior to the reset. 305 | // ``` 306 | // 307 | // Rearming after is incorrect: 308 | // 309 | // ```rust 310 | // barrier.wait_until_all_ready(); 311 | // 312 | // if custodian { 313 | // barrier.reset(x); // Not all threads may have exited the wait, and those who didn't now get stuck. 314 | // } 315 | // ``` 316 | #[derive(Clone, Debug)] 317 | struct RendezVous(&'static str, Arc); 318 | 319 | impl RendezVous { 320 | fn new(name: &'static str, count: usize) -> Self { 321 | Self(name, Arc::new(AtomicUsize::new(count))) 322 | } 323 | 324 | fn wait_until_all_ready(&self) { 325 | self.1.fetch_sub(1, Ordering::AcqRel); 326 | 327 | while !self.is_ready() {} 328 | } 329 | 330 | fn is_ready(&self) -> bool { self.1.load(Ordering::Acquire) == 0 } 331 | 332 | fn reset(&self, count: usize) { 333 | assert!(self.is_ready(), "{} not ready: {:?}", self.0, self.1); 334 | self.1.store(count, Ordering::Release); 335 | } 336 | } 337 | 338 | // 339 | // Implementation Details 340 | // 341 | 342 | fn number_iterations() -> usize { read_number_from_environment("LLMALLOC_MULTI_NUMBER_ITERATIONS", 10) } 343 | 344 | fn number_threads() -> usize { read_number_from_environment("LLMALLOC_MULTI_NUMBER_THREADS", 4) } 345 | 346 | fn read_number_from_environment(name: &str, default: usize) -> usize { 347 | for (n, value) in env::vars() { 348 | if n == name { 349 | if let Ok(result) = value.parse() { 350 | println!("read_number_from_environment - {}: {}", name, result); 351 | return result; 352 | } 353 | } 354 | } 355 | 356 | println!("read_number_from_environment - {}: {} (default)", name, default); 357 | default 358 | } 359 | 360 | struct Pointer { 361 | pointer: NonNull, 362 | } 363 | 364 | impl Pointer { 365 | fn new(value: T) -> Self { 366 | let size = mem::size_of::(); 367 | let align = mem::align_of::(); 368 | let pointer = LL_ALLOCATOR.allocate(layout(size, align)).unwrap().cast(); 369 | 370 | unsafe { ptr::write(pointer.as_ptr(), value) } 371 | 372 | Pointer { pointer } 373 | } 374 | 375 | fn replace_with_default(&mut self) -> T 376 | where 377 | T: Default 378 | { 379 | std::mem::replace(&mut *self, T::default()) 380 | } 381 | } 382 | 383 | impl Default for Pointer 384 | where 385 | T: Default 386 | { 387 | fn default() -> Self { Self::new(T::default()) } 388 | } 389 | 390 | impl Drop for Pointer { 391 | fn drop(&mut self) { 392 | unsafe { 393 | ptr::drop_in_place(self.pointer.as_ptr()); 394 | LL_ALLOCATOR.deallocate(self.pointer.cast()); 395 | } 396 | } 397 | } 398 | 399 | impl Deref for Pointer { 400 | type Target = T; 401 | 402 | fn deref(&self) -> &T { unsafe { &*self.pointer.as_ptr() } } 403 | } 404 | 405 | impl DerefMut for Pointer { 406 | fn deref_mut(&mut self) -> &mut T { unsafe { &mut *self.pointer.as_ptr() } } 407 | } 408 | 409 | unsafe impl Send for Pointer 410 | where 411 | T: Send 412 | {} 413 | 414 | fn layout(size: usize, alignment: usize) -> Layout { 415 | Layout::from_size_align(size, alignment).expect("Valid Layout") 416 | } 417 | 418 | // FIXME: use sys crates... properly configured for system libraries. 419 | #[link(name = "numa")] 420 | extern "C" {} 421 | --------------------------------------------------------------------------------