├── .gitignore ├── Cargo.toml ├── LICENSE-MIT ├── README.md └── src ├── ext.rs ├── lib.rs ├── small_float.rs └── tests.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "offset-allocator" 3 | version = "0.2.0" 4 | edition = "2021" 5 | authors = ["Patrick Walton "] 6 | description = "A port of Sebastian Aaltonen's `OffsetAllocator` to Rust" 7 | readme = "README.md" 8 | license = "MIT" 9 | repository = "https://github.com/pcwalton/offset-allocator/" 10 | keywords = ["memory-management"] 11 | 12 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 13 | 14 | [dependencies] 15 | log = "0.4" 16 | nonmax = "0.5" 17 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Sebastian Aaltonen, Patrick Walton 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `offset-allocator` 2 | 3 | ## Overview 4 | 5 | This is a port of [Sebastian Aaltonen's `OffsetAllocator`] package for C++ to 100% safe Rust. It's a fast, simple, hard real time allocator. This is especially useful for managing GPU resources, and the goal is to use it in [Bevy]. 6 | 7 | The port has been made into more or less idiomatic Rust but is otherwise mostly line-for-line, preserving comments. That way, patches for the original `OffsetAllocator` should be readily transferable to this Rust port. 8 | 9 | Please note that `offset-allocator` isn't a Rust allocator conforming to the `GlobalAlloc` trait. You can't use this crate as a drop-in replacement for the system allocator, `jemalloc`, `wee_alloc`, etc. The general algorithm that this crate uses could be adapted to construct a Rust allocator, but that's beyond the scope of this particular implementation. This is by design, so that this allocator can be used to manage resources that aren't just CPU memory: in particular, you can manage allocations inside GPU buffers with it. By contrast, Rust allocators are hard-wired to the CPU and can't be used to manage GPU resources. 10 | 11 | ## Description 12 | 13 | This allocator is completely agnostic to what it's allocating: it only knows 14 | about a contiguous block of memory of a specific size. That size need not be in 15 | bytes: this is especially useful when allocating inside a buffer of fixed-size 16 | structures. For example, if using this allocator to divide up a GPU index 17 | buffer object, one might want to treat the units of allocation as 32-bit 18 | floats. 19 | 20 | From [the original README]: 21 | 22 | > Fast hard realtime O(1) offset allocator with minimal fragmentation. 23 | 24 | > Uses 256 bins with 8 bit floating point distribution (3 bit mantissa + 5 bit exponent) and a two level bitfield to find the next available bin using 2x LZCNT instructions to make all operations O(1). Bin sizes following the floating point distribution ensures hard bounds for memory overhead percentage regarless of size class. Pow2 bins would waste up to +100% memory (+50% on average). Our float bins waste up to +12.5% (+6.25% on average). 25 | 26 | > The allocation metadata is stored in a separate data structure, making this allocator suitable for sub-allocating any resources, such as GPU heaps, buffers and arrays. Returns an offset to the first element of the allocated contiguous range. 27 | 28 | ## References 29 | 30 | Again per [the original README]: 31 | 32 | > This allocator is similar to the two-level segregated fit (TLSF) algorithm. 33 | 34 | > Comparison paper shows that TLSF algorithm provides best in class performance and fragmentation: 35 | 36 | ## Author 37 | 38 | C++ version: Sebastian Aaltonen 39 | 40 | Rust port: Patrick Walton, @pcwalton 41 | 42 | ## License 43 | 44 | Licensed under the MIT license. See `LICENSE-MIT` for details. 45 | 46 | ## Code of conduct 47 | 48 | `offset-allocator` follows the same code of conduct as Rust itself. Reports can be made to the project authors. 49 | 50 | [Sebastian Aaltonen's `OffsetAllocator`]: https://github.com/sebbbi/OffsetAllocator 51 | [the original README]: https://github.com/sebbbi/OffsetAllocator/blob/main/README.md 52 | [Bevy]: https://github.com/bevyengine/bevy/ 53 | -------------------------------------------------------------------------------- /src/ext.rs: -------------------------------------------------------------------------------- 1 | //! Extension functions not present in the original C++ `OffsetAllocator`. 2 | 3 | use crate::small_float; 4 | 5 | /// Returns the minimum allocator size needed to hold an object of the given 6 | /// size. 7 | pub fn min_allocator_size(needed_object_size: u32) -> u32 { 8 | small_float::float_to_uint(small_float::uint_to_float_round_up(needed_object_size)) 9 | } 10 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // offset-allocator/src/lib.rs 2 | 3 | #![doc = include_str!("../README.md")] 4 | #![deny(unsafe_code)] 5 | #![warn(missing_docs)] 6 | 7 | use std::fmt::{Debug, Display, Formatter, Result as FmtResult}; 8 | 9 | use log::debug; 10 | use nonmax::{NonMaxU16, NonMaxU32}; 11 | 12 | pub mod ext; 13 | 14 | mod small_float; 15 | 16 | #[cfg(test)] 17 | mod tests; 18 | 19 | const NUM_TOP_BINS: usize = 32; 20 | const BINS_PER_LEAF: usize = 8; 21 | const TOP_BINS_INDEX_SHIFT: u32 = 3; 22 | const LEAF_BINS_INDEX_MASK: u32 = 7; 23 | const NUM_LEAF_BINS: usize = NUM_TOP_BINS * BINS_PER_LEAF; 24 | 25 | /// Determines the number of allocations that the allocator supports. 26 | /// 27 | /// By default, [`Allocator`] and related functions use `u32`, which allows for 28 | /// `u32::MAX - 1` allocations. You can, however, use `u16` instead, which 29 | /// causes the allocator to use less memory but limits the number of allocations 30 | /// within a single allocator to at most 65,534. 31 | pub trait NodeIndex: Clone + Copy + Default { 32 | /// The `NonMax` version of this type. 33 | /// 34 | /// This is used extensively to optimize `enum` representations. 35 | type NonMax: NodeIndexNonMax + TryFrom + Into; 36 | 37 | /// The maximum value representable in this type. 38 | const MAX: u32; 39 | 40 | /// Converts from a unsigned 32-bit integer to an instance of this type. 41 | fn from_u32(val: u32) -> Self; 42 | 43 | /// Converts this type to an unsigned machine word. 44 | fn to_usize(self) -> usize; 45 | } 46 | 47 | /// The `NonMax` version of the [`NodeIndex`]. 48 | /// 49 | /// For example, for `u32`, the `NonMax` version is [`NonMaxU32`]. 50 | pub trait NodeIndexNonMax: Clone + Copy + PartialEq + Default + Debug + Display { 51 | /// Converts this type to an unsigned machine word. 52 | fn to_usize(self) -> usize; 53 | } 54 | 55 | /// An allocator that manages a single contiguous chunk of space and hands out 56 | /// portions of it as requested. 57 | pub struct Allocator 58 | where 59 | NI: NodeIndex, 60 | { 61 | size: u32, 62 | max_allocs: u32, 63 | free_storage: u32, 64 | 65 | used_bins_top: u32, 66 | used_bins: [u8; NUM_TOP_BINS], 67 | bin_indices: [Option; NUM_LEAF_BINS], 68 | 69 | nodes: Vec>, 70 | free_nodes: Vec, 71 | free_offset: u32, 72 | } 73 | 74 | /// A single allocation. 75 | #[derive(Clone, Copy)] 76 | pub struct Allocation 77 | where 78 | NI: NodeIndex, 79 | { 80 | /// The location of this allocation within the buffer. 81 | pub offset: NI, 82 | /// The node index associated with this allocation. 83 | metadata: NI::NonMax, 84 | } 85 | 86 | /// Provides a summary of the state of the allocator, including space remaining. 87 | #[derive(Debug)] 88 | pub struct StorageReport { 89 | /// The amount of free space left. 90 | pub total_free_space: u32, 91 | /// The maximum potential size of a single contiguous allocation. 92 | pub largest_free_region: u32, 93 | } 94 | 95 | /// Provides a detailed accounting of each bin within the allocator. 96 | #[derive(Debug)] 97 | pub struct StorageReportFull { 98 | /// Each bin within the allocator. 99 | pub free_regions: [StorageReportFullRegion; NUM_LEAF_BINS], 100 | } 101 | 102 | /// A detailed accounting of each allocator bin. 103 | #[derive(Clone, Copy, Debug, Default)] 104 | pub struct StorageReportFullRegion { 105 | /// The size of the bin, in units. 106 | pub size: u32, 107 | /// The number of allocations in the bin. 108 | pub count: u32, 109 | } 110 | 111 | #[derive(Clone, Copy, Default)] 112 | struct Node 113 | where 114 | NI: NodeIndex, 115 | { 116 | data_offset: u32, 117 | data_size: u32, 118 | bin_list_prev: Option, 119 | bin_list_next: Option, 120 | neighbor_prev: Option, 121 | neighbor_next: Option, 122 | used: bool, // TODO: Merge as bit flag 123 | } 124 | 125 | // Utility functions 126 | fn find_lowest_bit_set_after(bit_mask: u32, start_bit_index: u32) -> Option { 127 | let mask_before_start_index = (1 << start_bit_index) - 1; 128 | let mask_after_start_index = !mask_before_start_index; 129 | let bits_after = bit_mask & mask_after_start_index; 130 | if bits_after == 0 { 131 | None 132 | } else { 133 | NonMaxU32::try_from(bits_after.trailing_zeros()).ok() 134 | } 135 | } 136 | 137 | impl Allocator 138 | where 139 | NI: NodeIndex, 140 | { 141 | /// Creates a new allocator, managing a contiguous block of memory of `size` 142 | /// units, with a default reasonable number of maximum allocations. 143 | pub fn new(size: u32) -> Self { 144 | Allocator::with_max_allocs(size, u32::min(128 * 1024, NI::MAX - 1)) 145 | } 146 | 147 | /// Creates a new allocator, managing a contiguous block of memory of `size` 148 | /// units, with the given number of maximum allocations. 149 | /// 150 | /// Note that the maximum number of allocations must be less than 151 | /// [`NodeIndex::MAX`] minus one. If this restriction is violated, this 152 | /// constructor will panic. 153 | pub fn with_max_allocs(size: u32, max_allocs: u32) -> Self { 154 | assert!(max_allocs < NI::MAX - 1); 155 | 156 | let mut this = Self { 157 | size, 158 | max_allocs, 159 | free_storage: 0, 160 | used_bins_top: 0, 161 | free_offset: 0, 162 | used_bins: [0; NUM_TOP_BINS], 163 | bin_indices: [None; NUM_LEAF_BINS], 164 | nodes: vec![], 165 | free_nodes: vec![], 166 | }; 167 | this.reset(); 168 | this 169 | } 170 | 171 | /// Clears out all allocations. 172 | pub fn reset(&mut self) { 173 | self.free_storage = 0; 174 | self.used_bins_top = 0; 175 | self.free_offset = self.max_allocs - 1; 176 | 177 | self.used_bins.iter_mut().for_each(|bin| *bin = 0); 178 | 179 | self.bin_indices.iter_mut().for_each(|index| *index = None); 180 | 181 | self.nodes = vec![Node::default(); self.max_allocs as usize]; 182 | 183 | // Freelist is a stack. Nodes in inverse order so that [0] pops first. 184 | self.free_nodes = (0..self.max_allocs) 185 | .map(|i| { 186 | NI::NonMax::try_from(NI::from_u32(self.max_allocs - i - 1)).unwrap_or_default() 187 | }) 188 | .collect(); 189 | 190 | // Start state: Whole storage as one big node 191 | // Algorithm will split remainders and push them back as smaller nodes 192 | self.insert_node_into_bin(self.size, 0); 193 | } 194 | 195 | /// Allocates a block of `size` elements and returns its allocation. 196 | /// 197 | /// If there's not enough contiguous space for this allocation, returns 198 | /// None. 199 | pub fn allocate(&mut self, size: u32) -> Option> { 200 | // Out of allocations? 201 | if self.free_offset == 0 { 202 | return None; 203 | } 204 | 205 | // Round up to bin index to ensure that alloc >= bin 206 | // Gives us min bin index that fits the size 207 | let min_bin_index = small_float::uint_to_float_round_up(size); 208 | 209 | let min_top_bin_index = min_bin_index >> TOP_BINS_INDEX_SHIFT; 210 | let min_leaf_bin_index = min_bin_index & LEAF_BINS_INDEX_MASK; 211 | 212 | let mut top_bin_index = min_top_bin_index; 213 | let mut leaf_bin_index = None; 214 | 215 | // If top bin exists, scan its leaf bin. This can fail (NO_SPACE). 216 | if (self.used_bins_top & (1 << top_bin_index)) != 0 { 217 | leaf_bin_index = find_lowest_bit_set_after( 218 | self.used_bins[top_bin_index as usize] as _, 219 | min_leaf_bin_index, 220 | ); 221 | } 222 | 223 | // If we didn't find space in top bin, we search top bin from +1 224 | let leaf_bin_index = match leaf_bin_index { 225 | Some(leaf_bin_index) => leaf_bin_index, 226 | None => { 227 | top_bin_index = 228 | find_lowest_bit_set_after(self.used_bins_top, min_top_bin_index + 1)?.into(); 229 | 230 | // All leaf bins here fit the alloc, since the top bin was 231 | // rounded up. Start leaf search from bit 0. 232 | // 233 | // NOTE: This search can't fail since at least one leaf bit was 234 | // set because the top bit was set. 235 | NonMaxU32::try_from(self.used_bins[top_bin_index as usize].trailing_zeros()) 236 | .unwrap() 237 | } 238 | }; 239 | 240 | let bin_index = (top_bin_index << TOP_BINS_INDEX_SHIFT) | u32::from(leaf_bin_index); 241 | 242 | // Pop the top node of the bin. Bin top = node.next. 243 | let node_index = self.bin_indices[bin_index as usize].unwrap(); 244 | let node = &mut self.nodes[node_index.to_usize()]; 245 | let node_total_size = node.data_size; 246 | node.data_size = size; 247 | node.used = true; 248 | self.bin_indices[bin_index as usize] = node.bin_list_next; 249 | if let Some(bin_list_next) = node.bin_list_next { 250 | self.nodes[bin_list_next.to_usize()].bin_list_prev = None; 251 | } 252 | self.free_storage -= node_total_size; 253 | debug!( 254 | "Free storage: {} (-{}) (allocate)", 255 | self.free_storage, node_total_size 256 | ); 257 | 258 | // Bin empty? 259 | if self.bin_indices[bin_index as usize].is_none() { 260 | // Remove a leaf bin mask bit 261 | self.used_bins[top_bin_index as usize] &= !(1 << u32::from(leaf_bin_index)); 262 | 263 | // All leaf bins empty? 264 | if self.used_bins[top_bin_index as usize] == 0 { 265 | // Remove a top bin mask bit 266 | self.used_bins_top &= !(1 << top_bin_index); 267 | } 268 | } 269 | 270 | // Push back remainder N elements to a lower bin 271 | let remainder_size = node_total_size - size; 272 | if remainder_size > 0 { 273 | let Node { 274 | data_offset, 275 | neighbor_next, 276 | .. 277 | } = self.nodes[node_index.to_usize()]; 278 | 279 | let new_node_index = self.insert_node_into_bin(remainder_size, data_offset + size); 280 | 281 | // Link nodes next to each other so that we can merge them later if both are free 282 | // And update the old next neighbor to point to the new node (in middle) 283 | let node = &mut self.nodes[node_index.to_usize()]; 284 | if let Some(neighbor_next) = node.neighbor_next { 285 | self.nodes[neighbor_next.to_usize()].neighbor_prev = Some(new_node_index); 286 | } 287 | self.nodes[new_node_index.to_usize()].neighbor_prev = Some(node_index); 288 | self.nodes[new_node_index.to_usize()].neighbor_next = neighbor_next; 289 | self.nodes[node_index.to_usize()].neighbor_next = Some(new_node_index); 290 | } 291 | 292 | let node = &mut self.nodes[node_index.to_usize()]; 293 | Some(Allocation { 294 | offset: NI::from_u32(node.data_offset), 295 | metadata: node_index, 296 | }) 297 | } 298 | 299 | /// Frees an allocation, returning the data to the heap. 300 | /// 301 | /// If the allocation has already been freed, the behavior is unspecified. 302 | /// It may or may not panic. Note that, because this crate contains no 303 | /// unsafe code, the memory safe of the allocator *itself* will be 304 | /// uncompromised, even on double free. 305 | pub fn free(&mut self, allocation: Allocation) { 306 | let node_index = allocation.metadata; 307 | 308 | // Merge with neighbors… 309 | let Node { 310 | data_offset: mut offset, 311 | data_size: mut size, 312 | used, 313 | .. 314 | } = self.nodes[node_index.to_usize()]; 315 | 316 | // Double delete check 317 | assert!(used); 318 | 319 | if let Some(neighbor_prev) = self.nodes[node_index.to_usize()].neighbor_prev { 320 | if !self.nodes[neighbor_prev.to_usize()].used { 321 | // Previous (contiguous) free node: Change offset to previous 322 | // node offset. Sum sizes 323 | let prev_node = &self.nodes[neighbor_prev.to_usize()]; 324 | offset = prev_node.data_offset; 325 | size += prev_node.data_size; 326 | 327 | // Remove node from the bin linked list and put it in the 328 | // freelist 329 | self.remove_node_from_bin(neighbor_prev); 330 | 331 | let prev_node = &self.nodes[neighbor_prev.to_usize()]; 332 | debug_assert_eq!(prev_node.neighbor_next, Some(node_index)); 333 | self.nodes[node_index.to_usize()].neighbor_prev = prev_node.neighbor_prev; 334 | } 335 | } 336 | 337 | if let Some(neighbor_next) = self.nodes[node_index.to_usize()].neighbor_next { 338 | if !self.nodes[neighbor_next.to_usize()].used { 339 | // Next (contiguous) free node: Offset remains the same. Sum 340 | // sizes. 341 | let next_node = &self.nodes[neighbor_next.to_usize()]; 342 | size += next_node.data_size; 343 | 344 | // Remove node from the bin linked list and put it in the 345 | // freelist 346 | self.remove_node_from_bin(neighbor_next); 347 | 348 | let next_node = &self.nodes[neighbor_next.to_usize()]; 349 | debug_assert_eq!(next_node.neighbor_prev, Some(node_index)); 350 | self.nodes[node_index.to_usize()].neighbor_next = next_node.neighbor_next; 351 | } 352 | } 353 | 354 | let Node { 355 | neighbor_next, 356 | neighbor_prev, 357 | .. 358 | } = self.nodes[node_index.to_usize()]; 359 | 360 | // Insert the removed node to freelist 361 | debug!( 362 | "Putting node {} into freelist[{}] (free)", 363 | node_index, 364 | self.free_offset + 1 365 | ); 366 | self.free_offset += 1; 367 | self.free_nodes[self.free_offset as usize] = node_index; 368 | 369 | // Insert the (combined) free node to bin 370 | let combined_node_index = self.insert_node_into_bin(size, offset); 371 | 372 | // Connect neighbors with the new combined node 373 | if let Some(neighbor_next) = neighbor_next { 374 | self.nodes[combined_node_index.to_usize()].neighbor_next = Some(neighbor_next); 375 | self.nodes[neighbor_next.to_usize()].neighbor_prev = Some(combined_node_index); 376 | } 377 | if let Some(neighbor_prev) = neighbor_prev { 378 | self.nodes[combined_node_index.to_usize()].neighbor_prev = Some(neighbor_prev); 379 | self.nodes[neighbor_prev.to_usize()].neighbor_next = Some(combined_node_index); 380 | } 381 | } 382 | 383 | fn insert_node_into_bin(&mut self, size: u32, data_offset: u32) -> NI::NonMax { 384 | // Round down to bin index to ensure that bin >= alloc 385 | let bin_index = small_float::uint_to_float_round_down(size); 386 | 387 | let top_bin_index = bin_index >> TOP_BINS_INDEX_SHIFT; 388 | let leaf_bin_index = bin_index & LEAF_BINS_INDEX_MASK; 389 | 390 | // Bin was empty before? 391 | if self.bin_indices[bin_index as usize].is_none() { 392 | // Set bin mask bits 393 | self.used_bins[top_bin_index as usize] |= 1 << leaf_bin_index; 394 | self.used_bins_top |= 1 << top_bin_index; 395 | } 396 | 397 | // Take a freelist node and insert on top of the bin linked list (next = old top) 398 | let top_node_index = self.bin_indices[bin_index as usize]; 399 | let free_offset = self.free_offset; 400 | let node_index = self.free_nodes[free_offset as usize]; 401 | self.free_offset -= 1; 402 | debug!( 403 | "Getting node {} from freelist[{}]", 404 | node_index, 405 | self.free_offset + 1 406 | ); 407 | self.nodes[node_index.to_usize()] = Node { 408 | data_offset, 409 | data_size: size, 410 | bin_list_next: top_node_index, 411 | ..Node::default() 412 | }; 413 | if let Some(top_node_index) = top_node_index { 414 | self.nodes[top_node_index.to_usize()].bin_list_prev = Some(node_index); 415 | } 416 | self.bin_indices[bin_index as usize] = Some(node_index); 417 | 418 | self.free_storage += size; 419 | debug!( 420 | "Free storage: {} (+{}) (insert_node_into_bin)", 421 | self.free_storage, size 422 | ); 423 | node_index 424 | } 425 | 426 | fn remove_node_from_bin(&mut self, node_index: NI::NonMax) { 427 | // Copy the node to work around borrow check. 428 | let node = self.nodes[node_index.to_usize()]; 429 | 430 | match node.bin_list_prev { 431 | Some(bin_list_prev) => { 432 | // Easy case: We have previous node. Just remove this node from the middle of the list. 433 | self.nodes[bin_list_prev.to_usize()].bin_list_next = node.bin_list_next; 434 | if let Some(bin_list_next) = node.bin_list_next { 435 | self.nodes[bin_list_next.to_usize()].bin_list_prev = node.bin_list_prev; 436 | } 437 | } 438 | None => { 439 | // Hard case: We are the first node in a bin. Find the bin. 440 | 441 | // Round down to bin index to ensure that bin >= alloc 442 | let bin_index = small_float::uint_to_float_round_down(node.data_size); 443 | 444 | let top_bin_index = (bin_index >> TOP_BINS_INDEX_SHIFT) as usize; 445 | let leaf_bin_index = (bin_index & LEAF_BINS_INDEX_MASK) as usize; 446 | 447 | self.bin_indices[bin_index as usize] = node.bin_list_next; 448 | if let Some(bin_list_next) = node.bin_list_next { 449 | self.nodes[bin_list_next.to_usize()].bin_list_prev = None; 450 | } 451 | 452 | // Bin empty? 453 | if self.bin_indices[bin_index as usize].is_none() { 454 | // Remove a leaf bin mask bit 455 | self.used_bins[top_bin_index as usize] &= !(1 << leaf_bin_index); 456 | 457 | // All leaf bins empty? 458 | if self.used_bins[top_bin_index as usize] == 0 { 459 | // Remove a top bin mask bit 460 | self.used_bins_top &= !(1 << top_bin_index); 461 | } 462 | } 463 | } 464 | } 465 | 466 | // Insert the node to freelist 467 | debug!( 468 | "Putting node {} into freelist[{}] (remove_node_from_bin)", 469 | node_index, 470 | self.free_offset + 1 471 | ); 472 | self.free_offset += 1; 473 | self.free_nodes[self.free_offset as usize] = node_index; 474 | 475 | self.free_storage -= node.data_size; 476 | debug!( 477 | "Free storage: {} (-{}) (remove_node_from_bin)", 478 | self.free_storage, node.data_size 479 | ); 480 | } 481 | 482 | /// Returns the *used* size of an allocation. 483 | /// 484 | /// Note that this may be larger than the size requested at allocation time, 485 | /// due to rounding. 486 | pub fn allocation_size(&self, allocation: Allocation) -> u32 { 487 | self.nodes 488 | .get(allocation.metadata.to_usize()) 489 | .map(|node| node.data_size) 490 | .unwrap_or_default() 491 | } 492 | 493 | /// Returns a structure containing the amount of free space remaining, as 494 | /// well as the largest amount that can be allocated at once. 495 | pub fn storage_report(&self) -> StorageReport { 496 | let mut largest_free_region = 0; 497 | let mut free_storage = 0; 498 | 499 | // Out of allocations? -> Zero free space 500 | if self.free_offset > 0 { 501 | free_storage = self.free_storage; 502 | if self.used_bins_top > 0 { 503 | let top_bin_index = 31 - self.used_bins_top.leading_zeros(); 504 | let leaf_bin_index = 505 | 31 - (self.used_bins[top_bin_index as usize] as u32).leading_zeros(); 506 | largest_free_region = small_float::float_to_uint( 507 | (top_bin_index << TOP_BINS_INDEX_SHIFT) | leaf_bin_index, 508 | ); 509 | debug_assert!(free_storage >= largest_free_region); 510 | } 511 | } 512 | 513 | StorageReport { 514 | total_free_space: free_storage, 515 | largest_free_region, 516 | } 517 | } 518 | 519 | /// Returns detailed information about the number of allocations in each 520 | /// bin. 521 | pub fn storage_report_full(&self) -> StorageReportFull { 522 | let mut report = StorageReportFull::default(); 523 | for i in 0..NUM_LEAF_BINS { 524 | let mut count = 0; 525 | let mut maybe_node_index = self.bin_indices[i]; 526 | while let Some(node_index) = maybe_node_index { 527 | maybe_node_index = self.nodes[node_index.to_usize()].bin_list_next; 528 | count += 1; 529 | } 530 | report.free_regions[i] = StorageReportFullRegion { 531 | size: small_float::float_to_uint(i as u32), 532 | count, 533 | } 534 | } 535 | report 536 | } 537 | } 538 | 539 | impl Default for StorageReportFull { 540 | fn default() -> Self { 541 | Self { 542 | free_regions: [Default::default(); NUM_LEAF_BINS], 543 | } 544 | } 545 | } 546 | 547 | impl Debug for Allocator 548 | where 549 | NI: NodeIndex, 550 | { 551 | fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { 552 | self.storage_report().fmt(f) 553 | } 554 | } 555 | 556 | impl NodeIndex for u32 { 557 | type NonMax = NonMaxU32; 558 | const MAX: u32 = u32::MAX; 559 | 560 | fn from_u32(val: u32) -> Self { 561 | val 562 | } 563 | 564 | fn to_usize(self) -> usize { 565 | self as usize 566 | } 567 | } 568 | 569 | impl NodeIndex for u16 { 570 | type NonMax = NonMaxU16; 571 | const MAX: u32 = u16::MAX as u32; 572 | 573 | fn from_u32(val: u32) -> Self { 574 | val as u16 575 | } 576 | 577 | fn to_usize(self) -> usize { 578 | self as usize 579 | } 580 | } 581 | 582 | impl NodeIndexNonMax for NonMaxU32 { 583 | fn to_usize(self) -> usize { 584 | u32::from(self) as usize 585 | } 586 | } 587 | 588 | impl NodeIndexNonMax for NonMaxU16 { 589 | fn to_usize(self) -> usize { 590 | u16::from(self) as usize 591 | } 592 | } 593 | -------------------------------------------------------------------------------- /src/small_float.rs: -------------------------------------------------------------------------------- 1 | // offset-allocator/src/small_float.rs 2 | 3 | pub const MANTISSA_BITS: u32 = 3; 4 | pub const MANTISSA_VALUE: u32 = 1 << MANTISSA_BITS; 5 | pub const MANTISSA_MASK: u32 = MANTISSA_VALUE - 1; 6 | 7 | // Bin sizes follow floating point (exponent + mantissa) distribution (piecewise linear log approx) 8 | // This ensures that for each size class, the average overhead percentage stays the same 9 | pub fn uint_to_float_round_up(size: u32) -> u32 { 10 | let mut exp = 0; 11 | let mut mantissa; 12 | 13 | if size < MANTISSA_VALUE { 14 | // Denorm: 0..(MANTISSA_VALUE-1) 15 | mantissa = size 16 | } else { 17 | // Normalized: Hidden high bit always 1. Not stored. Just like float. 18 | let leading_zeros = size.leading_zeros(); 19 | let highest_set_bit = 31 - leading_zeros; 20 | 21 | let mantissa_start_bit = highest_set_bit - MANTISSA_BITS; 22 | exp = mantissa_start_bit + 1; 23 | mantissa = (size >> mantissa_start_bit) & MANTISSA_MASK; 24 | 25 | let low_bits_mask = (1 << mantissa_start_bit) - 1; 26 | 27 | // Round up! 28 | if (size & low_bits_mask) != 0 { 29 | mantissa += 1; 30 | } 31 | } 32 | 33 | // + allows mantissa->exp overflow for round up 34 | (exp << MANTISSA_BITS) + mantissa 35 | } 36 | 37 | pub fn uint_to_float_round_down(size: u32) -> u32 { 38 | let mut exp = 0; 39 | let mantissa; 40 | 41 | if size < MANTISSA_VALUE { 42 | // Denorm: 0..(MANTISSA_VALUE-1) 43 | mantissa = size 44 | } else { 45 | // Normalized: Hidden high bit always 1. Not stored. Just like float. 46 | let leading_zeros = size.leading_zeros(); 47 | let highest_set_bit = 31 - leading_zeros; 48 | 49 | let mantissa_start_bit = highest_set_bit - MANTISSA_BITS; 50 | exp = mantissa_start_bit + 1; 51 | mantissa = (size >> mantissa_start_bit) & MANTISSA_MASK; 52 | } 53 | 54 | (exp << MANTISSA_BITS) | mantissa 55 | } 56 | 57 | pub fn float_to_uint(float_value: u32) -> u32 { 58 | let exponent = float_value >> MANTISSA_BITS; 59 | let mantissa = float_value & MANTISSA_MASK; 60 | if exponent == 0 { 61 | mantissa 62 | } else { 63 | (mantissa | MANTISSA_VALUE) << (exponent - 1) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/tests.rs: -------------------------------------------------------------------------------- 1 | // offset-allocator/src/tests.rs 2 | 3 | use std::array; 4 | 5 | use crate::{ext, small_float, Allocator}; 6 | 7 | #[test] 8 | fn small_float_uint_to_float() { 9 | // Denorms, exp=1 and exp=2 + mantissa = 0 are all precise. 10 | // NOTE: Assuming 8 value (3 bit) mantissa. 11 | // If this test fails, please change this assumption! 12 | let precise_number_count = 17; 13 | for i in 0..precise_number_count { 14 | let round_up = small_float::uint_to_float_round_up(i); 15 | let round_down = small_float::uint_to_float_round_down(i); 16 | assert_eq!(i, round_up); 17 | assert_eq!(i, round_down); 18 | } 19 | 20 | // Test some random picked numbers 21 | struct NumberFloatUpDown { 22 | number: u32, 23 | up: u32, 24 | down: u32, 25 | } 26 | 27 | let test_data = [ 28 | NumberFloatUpDown { 29 | number: 17, 30 | up: 17, 31 | down: 16, 32 | }, 33 | NumberFloatUpDown { 34 | number: 118, 35 | up: 39, 36 | down: 38, 37 | }, 38 | NumberFloatUpDown { 39 | number: 1024, 40 | up: 64, 41 | down: 64, 42 | }, 43 | NumberFloatUpDown { 44 | number: 65536, 45 | up: 112, 46 | down: 112, 47 | }, 48 | NumberFloatUpDown { 49 | number: 529445, 50 | up: 137, 51 | down: 136, 52 | }, 53 | NumberFloatUpDown { 54 | number: 1048575, 55 | up: 144, 56 | down: 143, 57 | }, 58 | ]; 59 | 60 | for v in test_data { 61 | let round_up = small_float::uint_to_float_round_up(v.number); 62 | let round_down = small_float::uint_to_float_round_down(v.number); 63 | assert_eq!(round_up, v.up); 64 | assert_eq!(round_down, v.down); 65 | } 66 | } 67 | 68 | #[test] 69 | fn small_float_float_to_uint() { 70 | // Denorms, exp=1 and exp=2 + mantissa = 0 are all precise. 71 | // NOTE: Assuming 8 value (3 bit) mantissa. 72 | // If this test fails, please change this assumption! 73 | let precise_number_count = 17; 74 | for i in 0..precise_number_count { 75 | let v = small_float::float_to_uint(i); 76 | assert_eq!(i, v); 77 | } 78 | 79 | // Test that float->uint->float conversion is precise for all numbers 80 | // NOTE: Test values < 240. 240->4G = overflows 32 bit integer 81 | for i in 0..240 { 82 | let v = small_float::float_to_uint(i); 83 | let round_up = small_float::uint_to_float_round_up(v); 84 | let round_down = small_float::uint_to_float_round_down(v); 85 | assert_eq!(i, round_up); 86 | assert_eq!(i, round_down); 87 | } 88 | } 89 | 90 | #[test] 91 | fn basic_offset_allocator() { 92 | let mut allocator = Allocator::new(1024 * 1024 * 256); 93 | let a = allocator.allocate(1337).unwrap(); 94 | let offset: u32 = a.offset; 95 | assert_eq!(offset, 0); 96 | allocator.free(a); 97 | } 98 | 99 | #[test] 100 | fn allocate_offset_allocator_simple() { 101 | let mut allocator: Allocator = Allocator::new(1024 * 1024 * 256); 102 | 103 | // Free merges neighbor empty nodes. Next allocation should also have offset = 0 104 | let a = allocator.allocate(0).unwrap(); 105 | assert_eq!(a.offset, 0); 106 | 107 | let b = allocator.allocate(1).unwrap(); 108 | assert_eq!(b.offset, 0); 109 | 110 | let c = allocator.allocate(123).unwrap(); 111 | assert_eq!(c.offset, 1); 112 | 113 | let d = allocator.allocate(1234).unwrap(); 114 | assert_eq!(d.offset, 124); 115 | 116 | allocator.free(a); 117 | allocator.free(b); 118 | allocator.free(c); 119 | allocator.free(d); 120 | 121 | // End: Validate that allocator has no fragmentation left. Should be 100% clean. 122 | let validate_all = allocator.allocate(1024 * 1024 * 256).unwrap(); 123 | assert_eq!(validate_all.offset, 0); 124 | allocator.free(validate_all); 125 | } 126 | 127 | #[test] 128 | fn allocate_offset_allocator_merge_trivial() { 129 | let mut allocator: Allocator = Allocator::new(1024 * 1024 * 256); 130 | 131 | // Free merges neighbor empty nodes. Next allocation should also have offset = 0 132 | let a = allocator.allocate(1337).unwrap(); 133 | assert_eq!(a.offset, 0); 134 | allocator.free(a); 135 | 136 | let b = allocator.allocate(1337).unwrap(); 137 | assert_eq!(b.offset, 0); 138 | allocator.free(b); 139 | 140 | // End: Validate that allocator has no fragmentation left. Should be 100% clean. 141 | let validate_all = allocator.allocate(1024 * 1024 * 256).unwrap(); 142 | assert_eq!(validate_all.offset, 0); 143 | allocator.free(validate_all); 144 | } 145 | 146 | #[test] 147 | fn allocate_offset_allocator_reuse_trivial() { 148 | let mut allocator: Allocator = Allocator::new(1024 * 1024 * 256); 149 | 150 | // Allocator should reuse node freed by A since the allocation C fits in the same bin (using pow2 size to be sure) 151 | let a = allocator.allocate(1024).unwrap(); 152 | assert_eq!(a.offset, 0); 153 | 154 | let b = allocator.allocate(3456).unwrap(); 155 | assert_eq!(b.offset, 1024); 156 | 157 | allocator.free(a); 158 | 159 | let c = allocator.allocate(1024).unwrap(); 160 | assert_eq!(c.offset, 0); 161 | 162 | allocator.free(c); 163 | allocator.free(b); 164 | 165 | // End: Validate that allocator has no fragmentation left. Should be 100% clean. 166 | let validate_all = allocator.allocate(1024 * 1024 * 256).unwrap(); 167 | assert_eq!(validate_all.offset, 0); 168 | allocator.free(validate_all); 169 | } 170 | 171 | #[test] 172 | fn allocate_offset_allocator_reuse_complex() { 173 | let mut allocator: Allocator = Allocator::new(1024 * 1024 * 256); 174 | 175 | // Allocator should not reuse node freed by A since the allocation C doesn't fits in the same bin 176 | // However node D and E fit there and should reuse node from A 177 | let a = allocator.allocate(1024).unwrap(); 178 | assert_eq!(a.offset, 0); 179 | 180 | let b = allocator.allocate(3456).unwrap(); 181 | assert_eq!(b.offset, 1024); 182 | 183 | allocator.free(a); 184 | 185 | let c = allocator.allocate(2345).unwrap(); 186 | assert_eq!(c.offset, 1024 + 3456); 187 | 188 | let d = allocator.allocate(456).unwrap(); 189 | assert_eq!(d.offset, 0); 190 | 191 | let e = allocator.allocate(512).unwrap(); 192 | assert_eq!(e.offset, 456); 193 | 194 | let report = allocator.storage_report(); 195 | assert_eq!( 196 | report.total_free_space, 197 | 1024 * 1024 * 256 - 3456 - 2345 - 456 - 512 198 | ); 199 | assert_ne!(report.largest_free_region, report.total_free_space); 200 | 201 | allocator.free(c); 202 | allocator.free(d); 203 | allocator.free(b); 204 | allocator.free(e); 205 | 206 | // End: Validate that allocator has no fragmentation left. Should be 100% clean. 207 | let validate_all = allocator.allocate(1024 * 1024 * 256).unwrap(); 208 | assert_eq!(validate_all.offset, 0); 209 | allocator.free(validate_all); 210 | } 211 | 212 | #[test] 213 | fn allocate_offset_allocator_zero_fragmentation() { 214 | let mut allocator: Allocator = Allocator::new(1024 * 1024 * 256); 215 | 216 | // Allocate 256x 1MB. Should fit. Then free four random slots and reallocate four slots. 217 | // Plus free four contiguous slots an allocate 4x larger slot. All must be zero fragmentation! 218 | let mut allocations: [_; 256] = array::from_fn(|i| { 219 | let allocation = allocator.allocate(1024 * 1024).unwrap(); 220 | assert_eq!(allocation.offset, i as u32 * 1024 * 1024); 221 | allocation 222 | }); 223 | 224 | let report = allocator.storage_report(); 225 | assert_eq!(report.total_free_space, 0); 226 | assert_eq!(report.largest_free_region, 0); 227 | 228 | // Free four random slots 229 | allocator.free(allocations[243]); 230 | allocator.free(allocations[5]); 231 | allocator.free(allocations[123]); 232 | allocator.free(allocations[95]); 233 | 234 | // Free four contiguous slots (allocator must merge) 235 | allocator.free(allocations[151]); 236 | allocator.free(allocations[152]); 237 | allocator.free(allocations[153]); 238 | allocator.free(allocations[154]); 239 | 240 | allocations[243] = allocator.allocate(1024 * 1024).unwrap(); 241 | allocations[5] = allocator.allocate(1024 * 1024).unwrap(); 242 | allocations[123] = allocator.allocate(1024 * 1024).unwrap(); 243 | allocations[95] = allocator.allocate(1024 * 1024).unwrap(); 244 | allocations[151] = allocator.allocate(1024 * 1024 * 4).unwrap(); // 4x larger 245 | 246 | for (i, allocation) in allocations.iter().enumerate() { 247 | if !(152..155).contains(&i) { 248 | allocator.free(*allocation); 249 | } 250 | } 251 | 252 | let report2 = allocator.storage_report(); 253 | assert_eq!(report2.total_free_space, 1024 * 1024 * 256); 254 | assert_eq!(report2.largest_free_region, 1024 * 1024 * 256); 255 | 256 | // End: Validate that allocator has no fragmentation left. Should be 100% clean. 257 | let validate_all = allocator.allocate(1024 * 1024 * 256).unwrap(); 258 | assert_eq!(validate_all.offset, 0); 259 | allocator.free(validate_all); 260 | } 261 | 262 | #[test] 263 | fn ext_min_allocator_size() { 264 | // Randomly generated integers on a log distribution, σ = 10. 265 | static TEST_OBJECT_SIZES: [u32; 42] = [ 266 | 0, 1, 2, 3, 4, 5, 8, 17, 23, 36, 51, 68, 87, 151, 165, 167, 201, 223, 306, 346, 394, 411, 267 | 806, 969, 1404, 1798, 2236, 4281, 4745, 13989, 21095, 26594, 27146, 29679, 144685, 153878, 268 | 495127, 727999, 1377073, 9440387, 41994490, 68520116, 269 | ]; 270 | 271 | for needed_object_size in TEST_OBJECT_SIZES { 272 | let allocator_size = ext::min_allocator_size(needed_object_size); 273 | let mut allocator: Allocator = Allocator::new(allocator_size); 274 | assert!(allocator.allocate(needed_object_size).is_some()); 275 | } 276 | } 277 | --------------------------------------------------------------------------------