├── .gitignore ├── .travis.yml ├── Cargo.toml ├── LICENSE ├── README.md ├── build.rs ├── c ├── cache.c ├── cache.h ├── constants.c ├── constants.h ├── mag.c ├── mag.h ├── map.c ├── map.h ├── span_metadata.c ├── span_metadata.h ├── stack.c └── stack.h ├── doc ├── design.md └── fast_path.md ├── examples └── demo.c ├── include └── slitter.h └── src ├── cache.rs ├── class.rs ├── debug_allocation_map.rs ├── debug_arange_map.rs ├── debug_type_map.rs ├── file_backed_mapper.rs ├── individual.rs ├── lib.rs ├── linear_ref.rs ├── magazine.rs ├── magazine_impl.rs ├── magazine_stack.rs ├── map.rs ├── mapper.rs ├── mill.rs ├── press.rs └── rack.rs /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | demo 10 | 11 | # These are backup files generated by rustfmt 12 | **/*.rs.bk 13 | proptest-regressions 14 | *~ 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | rust: 3 | - 1.53.0 4 | - stable 5 | - beta 6 | - nightly 7 | arch: 8 | - amd64 9 | 10 | jobs: 11 | allow_failures: 12 | - rust: nightly 13 | 14 | os: linux 15 | dist: bionic 16 | cache: cargo 17 | 18 | before_script: | 19 | if [ "x$TRAVIS_RUST_VERSION" == xstable -a "x$TRAVIS_CPU_ARCH" == xamd64 ]; then 20 | cargo install cargo-tarpaulin 21 | fi 22 | 23 | script: 24 | - cargo clean 25 | - cargo build 26 | - PROPTEST_FORK=true cargo test --release 27 | - PROPTEST_FORK=true cargo test --release --no-default-features --features='check_contracts_in_tests' 28 | - PROPTEST_FORK=true cargo test --release --no-default-features --features='c_fast_path' 29 | 30 | after_success: | 31 | if [ "x$TRAVIS_RUST_VERSION" == xstable -a "x$TRAVIS_CPU_ARCH" == xamd64 ]; then 32 | PROPTEST_FORK=true cargo tarpaulin --release --features='test_only_small_constants' --ciserver travis-ci --coveralls $TRAVIS_JOB_ID 33 | fi 34 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "slitter" 3 | version = "0.1.0" 4 | edition = "2018" 5 | license = "MIT" 6 | description = "A C- and Rust-callable slab allocator with a focus on safety" 7 | repository = "https://github.com/backtrace-labs/slitter" 8 | 9 | [lib] 10 | crate-type = ["lib", "staticlib"] 11 | 12 | [features] 13 | default = ["check_contracts_in_tests", "c_fast_path"] 14 | c_fast_path = [] # Use C, and not Rust, for the fast path. 15 | check_contracts_in_tests = [] # Enable contract checking for cfg(test). 16 | check_contracts = ["contracts"] # Enable contract checking. 17 | test_only_small_constants = [] # Shrink constants to cover more conditions. 18 | 19 | [dependencies] 20 | contracts = { version = "0.6", optional = true } 21 | disabled_contracts = "0.1" 22 | lazy_static = "1" 23 | static_assertions = "1.1" 24 | tempfile = "3" 25 | 26 | [build-dependencies] 27 | cc = "1" 28 | 29 | [dev-dependencies] 30 | contracts = { version = "0.6" } # Only used in tests. 31 | proptest = "1" # Run with `PROPTEST_FORK=true cargo test`, otherwise 32 | # global state fills up and slows down debug checks. 33 | # Also `--features='test_only_small_constants'` to help 34 | # exercise more edge cases. 35 | 36 | [package.metadata.x] # `cargo install cargo-x`, then `cargo x test`, etc. 37 | test = "PROPTEST_FORK=true cargo test" 38 | test_opt = "PROPTEST_FORK=true cargo test --release" 39 | test_release = "PROPTEST_FORK=true cargo test --release --no-default-features --features='c_fast_path'" 40 | test_small_constants = "PROPTEST_FORK=true cargo test --features='test_only_small_constants'" 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Backtrace I/O, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Slitter is a less footgunny slab allocator 2 | ========================================== 3 | [![Build Status](https://app.travis-ci.com/backtrace-labs/slitter.svg?branch=main)](https://app.travis-ci.com/backtrace-labs/slitter) 4 | 5 | Slitter is a classically structured thread-caching slab allocator 6 | that's meant to help write reliable long-running programs. 7 | 8 | Given this goal, Slitter does not prioritise speed. The primary goal 9 | is instead to help applications handle inevitable memory management 10 | errors--be it with built-in statistics, with runtime detection, or by 11 | controlling their blast radius--while keeping the allocator's 12 | performance competitive with the state of the art. The other 13 | important goal is to let applications customise how Slitter requests 14 | memory from the operating system. 15 | 16 | See `doc/fast_path.md` for details on allocation performance. For the 17 | rest of the allocator at a high level, refer to `doc/design.md`. Due 18 | to the type-stability guarantee, external fragmentation is essentially 19 | out of scope, and the slab/bump pointer allocation scheme keeps 20 | internal fragmentation to a very low level (only for metadata and 21 | guard pages, on the order of 2-3%, more once we add internal guard 22 | pages). 23 | 24 | Current guarantees: 25 | 26 | 1. Slitter will detect mismatching classes when recycling allocations, 27 | and will often also crash when it receives an address that it does 28 | not manage. Without this guarantee, it's too easy for heap usage 29 | statistics to become useless, and for incorrectly paired release 30 | calls to turn into memory corruption, far from the erroneous call. 31 | 32 | 2. Slitter does not have any in-band metadata. This means no metadata 33 | next to the application's allocations, ripe for buffer overflows 34 | (we maintain guard pages between data and metadata), and also no 35 | intrusive linked list that would be vulnerable to use-after-free. 36 | 37 | 3. Slitter-allocated data has a stable type: once an address has been 38 | returned to the mutator for a given allocation class, that address 39 | will always be valid, and will always be used for data of that 40 | class. Per #2, Slitter does not use intrusive linked lists, so the 41 | data reflects what the application stored there, even once it has 42 | been recycled. This lets application code rely on benign 43 | use-after-free in non-blocking algorithms instead of, e.g., SMR. 44 | The guarantee also means that any double-free or malign 45 | use-after-free will only affect the faulty allocation class. 46 | 47 | Future guarantees: 48 | 49 | 4. Slitter will detect when an interior pointer is freed. 50 | 51 | 5. Slitter will detect most buffer overflows that cross allocation 52 | classes, with guard pages. 53 | 54 | 6. Slitter will always detect frees of addresses it does not manage. 55 | 56 | 7. Slitter will detect most back-to-back double-frees. 57 | 58 | Future features: 59 | 60 | a. Slitter will let each allocation class determine how its backing 61 | memory should be allocated (e.g., cold data could live in a 62 | file-backed mapping for opt-in swapping). 63 | 64 | b. Slitter will track the number of objects allocated and recycled in 65 | each allocation class. 66 | 67 | c. Slitter will sample a small fraction of allocations for heap 68 | profiling. 69 | 70 | How to use Slitter as a C library 71 | --------------------------------- 72 | 73 | In order to use Slitter as a C library, we must first build a static 74 | library with Cargo. Slitter can *only* be called from C via static 75 | linkage because Cargo will otherwise hide our C functions. However, 76 | this also exposes internal Rust symbols, so there can only be one 77 | statically linked Rust library in any executable. 78 | 79 | See `examples/demo.c` for a sample integration, which also 80 | demonstrates some of the additional checks unlocked by explicit 81 | allocation class tags. Execute that file in sh (i.e., `sh 82 | examples/demo.c`) to build Slitter, build `demo.c` and link it against 83 | Slitter, and run the resulting `demo` executable. 84 | 85 | The `#ifdef MISMATCH` section allocates an object of a derived struct 86 | type, and releases it as the base type. The base field is the first 87 | member of the derived struct, so a plain malloc/free interface is 88 | unable to tell the difference. However, since the caller must tell 89 | `slitter_release` what allocation class it expects the freed pointer 90 | to be in, Slitter can detect the mismatch. 91 | 92 | How to use Slitter within a Rust uber-crate 93 | ------------------------------------------- 94 | 95 | When linking multiple Rust libraries with other languages like C or 96 | C++, one must build a single statically linked (`crate-type = 97 | ["staticlib"]`) Rust crate that depends on all the Rust libraries we 98 | want to expose, and make sure to re-export the public `extern "C"` 99 | definitions from each of the dependencies. 100 | 101 | How to use Slitter from Rust 102 | ---------------------------- 103 | 104 | We haven't explored that use case, except for tests. You can look at 105 | the unit tests at the bottom on `src/class.rs`. TL;DR: create new 106 | `Class` objects (they're copiable wrappers around `NonZeroU32`), and 107 | call `Class::allocate` and `Class::release`. 108 | -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let mut build = cc::Build::new(); 3 | 4 | // Match the override in magazine_impl.rs 5 | #[cfg(feature = "test_only_small_constants")] 6 | build.define("SLITTER__SMALL_CONSTANTS", "1"); 7 | 8 | for file in ["cache", "constants", "mag", "map", "span_metadata", "stack"].iter() { 9 | println!("cargo:rerun-if-changed=c/{}.c", file); 10 | println!("cargo:rerun-if-changed=c/{}.h", file); 11 | 12 | build.file(format!("c/{}.c", file)); 13 | } 14 | 15 | // This forces a cdylib to include the routines, but does not 16 | // export the symbols... 17 | println!("cargo:rustc-cdylib-link-arg=-uslitter_allocate"); 18 | println!("cargo:rustc-cdylib-link-arg=-uslitter_release"); 19 | 20 | build 21 | .include("include") 22 | .opt_level(2) 23 | .flag_if_supported("-mcx16") // enable CMPXCHB16B 24 | .flag("-W") 25 | .flag("-Wall") 26 | .compile("slitter_support") 27 | } 28 | -------------------------------------------------------------------------------- /c/cache.c: -------------------------------------------------------------------------------- 1 | #include "cache.h" 2 | 3 | #include 4 | 5 | #include "constants.h" 6 | #include "span_metadata.h" 7 | 8 | struct thread_cache { 9 | size_t n; 10 | struct cache_magazines *mags; 11 | }; 12 | 13 | struct thread_allocation { 14 | struct thread_cache cache; 15 | /* Add one more for the dummy class (and to avoid zero-sized arrays). */ 16 | struct cache_magazines preallocated[1 + SLITTER__CACHE_PREALLOC]; 17 | }; 18 | 19 | static __thread struct thread_allocation slitter_cache 20 | __attribute__((tls_model("initial-exec"))); 21 | 22 | /** 23 | * Defined in cache.rs 24 | */ 25 | extern void *slitter__allocate_slow(struct slitter_class); 26 | extern void slitter__release_slow(struct slitter_class, void *); 27 | 28 | struct cache_magazines * 29 | slitter__cache_borrow(size_t *OUT_n) 30 | { 31 | 32 | *OUT_n = sizeof(slitter_cache.preallocated) 33 | / sizeof(slitter_cache.preallocated[0]); 34 | return slitter_cache.preallocated; 35 | } 36 | 37 | void 38 | slitter__cache_register(struct cache_magazines *mags, size_t n) 39 | { 40 | slitter_cache.cache = (struct thread_cache) { 41 | .n = n, 42 | .mags = mags, 43 | }; 44 | 45 | return; 46 | } 47 | 48 | void * 49 | slitter_allocate(struct slitter_class class) 50 | { 51 | struct magazine *restrict mag; 52 | size_t next_index; 53 | uint32_t id = class.id; 54 | 55 | if (__builtin_expect(id >= slitter_cache.cache.n, 0)) 56 | return slitter__allocate_slow(class); 57 | 58 | mag = &slitter_cache.cache.mags[id].alloc; 59 | if (__builtin_usubl_overflow(mag->top_of_stack, 2, &next_index)) { 60 | next_index++; 61 | } 62 | 63 | if (__builtin_expect(slitter__magazine_is_exhausted(mag), 0)) 64 | return slitter__allocate_slow(class); 65 | 66 | /* 67 | * The magazine was not empty, so next_index did not overflow 68 | * by more than 1. 69 | */ 70 | __builtin_prefetch(mag->storage->allocations[next_index], 1); 71 | return slitter__magazine_get_non_empty(mag); 72 | } 73 | 74 | void 75 | slitter_release(struct slitter_class class, void *ptr) 76 | { 77 | uintptr_t address = (uintptr_t)ptr; 78 | uintptr_t chunk_base = address & -SLITTER__DATA_ALIGNMENT; 79 | uintptr_t chunk_offset = address % SLITTER__DATA_ALIGNMENT; 80 | size_t span_index = chunk_offset / SLITTER__SPAN_ALIGNMENT; 81 | uintptr_t meta_base = chunk_base - 82 | (SLITTER__GUARD_PAGE_SIZE + SLITTER__METADATA_PAGE_SIZE); 83 | struct magazine *restrict mag; 84 | uint32_t id = class.id; 85 | 86 | if (ptr == NULL) 87 | return; 88 | 89 | /* Check the span metadata. */ 90 | { 91 | const struct span_metadata *meta = (void *)meta_base; 92 | const struct span_metadata *span = &meta[span_index]; 93 | 94 | assert(class.id == span->class_id && "class mismatch"); 95 | } 96 | 97 | if (__builtin_expect(id >= slitter_cache.cache.n, 0)) 98 | return slitter__release_slow(class, ptr); 99 | 100 | mag = &slitter_cache.cache.mags[id].release; 101 | if (__builtin_expect(slitter__magazine_is_exhausted(mag), 0)) 102 | return slitter__release_slow(class, ptr); 103 | 104 | return slitter__magazine_put_non_full(mag, ptr); 105 | } 106 | -------------------------------------------------------------------------------- /c/cache.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "slitter.h" 3 | 4 | #include 5 | 6 | #include "mag.h" 7 | 8 | /* 9 | * Matches the `Magazines` struct in `cache.rs`. 10 | */ 11 | struct cache_magazines { 12 | struct magazine alloc; 13 | struct magazine release; 14 | }; 15 | 16 | /** 17 | * Returns the thread's pre-allocated array of `cache_magazines`. 18 | * 19 | * That array lives next to the fast-path's internal thread-local data 20 | * structure, so using that array improves locality. 21 | */ 22 | struct cache_magazines *slitter__cache_borrow(size_t *OUT_n); 23 | 24 | /** 25 | * Registers an array of `n` `cache_magazines` for this thread. 26 | * 27 | * That array may be the same one returned by `slitter__cache_borrow`. 28 | */ 29 | void slitter__cache_register(struct cache_magazines *, size_t n); 30 | -------------------------------------------------------------------------------- /c/constants.c: -------------------------------------------------------------------------------- 1 | #include "constants.h" 2 | 3 | size_t 4 | slitter__magazine_size(void) 5 | { 6 | 7 | return SLITTER__MAGAZINE_SIZE; 8 | } 9 | 10 | size_t 11 | slitter__data_alignment(void) 12 | { 13 | 14 | return SLITTER__DATA_ALIGNMENT; 15 | } 16 | 17 | size_t 18 | slitter__guard_page_size(void) 19 | { 20 | 21 | return SLITTER__GUARD_PAGE_SIZE; 22 | } 23 | 24 | size_t 25 | slitter__metadata_page_size(void) 26 | { 27 | 28 | return SLITTER__METADATA_PAGE_SIZE; 29 | } 30 | 31 | size_t 32 | slitter__span_alignment(void) 33 | { 34 | 35 | return SLITTER__SPAN_ALIGNMENT; 36 | } 37 | -------------------------------------------------------------------------------- /c/constants.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | /* 6 | * Define default constants 7 | */ 8 | #ifndef SLITTER__SMALL_CONSTANTS 9 | # define SLITTER__MAGAZINE_SIZE 30 10 | # define SLITTER__DATA_ALIGNMENT (1UL << 30) 11 | # define SLITTER__GUARD_PAGE_SIZE (2UL << 20) 12 | # define SLITTER__METADATA_PAGE_SIZE (2UL << 20) 13 | # define SLITTER__SPAN_ALIGNMENT (16UL << 10) 14 | 15 | /* 16 | * The default of classes we can handle in the thread-local cache's 17 | * preallocated array of `cache_magazines`. We need one more for the 18 | * dummy 0 class. 19 | * 20 | * At 32 bytes per class, 15 classes take up 512 thread-local bytes. 21 | */ 22 | #ifndef SLITTER__CACHE_PREALLOC 23 | # define SLITTER__CACHE_PREALLOC 15 24 | #endif 25 | 26 | #else 27 | # define SLITTER__MAGAZINE_SIZE 6 28 | # define SLITTER__DATA_ALIGNMENT (2UL << 20) 29 | # define SLITTER__GUARD_PAGE_SIZE (16UL << 10) 30 | # define SLITTER__METADATA_PAGE_SIZE (16UL << 10) 31 | # define SLITTER__SPAN_ALIGNMENT (4UL << 10) 32 | 33 | #ifndef SLITTER__CACHE_PREALLOC 34 | # define SLITTER__CACHE_PREALLOC 3 35 | #endif 36 | 37 | #endif 38 | 39 | /** 40 | * Returns the value of the `SLITTER__MAGAZINE_SIZE` constant on the C 41 | * side. 42 | * 43 | * The Rust code uses this function to confirm that the constant has 44 | * the same value on both sides. 45 | */ 46 | size_t slitter__magazine_size(void); 47 | 48 | /** 49 | * Returns the value of the `SLITTER__DATA_ALIGNMENT` constant on the 50 | * C side. 51 | */ 52 | size_t slitter__data_alignment(void); 53 | 54 | /** 55 | * Returns the value of the `SLITTER__GUARD_PAGE_SIZE` constant on the 56 | * C side. 57 | */ 58 | size_t slitter__guard_page_size(void); 59 | 60 | /** 61 | * Returns the value of the `SLITTER__METADATA_PAGE_SIZE` constant on 62 | * the C side. 63 | */ 64 | size_t slitter__metadata_page_size(void); 65 | 66 | /** 67 | * Returns the value of the `SLITTER__SPAN_ALIGNMENT` constant on the 68 | * C side. 69 | */ 70 | size_t slitter__span_alignment(void); 71 | -------------------------------------------------------------------------------- /c/mag.c: -------------------------------------------------------------------------------- 1 | #include "mag.h" 2 | 3 | extern bool slitter__magazine_is_exhausted(const struct magazine *); 4 | extern void *slitter__magazine_get_non_empty(struct magazine *); 5 | extern void slitter__magazine_put_non_full(struct magazine *, void *); 6 | 7 | void * 8 | slitter__magazine_get(struct magazine *restrict mag) 9 | { 10 | 11 | if (slitter__magazine_is_exhausted(mag)) 12 | return NULL; 13 | 14 | return slitter__magazine_get_non_empty(mag); 15 | } 16 | 17 | void * 18 | slitter__magazine_put(struct magazine *restrict mag, void *alloc) 19 | { 20 | 21 | if (slitter__magazine_is_exhausted(mag)) 22 | return alloc; 23 | 24 | slitter__magazine_put_non_full(mag, alloc); 25 | return NULL; 26 | } 27 | 28 | size_t 29 | slitter__magazine_capacity(void) 30 | { 31 | 32 | return SLITTER__MAGAZINE_SIZE; 33 | } 34 | 35 | size_t 36 | slitter__magazine_storage_sizeof(void) 37 | { 38 | 39 | return sizeof(struct magazine_storage); 40 | } 41 | 42 | size_t 43 | slitter__magazine_sizeof(void) 44 | { 45 | 46 | return sizeof(struct magazine); 47 | } 48 | -------------------------------------------------------------------------------- /c/mag.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "constants.h" 9 | 10 | /** 11 | * Matches `MagazineStorage` on the Rust side. 12 | */ 13 | struct magazine_storage { 14 | void *allocations[SLITTER__MAGAZINE_SIZE]; 15 | /* 16 | * The `link` pointer is only used by the C side. 17 | * It's always NULL (None) on the Rust side. 18 | */ 19 | struct magazine_storage *volatile link; 20 | uint32_t num_allocated_slow; 21 | }; 22 | 23 | /** 24 | * Matches `MagazineImpl` on the Rust side. 25 | * 26 | * `top_of_stack` goes from SLITTER__MAGAZINE_SIZE to 0 when popping, 27 | * and from -SLITTER__MAGAZINE_SIZE to 0 when pushing. In both cases, 28 | * `storage->allocations` is populated with cached objects 29 | * at low indices, and empty / garbage at high ones. 30 | */ 31 | struct magazine { 32 | ssize_t top_of_stack; 33 | struct magazine_storage *storage; 34 | }; 35 | 36 | /** 37 | * Returns whether the magazine is exhausted (empty for push 38 | * magazines, full for pop ones). 39 | */ 40 | inline bool 41 | slitter__magazine_is_exhausted(const struct magazine *restrict mag) 42 | { 43 | 44 | return mag->top_of_stack == 0; 45 | } 46 | 47 | /** 48 | * Consumes one cached allocation from a non-exhausted "Pop" magazine. 49 | */ 50 | inline void * 51 | slitter__magazine_get_non_empty(struct magazine *restrict mag) 52 | { 53 | void *ret; 54 | 55 | ret = mag->storage->allocations[--mag->top_of_stack]; 56 | if (ret == NULL) 57 | __builtin_unreachable(); 58 | 59 | return ret; 60 | } 61 | 62 | /** 63 | * Pushes one cached allocation to a non-exhausted "Push" magazine. 64 | */ 65 | inline void 66 | slitter__magazine_put_non_full(struct magazine *restrict mag, void *alloc) 67 | { 68 | 69 | mag->storage->allocations 70 | [SLITTER__MAGAZINE_SIZE + mag->top_of_stack++] = alloc; 71 | return; 72 | } 73 | 74 | /** 75 | * Attempts to consume one cached allocation from a "Pop" magazine. 76 | * 77 | * Returns the cached allocation on success, NULL on failure. 78 | */ 79 | void *slitter__magazine_get(struct magazine *restrict mag); 80 | 81 | /** 82 | * Attempts to push one allocation to a "Push" magazine. 83 | * 84 | * Returns NULL on success, `alloc` on failure. 85 | */ 86 | void *slitter__magazine_put(struct magazine *restrict mag, void *alloc); 87 | 88 | /** 89 | * Returns `sizeof(struct magazine_storage)`. 90 | */ 91 | size_t slitter__magazine_storage_sizeof(void); 92 | 93 | /** 94 | * Returns `sizeof(struct magazine)`. 95 | */ 96 | size_t slitter__magazine_sizeof(void); 97 | -------------------------------------------------------------------------------- /c/map.c: -------------------------------------------------------------------------------- 1 | #include "map.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | static_assert(sizeof(size_t) == sizeof(uintptr_t), 9 | "Our rust code assumes usize == size_t, but rust-the-language " 10 | "only guarantees usize == uintptr_t."); 11 | 12 | int64_t 13 | slitter__page_size(void) 14 | { 15 | long ret; 16 | 17 | ret = sysconf(_SC_PAGESIZE); 18 | if (ret < 0) 19 | return -errno; 20 | 21 | return ret; 22 | } 23 | 24 | void * 25 | slitter__reserve_region(size_t desired_size, int32_t *OUT_errno) 26 | { 27 | void *ret; 28 | 29 | *OUT_errno = 0; 30 | ret = mmap(NULL, desired_size, PROT_NONE, 31 | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 32 | if (ret != MAP_FAILED) { 33 | assert(ret != NULL && "We assume NULL is not a valid address"); 34 | return ret; 35 | } 36 | 37 | *OUT_errno = errno; 38 | return NULL; 39 | } 40 | 41 | int32_t 42 | slitter__release_region(void *base, size_t size) 43 | { 44 | 45 | if (size == 0 || munmap(base, size) == 0) 46 | return 0; 47 | 48 | return -errno; 49 | } 50 | 51 | int32_t 52 | slitter__allocate_region(void *base, size_t size) 53 | { 54 | void *ret; 55 | 56 | ret = mmap(base, size, PROT_READ | PROT_WRITE, 57 | MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 58 | if (ret != MAP_FAILED) 59 | return 0; 60 | 61 | return -errno; 62 | } 63 | 64 | int32_t 65 | slitter__allocate_fd_region(int fd, size_t offset, void *base, size_t size) 66 | { 67 | void *ret; 68 | 69 | ret = mmap(base, size, PROT_READ | PROT_WRITE, 70 | MAP_FIXED | MAP_SHARED, fd, (off_t)offset); 71 | if (ret != MAP_FAILED) 72 | return 0; 73 | 74 | return -errno; 75 | } 76 | -------------------------------------------------------------------------------- /c/map.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * Internal low-level memory mapping code for Slitter. We use C 4 | * instead of relying on (unstable) `libc`. 5 | * 6 | * The corresponding Rust definition live in `src/map.rs`. 7 | */ 8 | 9 | #include 10 | #include 11 | 12 | /** 13 | * Returns the system page size, or `-errno` on failure. 14 | */ 15 | int64_t slitter__page_size(void); 16 | 17 | /** 18 | * Attempts to reserve a region of address space of `desired_size` 19 | * bytes. 20 | * 21 | * On success, returns the address of the first byte in the 22 | * new region and overwrites `OUT_errno` with 0. 23 | * 24 | * On failure, returns NULL and overwrites `OUT_errno` with the 25 | * `errno` from `mmap`. 26 | */ 27 | void *slitter__reserve_region(size_t desired_size, 28 | int32_t *OUT_errno); 29 | 30 | /** 31 | * Attempts to release the region of address space starting at `base`, 32 | * and continuing for `size` bytes. 33 | * 34 | * Returns 0 on success, and `-errno` on failure. 35 | */ 36 | int32_t slitter__release_region(void *base, size_t size); 37 | 38 | /** 39 | * Attempts to back the region of address space starting at `base` 40 | * and continuing for `size` bytes with actual memory. The caller 41 | * must have first acquired ownership of the address space with 42 | * `slitter__reserve_region`. 43 | * 44 | * The region will be safe for read and writes, but may be 45 | * demand-faulted later. 46 | * 47 | * Returns 0 on success, and `-errno` on failure. 48 | */ 49 | int32_t slitter__allocate_region(void *base, size_t size); 50 | 51 | /** 52 | * Attempts to back the region of address space starting at `base` and 53 | * continuing for `size` bytes with memory from `fd`, starting at 54 | * `offset`. The caller must have first acquired ownership of the 55 | * address space with `slitter__reserve_region`. 56 | * 57 | * The region will be safe for read and writes, but may be 58 | * demand-faulted later. 59 | * 60 | * Returns 0 on success, and `-errno` on failure. 61 | */ 62 | int32_t slitter__allocate_fd_region(int fd, size_t offset, 63 | void *base, size_t size); 64 | -------------------------------------------------------------------------------- /c/span_metadata.c: -------------------------------------------------------------------------------- 1 | #include "span_metadata.h" 2 | 3 | size_t 4 | slitter__span_metadata_size(void) 5 | { 6 | 7 | return sizeof(struct span_metadata); 8 | } 9 | -------------------------------------------------------------------------------- /c/span_metadata.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | /** 7 | * Must match `SpanMetadata` in `mill.rs`. 8 | */ 9 | struct span_metadata { 10 | uint32_t class_id; 11 | uint32_t bump_limit; 12 | uintptr_t bump_ptr; 13 | uintptr_t span_begin; 14 | }; 15 | 16 | /** 17 | * Returns the size of `struct span_metadata` in C. 18 | */ 19 | size_t slitter__span_metadata_size(void); 20 | -------------------------------------------------------------------------------- /c/stack.c: -------------------------------------------------------------------------------- 1 | #include "stack.h" 2 | 3 | #include 4 | 5 | static_assert(sizeof(struct stack) == 2 * sizeof(void *), 6 | "A stack must consist of exactly two pointers."); 7 | 8 | #define LOAD_ACQUIRE(X) __atomic_load_n(&(X), __ATOMIC_ACQUIRE) 9 | #define STORE_RELEASE(X, V) __atomic_store_n(&(X), (V), __ATOMIC_RELEASE) 10 | 11 | void 12 | slitter__stack_push(struct stack *stack, struct magazine_storage *mag) 13 | { 14 | struct stack curr, next; 15 | 16 | /* 17 | * Make sure to load `generation` first: it's our monotonic 18 | * counter, so, if the CAS claims the `generation` hasn't 19 | * changed since the read of `top_of_stack`, we have a 20 | * consistent snapshot. 21 | * 22 | * These could be relaxed (and thus in any program order) if 23 | * we could easily access the CAS's success flag and the old 24 | * value on failure. Unfortunately, GCC always falls back to 25 | * libatomic for C11 u128 operations, so we have to make do 26 | * with the legacy. 27 | * 28 | * These acquires don't actually pair with any release (they 29 | * could pair with successful CASes). 30 | */ 31 | curr.generation = LOAD_ACQUIRE(stack->generation); 32 | curr.top_of_stack = LOAD_ACQUIRE(stack->top_of_stack); 33 | 34 | for (;;) { 35 | struct stack actual; 36 | 37 | /* The CAS should be the release barrier. */ 38 | STORE_RELEASE(mag->link, curr.top_of_stack); 39 | next = (struct stack) { 40 | .top_of_stack = mag, 41 | .generation = curr.generation + 1, 42 | }; 43 | 44 | /* 45 | * GCC only obeys -mcx16 for the legacy "sync" atomics: 46 | * the C11 standard operations still rely on libatomic 47 | * in order to offer atomic loads... 48 | */ 49 | actual.bits = __sync_val_compare_and_swap(&stack->bits, 50 | curr.bits, next.bits); 51 | /* 52 | * If the generation matches, the CAS succeeded: 53 | * tearing only happens in the first iteration, and 54 | * that comes from a sequence that loads generation 55 | * before top_of_stack. Subsequent iteration use the 56 | * atomic snapshot provided by the CAS. 57 | * 58 | * This sad workaround for the lack of a `__sync` 59 | * operation that returns both the success flag and 60 | * the actual value on failure *only works because 61 | * all operations increment the generation counter*. 62 | * 63 | * In theory, it would be safe for `push` to perform a 64 | * regular CAS, without changing the generation 65 | * counter. However, the difference is marginal (an 66 | * atomic is an atomic), correctness slightly more 67 | * involved, and we'd have to compare both words 68 | * when popping... and popping is on the allocation 69 | * path, which is marginally more latency sensitive 70 | * than the release path. 71 | */ 72 | if (__builtin_expect(actual.generation == curr.generation, 1)) 73 | return; 74 | 75 | curr = actual; 76 | } 77 | 78 | return; 79 | } 80 | 81 | bool 82 | slitter__stack_pop(struct stack *stack, struct magazine_storage **out) 83 | { 84 | struct stack curr, next; 85 | 86 | curr.generation = LOAD_ACQUIRE(stack->generation); 87 | curr.top_of_stack = LOAD_ACQUIRE(stack->top_of_stack); 88 | if (curr.top_of_stack == NULL) 89 | return false; 90 | 91 | for (;;) { 92 | struct stack actual; 93 | struct magazine_storage *tos = curr.top_of_stack; 94 | 95 | if (__builtin_expect(tos == NULL, 0)) 96 | return false; 97 | 98 | /* 99 | * The ordering semantics of 100 | * `__sync_val_compare_and_swap` aren't super clear. 101 | * Insert an explicit acquire (load-load) fence here, 102 | * before deferencing `curr.top_of_stack`, which may 103 | * come from the CAS. The CAS should be acquire on 104 | * failure. 105 | */ 106 | __atomic_thread_fence(__ATOMIC_ACQUIRE); 107 | next = (struct stack) { 108 | .top_of_stack = LOAD_ACQUIRE(tos->link), 109 | .generation = curr.generation + 1, 110 | }; 111 | 112 | actual.bits = __sync_val_compare_and_swap(&stack->bits, 113 | curr.bits, next.bits); 114 | if (__builtin_expect(actual.generation == curr.generation, 1)) { 115 | tos->link = NULL; 116 | *out = tos; 117 | return true; 118 | } 119 | 120 | curr = actual; 121 | } 122 | 123 | return false; 124 | } 125 | 126 | bool 127 | slitter__stack_try_pop(struct stack *stack, struct magazine_storage **out) 128 | { 129 | struct stack actual, curr, next; 130 | struct magazine_storage *tos; 131 | 132 | curr.generation = LOAD_ACQUIRE(stack->generation); 133 | curr.top_of_stack = LOAD_ACQUIRE(stack->top_of_stack); 134 | 135 | tos = curr.top_of_stack; 136 | if (tos == NULL) 137 | return false; 138 | 139 | next = (struct stack) { 140 | .top_of_stack = tos->link, 141 | .generation = curr.generation + 1, 142 | }; 143 | 144 | actual.bits = __sync_val_compare_and_swap(&stack->bits, 145 | curr.bits, next.bits); 146 | if (__builtin_expect(actual.generation == curr.generation, 1)) { 147 | tos->link = NULL; 148 | *out = tos; 149 | return true; 150 | } 151 | 152 | return false; 153 | } 154 | -------------------------------------------------------------------------------- /c/stack.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "mag.h" 7 | 8 | /** 9 | * A `stack` matches the `MagazineStack` in Rust. 10 | * 11 | * The lock-free implementation is a straightforward double-wide-CAS 12 | * logic, with a generation counter for ABA protection: we don't have 13 | * to worry about safe memory reclamation because `struct 14 | * magazine_storage` are immortal. 15 | */ 16 | struct __attribute__((__aligned__(16))) stack { 17 | union { 18 | struct { 19 | struct magazine_storage *top_of_stack; 20 | uintptr_t generation; 21 | }; 22 | __uint128_t bits; 23 | }; 24 | }; 25 | 26 | /** 27 | * Pushes a new magazine to the stack. 28 | */ 29 | void slitter__stack_push(struct stack *, struct magazine_storage *); 30 | 31 | /** 32 | * Attempts to pop one element from the stack. 33 | * 34 | * On success, returns true and populates the `out` pointer. 35 | * On failure, returns false. 36 | */ 37 | bool slitter__stack_pop(struct stack *, struct magazine_storage **out); 38 | 39 | /** 40 | * Quickly attempts to pop one element from the stack. 41 | * 42 | * Once success, returns true and populates the `out` pointer. 43 | * On failure, returns false. 44 | * 45 | * Unlike `slitter__stack_pop`, this function may fail for any reason. 46 | */ 47 | bool slitter__stack_try_pop(struct stack *, struct magazine_storage **out); 48 | -------------------------------------------------------------------------------- /doc/design.md: -------------------------------------------------------------------------------- 1 | The high-level design of Slitter 2 | ================================ 3 | 4 | Slitter is a slab allocator that heavily relies on monotonic state to 5 | keep its code easy to understand, even once we replace locks with 6 | non-blocking atomic operations, and to ensure misuses crash early or 7 | at least remain localised to the buggy allocation class. 8 | 9 | At its core, Slitter is a [magazine-caching slab allocator](https://www.usenix.org/legacy/publications/library/proceedings/usenix01/full_papers/bonwick/bonwick.pdf), 10 | except that the caches are per-thread rather than per-CPU. 11 | 12 | Each allocation class must be registered before being used in 13 | allocations or deallocations. Classes are assigned opaque identifiers 14 | linearly, and are immortal: once a class has been registered, it 15 | cannot be unregistered, and its identifier is never reused (i.e., 16 | the set of classes is another instance of monotonic state). 17 | 18 | The data model 19 | -------------- 20 | 21 | The data model is hierarchical; the only deviation from a pure 22 | hierarchy (`Mapper`, `Mill`, `Press`/`ClassInfo`, thread cache entry) 23 | is the thread cache (one thread cache contains one entry for each 24 | allocation class), and the `Rack`, which is shared by multiple 25 | `ClassInfo`, independently of the `Mill`. 26 | 27 | 1. Each thread has a thread cache (`cache.rs`) for all the allocations 28 | classes that are known (a vector, indexed with the class's id). 29 | 30 | 2. Each cache entry refers to its class's immortal `ClassInfo` struct 31 | (`class.rs`). 32 | 33 | 3. The `ClassInfo` (`class.rs`) struct contains read-only information 34 | about the class and two freelists of magazines, and refers to an 35 | immortal `Rack` (`magazine.rs`), and owns a `Press` (`press.rs`). 36 | 37 | 4. The `Rack` is shared between an arbitrary number of `ClassInfo`, 38 | and handles the allocation and recycling of empty magazines 39 | (bounded vectors of cached allocations). 40 | 41 | 5. Each `Press` is specific to the class, and allocates new objects 42 | for the class. Allocations are mostly serviced with a bump 43 | pointer; a `Press` refers to an immortal `Mill` (`mill.rs`) from 44 | which it obtains new address ranges for bump allocation. 45 | 46 | 6. The `Mill` is shared between an arbitrary number of `Press`es, 47 | and handles parcelling out address space. Requests are again 48 | mostly serviced from a 1GB "chunk" with a bump pointer. When 49 | a `Mill` needs a new chunk, it defers to an immortal `Mapper` 50 | to grab address space from the OS, release unused slop at the 51 | edges, and mark some of that space as ready for memory accesses. 52 | 53 | The heap layout 54 | --------------- 55 | 56 | Slitter carves out allocations from independent `Chunk`s of data. 57 | Each `Chunk`'s data is a 1 GB-aligned range of 1 GB, and its array of 58 | metadata lives in a 2 MB region that starts 4 MB below the data 59 | region. Slitter also keeps 2 MB guard regions before the metadata, 60 | between the metadata and the data, and after the data. 61 | 62 | The data region is incrementally partitioned into `Span`s, which are 63 | always aligned to `SPAN_ALIGNMENT` both in address and in size. Each 64 | `SPAN_ALIGNMENT` range of data bytes maps to a `SpanMetadata` struct 65 | in the metadata region: the first range maps to the first struct in 66 | the region, the second to the next, etc. 67 | 68 | A given `Span` (and thus its constituent span-aligned ranges) is only 69 | used to satisfy allocations of a single class. This makes it easy to 70 | guarantee alignment, and to confirm that deallocation requests make 71 | sense. 72 | 73 | The allocation flow, from the outside in 74 | ---------------------------------------- 75 | 76 | The allocation fast path for a class id `C` finds the `C`th entry in 77 | the thread-local cache, and looks at the magazine stored there. 78 | Slitter currently only has one allocation magazine and one 79 | deallocation magazine per cache, as well as a "buffer": our target 80 | application mostly performs bursts of allocations and bursts of 81 | deallocation, so locality of repeated allocations and deallocations 82 | isn't a concern... at least not as much as checking deallocations in 83 | the slow path. 84 | 85 | If that magazine still has some allocations, we pop off one 86 | allocation, and return that to the caller. 87 | 88 | Otherwise, we must enter the slow path. 89 | 90 | The slow path (in `cache.rs`) ensures that: 91 | 92 | 1. The thread-local cache has an entry for class id `C`; if it doesn't 93 | (the local cache array is too short), the local cache is extended to 94 | match all the allocation classes that are currently registered, which 95 | must include `C`, but may include other allocation classes. 96 | 97 | 2. The thread-local entry for class `C` has a non-empty magazine. 98 | 99 | 3. The allocation request is satisfied (usually from that magazine). 100 | 101 | The magazine allocation / refilling logic lives in `magazine.rs`, and 102 | mostly manipulates two intrusive LIFO freelists of magazines in the 103 | class's `ClassInfo` (one immortal struct per class): one for magazines 104 | that are fully populated, and another for magazines are partially 105 | populated (fully empty magazines go in the `Rack`). 106 | 107 | When the thread-local array must be extended, each entry is filled 108 | with a magazine, in an arbitrary state. The `ClassInfo` (all 109 | thread-local cache entries for a given class share the same 110 | `ClassInfo`) first pops from its freelists, and only defers to the 111 | `Rack` when both freelists are empty (multiple `ClassInfo`s share the 112 | same `Rack`). 113 | 114 | When the thread-local entry instead has an empty magazine, the 115 | `ClassInfo` refills that magazine. If the freelists aren't empty, the 116 | empty magazine is released to the `ClassInfo`'s `Rack` and replaced 117 | with one from the `ClassInfo`'s freelists. These freelists only 118 | contain non-empty magazines, so we can always satisfy at least one 119 | allocation from the newly popped magazine. 120 | 121 | When the freelists are empty, the `ClassInfo` hits the `Press` (each 122 | `ClassInfo` owns one `Press`) for new allocations: first, for the 123 | allocation itself, and then to opportunistically refill the currently 124 | empty magazine. 125 | 126 | The `Press` allocates from its current `Span` with a bump pointer. 127 | When the `Span` is exhausted, the `Press` asks its `Mill` (multiple 128 | `Press`es share the same `Mill`) for another span, and bumps the 129 | allocation index in that new spac. 130 | 131 | The `Mill` allocates from its current `Chunk` with a bump pointer. 132 | When the `Chunk` is exhausted, it asks its `Mapper` (multiple `Mill`s 133 | share the same `Mapper`) for another one. 134 | 135 | Finally, the mapper allocates address space by asking the operating 136 | system. 137 | 138 | The deallocation flow, from the outside in 139 | ------------------------------------------ 140 | 141 | The allocation fast path for a class id `C` finds the `C`th entry in 142 | the thread-local cache, and looks at the magazine stored there. 143 | 144 | If that magazine isn't full, we push the newly released allocation 145 | to the magazine, and return to the caller. 146 | 147 | Otherwise, we must enter the deallocation slow path. 148 | 149 | The slow path (in `cache.rs`) ensures that: 150 | 151 | 1. The thread-local cache has an entry for class id `C`; if it doesn't 152 | (the local cache array is too short), the local cache is extended to 153 | match all the allocation classes that are currently registered, which 154 | must include `C`, but may include other allocation classes. 155 | 156 | 2. The thread-local entry for class `C` has a non-full magazine. 157 | 158 | 3. The newly released allocation is pushed to that magazine. 159 | 160 | We must handle the case when there is no entry for class `C` in the 161 | thread-local cache, because allocation and deallocation can happen 162 | in different threads. 163 | 164 | In order to get a non-full magazine, the `ClassInfo` (all the cache 165 | entries for a given class refer to the same `ClassInfo`) pops from its 166 | freelist of partial-filled magazines, and otherwise asks its `Rack` 167 | (multiple `ClassInfo`s share the same `Rack`) for a new empty magazine. 168 | 169 | The thread cache entry's current full magazine enters the `ClassInfo`'s 170 | freelist, and is replaced by the new non-full magazine. At this point, 171 | there must be room to push the newly released deallocation to the 172 | magazine in the thread cache entry. 173 | 174 | Exceptional situations 175 | ---------------------- 176 | 177 | Until now, we have never populated the freelist of partially populated 178 | magazines: allocation only releases empty ones back to the `Rack`, and 179 | deallocation pushes full magazines on the `ClassInfo`'s freelist. 180 | 181 | We need to handle partially populated magazines to clean up when 182 | threads are shut down. 183 | 184 | When a thread is shutdown, we must evict everything from its 185 | thread-local cache (otherwise we'll leak magazines and allocations). 186 | Full magazines still go to the relevant `ClassInfo`, and empty ones to 187 | their `Rack`. However, there may also be magazines that are neither 188 | full nor empty; these enter the `ClassInfo`'s freelist of partial 189 | magazines. 190 | 191 | Once a thread has begun the shutdown process, we also don't want to 192 | repopulate its thread cache. We instead satisfy allocations by 193 | hitting the `Press` directly (we should first pop from any non-empty 194 | magazines), and deallocations by grabbing a non-full magazine, pushing 195 | to it, and immediately giving the resulting magazing back to the 196 | `ClassInfo`. 197 | 198 | The allocation slow path, from the inside out 199 | --------------------------------------------- 200 | 201 | Each `Mill` carves out `Span`s from one `Chunk` at a time. A `Chunk` 202 | is a data region of 1 GB aligned to 1 GB, with a 2 MB region of 203 | metadata that begins 4 MB before the data region. A `Mill` obtains 204 | such a chunk of data and associated metadata by asking a `Mapper` to 205 | reserve address space, using the same `Mapper` to cut off any 206 | over-reservation at the edges, and finally letting the `Mapper` ask 207 | the OS to back the data and metadata regions with memory. 208 | 209 | Within a `Chunk`'s data region, `Span`s are aligned to 16 KB, and each 210 | such 16 KB range is associated 1:1 with an entry in the metadata array. 211 | The metadata for a Span-aligned range must thus not exceed 32 bytes 212 | (a constraint that is checked at compile-time), so that the metadata 213 | array can fit in the 2 MB region. 214 | 215 | This layout avoids interleaving Slitter metadata in-band with the 216 | mutator's allocations. The `Mill` also leaves guard pages not only 217 | between the metadata and the data region, but also before the metadata 218 | region and after the data region. This makes it unlikely that buffer 219 | overflows will affect a metadata region, or scribble past a Slitter 220 | chunk. 221 | 222 | The simple mapping between Span-aligned ranges and `SpanMetadata`s in 223 | the metadata array means we can efficiently check that a deallocation 224 | request is valid. While we currently only confirm that the class id 225 | provided by the caller matches the class in the allocation's 226 | `SpanMetadata`, we plan to add more checks: 227 | 228 | 1. The deallocated address must be a multiple of the allocation size 229 | 2. The metadata region must actually be managed by Slitter 230 | 231 | Chunks never go away: once allocated, they are immortal. That's why 232 | it's important to avoid fragmenting the address space with chunks. 233 | 234 | When a `Press` needs a new bump region, its `Mill` will return a 235 | single `Span` that usually contains multiple Span-aligned ranges in 236 | the data region, and is thus associated with multiple `SpanMetadata` 237 | objects. The `Press` will use one of these metadata objects to track 238 | its bump allocation, but must initialise all of them to signal that 239 | the corresponding ranges belong to the `Press`'s allocation class. 240 | 241 | Once a Span-aligned range is associated with an allocation class, it 242 | stays that way: the address range is never released back to the OS, 243 | nor can it be recycled for a different class. 244 | 245 | Eventually, a `ClassInfo` will ask its press for a newly allocated 246 | object. That allocation will either be immediately returned to the 247 | mutator, or cached in a magazine. Either way, an allocation never 248 | returns to the `Press`: it will always be owned by the mutator, or by 249 | a magazine. 250 | 251 | A `ClassInfo` also manages magazines. Each `ClassInfo` manages its 252 | own freelist of non-empty magazines (they contain allocations 253 | for the `ClassInfo`'s class, so are specific to that `ClassInfo`). 254 | 255 | However, `ClassInfo`s defer to their `Rack` for empty magazines. A 256 | `Rack` allocates and releases empty magazines. When we switch to 257 | lock-free `MagazineStack`s, `Magazine` too will become immortal: while 258 | they will be allocated arbitrarily (from the system allocator at 259 | first), they will never be released, and will instead enter the 260 | `Rack`'s freelist. 261 | 262 | This monotonicity will make it trivial to implement lock-free stacks 263 | without safe memory protection. 264 | -------------------------------------------------------------------------------- /doc/fast_path.md: -------------------------------------------------------------------------------- 1 | The allocation and release fast paths 2 | ===================================== 3 | 4 | Slitter attempts to use as much Rust as possible; while this does 5 | imply a certain loss of control, the structure of a thread-caching 6 | slab allocator means that we can easily pinpoint what little code 7 | is really hot, and rewrite that in C or assembly language. 8 | 9 | Allocations 10 | ----------- 11 | 12 | The core of object allocation is `slitter_allocate`. 13 | 14 | ``` 15 | void * 16 | slitter_allocate(struct slitter_class class) 17 | { 18 | struct magazine *restrict mag; 19 | size_t next_index; 20 | uint32_t id = class.id; 21 | 22 | if (__builtin_expect(id >= slitter_cache.n, 0)) 23 | return slitter__allocate_slow(class); 24 | 25 | mag = &slitter_cache.mags[id].alloc; 26 | if (__builtin_usubl_overflow(mag->top_of_stack, 2, &next_index)) { 27 | next_index++; 28 | } 29 | 30 | if (__builtin_expect(slitter__magazine_is_exhausted(mag), 0)) 31 | return slitter__allocate_slow(class); 32 | 33 | /* 34 | * The magazine was not empty, so next_index did not overflow 35 | * by more than 1. 36 | */ 37 | __builtin_prefetch(mag->storage->allocations[next_index], 1); 38 | return slitter__magazine_get_non_empty(mag); 39 | } 40 | ``` 41 | 42 | Which assembles to something like the following on x86-64: 43 | 44 | ``` 45 | 0: 48 8b 15 00 00 00 00 mov 0x0(%rip),%rdx # 7 46 | 3: R_X86_64_GOTTPOFF slitter_cache-0x4 47 | 7: 89 f8 mov %edi,%eax 48 | 9: 64 48 3b 02 cmp %fs:(%rdx),%rax 49 | d: 73 39 jae 48 ; Check if the cache must be (re-)initialised 50 | f: 48 c1 e0 05 shl $0x5,%rax 51 | 13: 64 48 03 42 08 add %fs:0x8(%rdx),%rax 52 | 18: 48 8b 10 mov (%rax),%rdx 53 | 1b: 48 83 fa 02 cmp $0x2,%rdx 54 | 1f: 48 89 d6 mov %rdx,%rsi 55 | 22: 48 83 d6 fe adc $0xfffffffffffffffe,%rsi ; Generate the prefetch index 56 | 26: 48 85 d2 test %rdx,%rdx 57 | 29: 74 1d je 48 ; Is the magazine empty? 58 | 2b: 48 8b 48 08 mov 0x8(%rax),%rcx ; Load the magazine's array 59 | 2f: 48 83 ea 01 sub $0x1,%rdx 60 | 33: 48 8b 34 f1 mov (%rcx,%rsi,8),%rsi 61 | 37: 0f 18 0e prefetcht0 (%rsi) ; Prefetch the next allocation 62 | 3a: 48 89 10 mov %rdx,(%rax) ; Update the allocation index 63 | 3d: 48 8b 04 d1 mov (%rcx,%rdx,8),%rax ; Grab our allocation 64 | 41: c3 retq 65 | 42: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1) 66 | 48: e9 00 00 00 00 jmpq 4d 67 | 49: R_X86_64_PLT32 slitter__allocate_slow-0x4 68 | ``` 69 | 70 | The first branch is taken ~once per thread, and the second once per 71 | 30-allocation magazine. Most of the remaining instructions are used 72 | to prefetch the next allocation; the prefetch isn't part of any 73 | dependency chain, and code that allocates memory typically doesn't 74 | saturate execution units, so that's not a problem. 75 | 76 | When we must refill the magazine, the slow path isn't that slow 77 | either. At a high level, we first check if there's a full magazine in 78 | the thread cache's deallocation buffer; if there isn't, we try to pop 79 | some non-empty magazines off the allocation class's linked stack. When 80 | both stacks are empty, that's just a quick load and comparison with 0. 81 | Otherwise, we call `slitter__stack_pop` or `slitter__stack_try_pop`, 82 | straightforward double-wide CAS jobs (we avoid ABA with a generation 83 | counter, and don't have to worry about reclamation races because 84 | magazines are immortal). The plain `pop` looks like: 85 | 86 | ``` 87 | 0: 4c 8b 4f 08 mov 0x8(%rdi),%r9 88 | 4: 4c 8b 07 mov (%rdi),%r8 89 | 7: 4d 85 c0 test %r8,%r8 90 | a: 74 34 je 40 91 | c: 53 push %rbx 92 | d: 4c 89 c0 mov %r8,%rax 93 | 10: 4c 89 ca mov %r9,%rdx 94 | 13: 49 8d 49 01 lea 0x1(%r9),%rcx 95 | 17: 49 8b 98 f0 00 00 00 mov 0xf0(%r8),%rbx 96 | 1e: f0 48 0f c7 0f lock cmpxchg16b (%rdi) 97 | 23: 49 39 d1 cmp %rdx,%r9 98 | 26: 75 20 jne 48 99 | 28: 49 c7 80 f0 00 00 00 movq $0x0,0xf0(%r8) 100 | 2f: 00 00 00 00 101 | 33: b8 01 00 00 00 mov $0x1,%eax 102 | 38: 5b pop %rbx 103 | 39: 4c 89 06 mov %r8,(%rsi) 104 | 3c: c3 retq 105 | 3d: 0f 1f 00 nopl (%rax) 106 | 40: 31 c0 xor %eax,%eax 107 | 42: c3 retq 108 | 43: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 109 | 48: 49 89 c0 mov %rax,%r8 110 | 4b: 49 89 d1 mov %rdx,%r9 111 | 4e: 48 85 c0 test %rax,%rax 112 | 51: 75 c0 jne 13 113 | 53: 31 c0 xor %eax,%eax 114 | 55: 5b pop %rbx 115 | 56: c3 retq 116 | ``` 117 | 118 | If we found a non-empty magazine, we must push our currenty empty one 119 | to a freelist for recycling. That's another compare-and-swap. 120 | 121 | If the local buffer and both class-global stacks are empty, we must 122 | allocate more objects. We implement that in `press.rs` with a single 123 | atomic increment, which does not fail under contention, but only when 124 | we notice (after the fact) that we must find a new bump allocation 125 | region. 126 | 127 | At a high level, we expect the slow path to incur one atomic increment 128 | during bulk allocation phases, when no allocation is cached in 129 | magazines. When mixing allocation and deallocation (steady state), 130 | this turns into two atomics, one CAS to acquire a new magazine of 131 | cahed allocations, and another to recycle our empty magazine. 132 | 133 | However, we will enter an even slower path whenever we exhaust the 134 | current bump allocation reigon. When that happens (roughly once per 135 | megabyte), we take a lock and grab another piece of address space. 136 | Finally, we can also run out of pre-reserved address space, in which 137 | case we must `mmap` to ask the kernel for more address space; we only 138 | do that in 1 GB increments (and never release memory to the OS), so 139 | that's a rare occasion. 140 | 141 | Release 142 | ------- 143 | 144 | Releasing allocations is special because it's a fundamentally 145 | asynchronous operation: `slitter_release`, like `free` doesn't return 146 | anything, so nothing can wait on it. That's why we try to pack more 147 | safety checks in the release code, as long as we can avoid 148 | (unpredictable) control flow. 149 | 150 | The code for `slitter_release` is 151 | 152 | ``` 153 | void 154 | slitter_release(struct slitter_class class, void *ptr) 155 | { 156 | uintptr_t address = (uintptr_t)ptr; 157 | uintptr_t chunk_base = address & -SLITTER__DATA_ALIGNMENT; 158 | uintptr_t chunk_offset = address % SLITTER__DATA_ALIGNMENT; 159 | size_t span_index = chunk_offset / SLITTER__SPAN_ALIGNMENT; 160 | uintptr_t meta_base = chunk_base - 161 | (SLITTER__GUARD_PAGE_SIZE + SLITTER__METADATA_PAGE_SIZE); 162 | struct magazine *restrict mag; 163 | uint32_t id = class.id; 164 | 165 | if (ptr == NULL) 166 | return; 167 | 168 | /* Check the span metadata. */ 169 | { 170 | const struct span_metadata *meta = (void *)meta_base; 171 | const struct span_metadata *span = &meta[span_index]; 172 | 173 | assert(class.id == span->class_id && "class mismatch"); 174 | } 175 | 176 | if (__builtin_expect(id >= slitter_cache.n, 0)) 177 | return slitter__release_slow(class, ptr); 178 | 179 | mag = &slitter_cache.mags[id].release; 180 | if (__builtin_expect(slitter__magazine_is_exhausted(mag), 0)) 181 | return slitter__release_slow(class, ptr); 182 | 183 | return slitter__magazine_put_non_full(mag, ptr); 184 | } 185 | ``` 186 | 187 | All the shifting and masking above are there to help detect 188 | mismatching releases. The real work starts at 189 | `if (__builtin_expect(id >= slitter_cache.n, 0))`. 190 | 191 | Again, the majority of instructions aren't the deallocation itself: 192 | that's just two range checks followed by a store and a stack index 193 | update. 194 | 195 | ``` 196 | 0: 48 89 f2 mov %rsi,%rdx 197 | 3: 48 89 f0 mov %rsi,%rax 198 | 6: 48 81 e2 00 00 00 c0 and $0xffffffffc0000000,%rdx 199 | d: 48 c1 e8 0e shr $0xe,%rax 200 | 11: 48 81 ea 00 00 40 00 sub $0x400000,%rdx 201 | 18: 48 85 f6 test %rsi,%rsi 202 | 1b: 74 41 je 5e ; check for NULL 203 | 1d: 0f b7 c0 movzwl %ax,%eax 204 | 20: 48 8d 04 40 lea (%rax,%rax,2),%rax 205 | 24: 39 3c c2 cmp %edi,(%rdx,%rax,8) 206 | 27: 75 3c jne 65 ; Assert out on class mismatch 207 | 29: 48 8b 15 00 00 00 00 mov 0x0(%rip),%rdx # 30 208 | 2c: R_X86_64_GOTTPOFF slitter_cache-0x4 209 | 30: 89 f8 mov %edi,%eax 210 | 32: 64 48 3b 02 cmp %fs:(%rdx),%rax 211 | 36: 73 28 jae 60 ; Maybe (re-)initialise the cache 212 | 38: 48 c1 e0 05 shl $0x5,%rax 213 | 3c: 64 48 03 42 08 add %fs:0x8(%rdx),%rax 214 | 41: 48 8b 50 10 mov 0x10(%rax),%rdx 215 | 45: 48 85 d2 test %rdx,%rdx ; Is the target magazine full? 216 | 48: 74 16 je 60 217 | 4a: 48 8b 48 18 mov 0x18(%rax),%rcx 218 | 4e: 48 8d 7a 01 lea 0x1(%rdx),%rdi 219 | 52: 48 89 78 10 mov %rdi,0x10(%rax) ; Update the release index 220 | 56: 48 89 b4 d1 f0 00 00 mov %rsi,0xf0(%rcx,%rdx,8) ; Store the freed object 221 | 5d: 00 222 | 5e: c3 retq 223 | 5f: 90 nop 224 | 60: e9 00 00 00 00 jmpq 65 225 | 61: R_X86_64_PLT32 slitter__release_slow-0x4 226 | 65: 50 push %rax 227 | 66: 48 8d 0d 00 00 00 00 lea 0x0(%rip),%rcx # 6d 228 | 69: R_X86_64_PC32 .rodata.__PRETTY_FUNCTION__.2337-0x4 229 | 6d: ba 4e 00 00 00 mov $0x4e,%edx 230 | 72: 48 8d 35 00 00 00 00 lea 0x0(%rip),%rsi # 79 231 | 75: R_X86_64_PC32 .LC0-0x4 232 | 79: 48 8d 3d 00 00 00 00 lea 0x0(%rip),%rdi # 80 233 | 7c: R_X86_64_PC32 .LC1-0x4 234 | 80: e8 00 00 00 00 callq 85 235 | 81: R_X86_64_PLT32 __assert_fail-0x4 236 | ``` 237 | 238 | Here as well, the first slow path branch is taken ~once per thread, 239 | and the second once per 30-allocation magazine. 240 | 241 | When a magazine is full, we must push it to the allocation class's 242 | stack (`slitter__stack_push`). That's another double-wide CAS loop. 243 | 244 | After that, we must find an empty one. We hope to find one from the 245 | global cache of empty magazines (an atomic stack pop); otherwise, we 246 | create a new one... by calling the system allocator for now. 247 | 248 | In total, that's two atomics, or one atomic and a malloc... a bit 249 | worse than allocation, but the application for which we wrote slitter 250 | cares more about the performance of parallel allocation during startup 251 | than anything else. 252 | -------------------------------------------------------------------------------- /examples/demo.c: -------------------------------------------------------------------------------- 1 | #define RUN_ME /* 2 | set -e 3 | 4 | CURRENT="$PWD" 5 | SELF=$(readlink -f "$0") 6 | EXEC=$(basename "$SELF" .c) 7 | BASE="$(dirname "$SELF")/../" 8 | 9 | (cd "$BASE"; cargo build --release --target-dir "$CURRENT/target") 10 | 11 | echo "Build with CFLAGS=-DMISMATCH for a crash" 12 | 13 | cc $CFLAGS -I"$BASE/include" "$SELF" "$CURRENT/target/release/libslitter.a" -lpthread -ldl -o "$EXEC" 14 | 15 | exec "./$EXEC" 16 | */ 17 | #include 18 | #include 19 | #include 20 | 21 | struct base { 22 | int x; 23 | }; 24 | 25 | struct derived { 26 | struct base base; 27 | int y; 28 | }; 29 | 30 | DEFINE_SLITTER_CLASS(base_tag, 31 | .name = "base", 32 | .size = sizeof(struct base), 33 | .zero_init = true); 34 | 35 | DEFINE_SLITTER_CLASS(derived_tag, 36 | .name = "derived", 37 | .size = sizeof(struct derived), 38 | .zero_init = true); 39 | 40 | int 41 | main() 42 | { 43 | struct base *base; 44 | struct derived *derived; 45 | 46 | /* Release is NULL-safe. */ 47 | slitter_release(base_tag, NULL); 48 | 49 | /* Allocate from our two class tags. */ 50 | base = slitter_allocate(base_tag); 51 | derived = slitter_allocate(derived_tag); 52 | 53 | /* We asked for zero-initialisation. */ 54 | assert(base->x == 0); 55 | assert(derived->base.x == 0 && derived->y == 0); 56 | 57 | base->x = 1; 58 | derived->y = 2; 59 | 60 | /* Release the two objects. */ 61 | slitter_release(base_tag, base); 62 | slitter_release(derived_tag, derived); 63 | 64 | /* Allocate again, they're still zero-filled. */ 65 | base = slitter_allocate(base_tag); 66 | derived = slitter_allocate(derived_tag); 67 | assert(base->x == 0); 68 | assert(derived->base.x == 0 && derived->y == 0); 69 | 70 | slitter_release(base_tag, base); 71 | slitter_release(derived_tag, derived); 72 | 73 | #ifdef MISMATCH 74 | /* Allocate from the "derived" tag. */ 75 | derived = slitter_allocate(derived_tag); 76 | 77 | /* 78 | * Free its "base" member. This will crash with 79 | * something like 80 | * `demo: c/cache.c:94: slitter_release: Assertion `class.id == span->class_id && "class mismatch"' failed.` 81 | */ 82 | slitter_release(base_tag, &derived->base); 83 | #endif 84 | 85 | printf("exiting demo.c\n"); 86 | return 0; 87 | } 88 | -------------------------------------------------------------------------------- /include/slitter.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | /** 8 | * Each slitter object class is uniquely identified by a non-zero 9 | * 32-bit integer. 10 | * 11 | * Passing a zero-initialised `slitter_class` to `slitter_allocate` 12 | * or `slitter_release` causes undefined behaviour. 13 | */ 14 | struct slitter_class { 15 | uint32_t id; 16 | }; 17 | 18 | struct slitter_class_config { 19 | /* 20 | * The name of the object class. Nullable. 21 | * 22 | * Must point to a NUL-terminated string of utf-8 bytes if non-NULL. 23 | */ 24 | const char *name; 25 | 26 | /* 27 | * The size of each object in the allocation class. Allocations 28 | * are only guaranteed alignment to 8 bytes. 29 | */ 30 | size_t size; 31 | 32 | /* 33 | * If true, zero-fill recycled allocations. 34 | */ 35 | bool zero_init; 36 | 37 | /* 38 | * The name of the underlying mapper, or NULL for default. 39 | * 40 | * A mapper name of "file" will use the file-backed mapper. 41 | */ 42 | const char *mapper_name; 43 | }; 44 | 45 | #define DEFINE_SLITTER_CLASS(NAME, ...) \ 46 | struct slitter_class NAME; \ 47 | \ 48 | __attribute__((__constructor__)) \ 49 | static void slitter_register_##NAME##_fn(void) \ 50 | { \ 51 | \ 52 | NAME = slitter_class_register( \ 53 | &(struct slitter_class_config) { __VA_ARGS__ }); \ 54 | return; \ 55 | } 56 | 57 | 58 | /** 59 | * Registers a new allocation class, or dies trying. 60 | * 61 | * The config must be a valid pointer. On error, this function will abort. 62 | */ 63 | struct slitter_class slitter_class_register(const struct slitter_class_config *); 64 | 65 | /** 66 | * Updates the parent directory for the file-backed slabs' backing 67 | * files. NULL resets to the default. 68 | * 69 | * The default directory is `TMPFILE`, or `/tmp/`, and will be used if 70 | * this function is not called before the first file-backed slab 71 | * allocation. 72 | * 73 | * It is safe to call this function at any time. However, the update 74 | * will only take effect when Slitter maps a new 1 GB chunk of data. 75 | */ 76 | void slitter_set_file_backed_slab_directory(const char *directory); 77 | 78 | /** 79 | * Returns a new allocation for the object class. 80 | * 81 | * On error, this function will abort. 82 | * 83 | * Behaviour is undefined if the `slitter_class` argument is 84 | * zero-filled or was otherwise not returned by 85 | * `slitter_class_register`. 86 | */ 87 | void *slitter_allocate(struct slitter_class); 88 | 89 | /** 90 | * Passes ownership of `ptr` back to the object class. 91 | * 92 | * `ptr` must be NULL, or have been returned by a call to 93 | * `slitter_alloc`. 94 | * 95 | * On error, this function will abort. 96 | * 97 | * Behaviour is undefined if the `slitter_class` argument is 98 | * zero-filled or was otherwise not returned by 99 | * `slitter_class_register`. 100 | */ 101 | void slitter_release(struct slitter_class, void *ptr); 102 | -------------------------------------------------------------------------------- /src/cache.rs: -------------------------------------------------------------------------------- 1 | //! Slitter stashes allocations for each size class in a thread-local 2 | //! cache. 3 | #[cfg(any( 4 | all(test, feature = "check_contracts_in_tests"), 5 | feature = "check_contracts" 6 | ))] 7 | use contracts::*; 8 | #[cfg(not(any( 9 | all(test, feature = "check_contracts_in_tests"), 10 | feature = "check_contracts" 11 | )))] 12 | use disabled_contracts::*; 13 | 14 | use std::cell::RefCell; 15 | use std::num::NonZeroU32; 16 | 17 | #[cfg(any( 18 | all(test, feature = "check_contracts_in_tests"), 19 | feature = "check_contracts" 20 | ))] 21 | use crate::debug_allocation_map; 22 | #[cfg(any( 23 | all(test, feature = "check_contracts_in_tests"), 24 | feature = "check_contracts" 25 | ))] 26 | use crate::debug_type_map; 27 | 28 | use crate::class::ClassInfo; 29 | use crate::linear_ref::LinearRef; 30 | use crate::magazine::LocalMagazineCache; 31 | use crate::magazine::PopMagazine; 32 | use crate::magazine::PushMagazine; 33 | use crate::press; 34 | use crate::Class; 35 | 36 | /// Start with a fast path array of this many `Magazines` 37 | /// structs, including one for the dummy 0 class. 38 | #[cfg(not(feature = "c_fast_path"))] 39 | const INITIAL_CACHE_SIZE: usize = 4; 40 | 41 | #[derive(Default)] 42 | #[repr(C)] 43 | struct Magazines { 44 | /// The cache allocates from this magazine. 45 | alloc: PopMagazine, 46 | /// The cache releases into this magazine. 47 | release: PushMagazine, 48 | } 49 | 50 | struct Info { 51 | /// The class info for the corresponding class id. 52 | info: Option<&'static ClassInfo>, 53 | cache: LocalMagazineCache, 54 | } 55 | 56 | /// For each allocation class, we cache up to one magazine's worth of 57 | /// allocations, and another magazine's worth of newly deallocated 58 | /// objects. 59 | /// 60 | /// Classically, we would use one magazine as a stack for both 61 | /// allocations and deallocations, and a backup to avoid bouncing 62 | /// nearly-full or nearly-empty magazines back to the `ClassInfo`. 63 | /// We instead use two magazines to let us do smarter things on 64 | /// deallocation (e.g., more easily check for double free or take 65 | /// advantage of pre-zeroed out allocations). 66 | /// 67 | /// The cache consists of parallel vectors that are directly indexed 68 | /// with the class id; the first element at index 0 is thus never 69 | /// used, so we must increment SMALL_CACHE_SIZE by 1. 70 | /// 71 | /// The source of truth on the number of allocation classes in the 72 | /// Cache is the `per_class_info` vector; `per_class` may have extra 73 | /// element but is never shorter than the `per_class_info`. 74 | struct Cache { 75 | /// This array of magazines may be longer than necessary: 76 | /// zero-initialised magazines will correctly trigger a 77 | /// slow path. 78 | per_class: &'static mut [Magazines], 79 | /// This parallel vector holds a reference to ClassInfo; it is 80 | /// only `None` for the dummy entry we keep around for the invalid 81 | /// "0" class id. 82 | per_class_info: Vec, 83 | /// If true, the `Cache` owns allocation backing the `per_class` 84 | /// slice. Otherwise, we should let it leak. 85 | /// 86 | /// Once true, this never becomes false. 87 | per_class_is_owned: bool, 88 | } 89 | 90 | extern "C" { 91 | fn slitter__cache_register(region: *mut Magazines, count: usize); 92 | } 93 | 94 | // TODO: keyed thread-local is slow. We should `#![feature(thread_local)]` 95 | // and `#[thread_local] static mut CACHE: ...` in nightly builds. We'll 96 | // still want a `thread_local!` to trigger cleanup... 97 | thread_local!(static CACHE: RefCell = RefCell::new(Cache::new())); 98 | 99 | /// Attempts to return an allocation for an object of this `class`. 100 | #[ensures(ret.is_some() -> 101 | debug_allocation_map::can_be_allocated(class, ret.as_ref().unwrap().get()).is_ok(), 102 | "Successful allocations must be in the correct class and not double allocate")] 103 | #[ensures(ret.is_some() -> 104 | debug_type_map::is_class(class, ret.as_ref().unwrap()).is_ok(), 105 | "Successful allocations must match the class of the address range.")] 106 | #[ensures(ret.is_some() -> 107 | press::check_allocation(class, ret.as_ref().unwrap().get().as_ptr() as usize).is_ok(), 108 | "Sucessful allocations must have the allocation metadata set correctly.")] 109 | #[inline(always)] 110 | pub fn allocate(class: Class) -> Option { 111 | let result = if cfg!(feature = "c_fast_path") { 112 | extern "C" { 113 | fn slitter_allocate(class: Class) -> Option; 114 | } 115 | 116 | Ok(unsafe { slitter_allocate(class) }) 117 | } else { 118 | CACHE.try_with(|cache| cache.borrow_mut().allocate(class)) 119 | }; 120 | 121 | result.unwrap_or_else(|_| class.info().allocate_slow()) 122 | } 123 | 124 | /// C-accessible slow path for the allocation. The slow-path code is 125 | /// identical to regular non-C allocation, so it's always safe to call 126 | /// this function. 127 | #[ensures(ret.is_some() -> 128 | debug_allocation_map::can_be_allocated(class, ret.as_ref().unwrap().get()).is_ok(), 129 | "Successful allocations must be in the correct class and not double allocate")] 130 | #[ensures(ret.is_some() -> 131 | debug_type_map::is_class(class, ret.as_ref().unwrap()).is_ok(), 132 | "Successful allocations must match the class of the address range.")] 133 | #[ensures(ret.is_some() -> 134 | press::check_allocation(class, ret.as_ref().unwrap().get().as_ptr() as usize).is_ok(), 135 | "Sucessful allocations must have the allocation metadata set correctly.")] 136 | #[no_mangle] 137 | pub extern "C" fn slitter__allocate_slow(class: Class) -> Option { 138 | let ret = CACHE 139 | .try_with(|cache| cache.borrow_mut().allocate_slow(class)) 140 | .unwrap_or_else(|_| class.info().allocate_slow()); 141 | assert!(ret.is_some(), "Allocation failed"); 142 | 143 | ret 144 | } 145 | 146 | /// Returns an allocation back to this `class`. 147 | #[requires(debug_allocation_map::has_been_released(class, block.get()).is_ok(), 148 | "Blocks passed to `release` must have already been marked as released.")] 149 | #[requires(debug_type_map::is_class(class, &block).is_ok(), 150 | "Deallocated blocks must match the class of the address range.")] 151 | #[requires(press::check_allocation(class, block.get().as_ptr() as usize).is_ok(), 152 | "Deallocated block must have the allocation metadata set correctly.")] 153 | #[inline(always)] 154 | pub fn release(class: Class, block: LinearRef) { 155 | let mut cell = Some(block); 156 | 157 | let result = if cfg!(feature = "c_fast_path") { 158 | extern "C" { 159 | fn slitter_release(class: Class, block: LinearRef); 160 | } 161 | 162 | Ok(unsafe { slitter_release(class, cell.take().unwrap()) }) 163 | } else { 164 | CACHE.try_with(|cache| cache.borrow_mut().release(class, cell.take().unwrap())) 165 | }; 166 | 167 | result.unwrap_or_else(|_| { 168 | if let Some(alloc) = cell { 169 | class.info().release_slow(alloc) 170 | } 171 | }); 172 | } 173 | 174 | /// C-accessible slow path for the allocation. The slow-path code is 175 | /// identical to regular non-C release call, so it's always safe to 176 | /// call this function. 177 | #[requires(debug_allocation_map::has_been_released(class, block.get()).is_ok(), 178 | "Blocks passed to `release` must have already been marked as released.")] 179 | #[requires(debug_type_map::is_class(class, &block).is_ok(), 180 | "Deallocated blocks must match the class of the address range.")] 181 | #[requires(press::check_allocation(class, block.get().as_ptr() as usize).is_ok(), 182 | "Deallocated block must have the allocation metadata set correctly.")] 183 | #[no_mangle] 184 | pub extern "C" fn slitter__release_slow(class: Class, block: LinearRef) { 185 | let mut cell = Some(block); 186 | 187 | CACHE 188 | .try_with(|cache| cache.borrow_mut().release_slow(class, cell.take().unwrap())) 189 | .unwrap_or_else(|_| { 190 | if let Some(alloc) = cell { 191 | class.info().release_slow(alloc) 192 | } 193 | }) 194 | } 195 | 196 | impl Drop for Cache { 197 | #[requires(self.check_rep_or_err().is_ok(), "Internal invariants hold.")] 198 | fn drop(&mut self) { 199 | unsafe { 200 | slitter__cache_register(std::ptr::null_mut(), 0); 201 | } 202 | 203 | while let Some(slot) = self.per_class_info.pop() { 204 | use LocalMagazineCache::*; 205 | 206 | let index = self.per_class_info.len(); 207 | let mut mags = Default::default(); 208 | 209 | std::mem::swap( 210 | &mut mags, 211 | self.per_class 212 | .get_mut(index) 213 | .expect("per_class should be at least as long as per_class_info"), 214 | ); 215 | 216 | if let Some(info) = slot.info { 217 | info.release_magazine(mags.alloc, None); 218 | info.release_magazine(mags.release, None); 219 | match slot.cache { 220 | Nothing => (), 221 | Empty(mag) => info.release_magazine(mag, None), 222 | Full(mag) => info.release_magazine(mag, None), 223 | } 224 | } else { 225 | // This must be the padding slot at index 0. 226 | assert!(self.per_class_info.is_empty()); 227 | 228 | let default_rack = crate::rack::get_default_rack(); 229 | default_rack.release_empty_magazine(mags.alloc); 230 | default_rack.release_empty_magazine(mags.release); 231 | 232 | match slot.cache { 233 | Nothing => (), 234 | Empty(_) => panic!("Found used cache in dummy slot"), 235 | Full(_) => panic!("Found used cache in dummy slot"), 236 | } 237 | } 238 | } 239 | 240 | if self.per_class_is_owned { 241 | // Replace the magazine slice with a dummy, and drop it. 242 | // An empty slice is correct: `per_class_info.len() == 0`. 243 | let mut dummy: &mut [Magazines] = &mut []; 244 | 245 | std::mem::swap(&mut dummy, &mut self.per_class); 246 | unsafe { Box::from_raw(dummy) }; 247 | } 248 | 249 | // If we don't own the `per_class` slice, it's safe to leave 250 | // it here: it has been fully reset to default `Magazines` 251 | // that do not own anything. 252 | } 253 | } 254 | 255 | impl Cache { 256 | fn new() -> Cache { 257 | #[cfg(feature = "c_fast_path")] 258 | let (mags, is_owned) = { 259 | extern "C" { 260 | fn slitter__cache_borrow(OUT_n: &mut usize) -> *mut Magazines; 261 | } 262 | 263 | let mut slice_size = 0usize; 264 | let mags = unsafe { slitter__cache_borrow(&mut slice_size) }; 265 | let slice = unsafe { std::slice::from_raw_parts_mut(mags, slice_size) }; 266 | (slice, false) 267 | }; 268 | 269 | #[cfg(not(feature = "c_fast_path"))] 270 | let (mags, is_owned) = { 271 | let mags: [Magazines; INITIAL_CACHE_SIZE] = Default::default(); 272 | 273 | (Box::leak(Box::new(mags)), true) 274 | }; 275 | 276 | Cache { 277 | per_class: mags, 278 | per_class_info: Vec::new(), 279 | per_class_is_owned: is_owned, 280 | } 281 | } 282 | 283 | /// Returns `Err` when some of the Cache's invariants are violated. 284 | #[cfg(any( 285 | all(test, feature = "check_contracts_in_tests"), 286 | feature = "check_contracts" 287 | ))] 288 | fn check_rep_or_err(&self) -> Result<(), &'static str> { 289 | if self.per_class.len() < self.per_class_info.len() { 290 | return Err("Vector of magazines is shorter than vector of info."); 291 | } 292 | 293 | for mags in self.per_class.iter().skip(self.per_class_info.len()) { 294 | if !mags.alloc.is_empty() { 295 | return Err("Padding cache entry has a non-empty allocation magazine."); 296 | } 297 | 298 | if !mags.release.is_full() { 299 | return Err("Padding cache entry has a non-full release magazine."); 300 | } 301 | } 302 | 303 | if !self 304 | .per_class_info 305 | .iter() 306 | .enumerate() 307 | .all(|(i, x)| i == 0 || x.info.unwrap().id.id().get() as usize == i) 308 | { 309 | return Err("Some cache entries are missing their info."); 310 | } 311 | 312 | if let Some(_) = self.per_class_info.get(0) { 313 | if !self.per_class[0].alloc.is_empty() { 314 | return Err("Dummy cache entry has a non-empty allocation magazine."); 315 | } 316 | 317 | if !self.per_class[0].release.is_full() { 318 | return Err("Dummy cache entry has a non-full release magazine."); 319 | } 320 | } 321 | 322 | // All magazines must be in a good state, and only contain 323 | // *available* allocations for the correct class. 324 | for (mags, info) in self.per_class.iter().zip(&self.per_class_info) { 325 | mags.alloc.check_rep(info.info.map(|info| info.id))?; 326 | mags.release.check_rep(info.info.map(|info| info.id))?; 327 | } 328 | 329 | Ok(()) 330 | } 331 | 332 | /// Ensure the cache's `per_class` array has at least `min_length` 333 | /// elements. 334 | #[invariant(self.check_rep_or_err().is_ok(), "Internal invariants hold.")] 335 | #[requires(min_length < usize::MAX / 2)] 336 | #[ensures(self.per_class.len() >= min_length)] 337 | #[ensures(self.per_class.len() >= old(self.per_class.len()))] 338 | #[ensures(old(self.per_class_is_owned) -> self.per_class_is_owned, 339 | "per_class_is_owned flag is monotonic (false -> true)")] 340 | fn ensure_per_class_length(&mut self, min_length: usize) { 341 | if self.per_class.len() >= min_length { 342 | return; 343 | } 344 | 345 | let new_length = min_length 346 | .checked_next_power_of_two() 347 | .expect("&CacheInfo are too big for len > usize::MAX / 2"); 348 | let mut vec = Vec::with_capacity(new_length); 349 | vec.resize_with(new_length, Default::default); 350 | 351 | let mut new_slice = Box::leak(vec.into_boxed_slice()); 352 | self.per_class 353 | .swap_with_slice(&mut new_slice[0..self.per_class.len()]); 354 | 355 | let mut is_owned = true; 356 | std::mem::swap(&mut new_slice, &mut self.per_class); 357 | // Swap the `is_owned` flag after the slice: the flag only 358 | // goes from false to true, never the other way, and it's 359 | // safer to leak on unwind/panic than to free something 360 | // we don't own. 361 | std::mem::swap(&mut is_owned, &mut self.per_class_is_owned); 362 | 363 | if is_owned { 364 | unsafe { Box::from_raw(new_slice) }; 365 | } 366 | } 367 | 368 | /// Ensures the cache's `per_class_info` array has one entry for every 369 | /// allocation class currently defined. 370 | #[invariant(self.check_rep_or_err().is_ok(), "Internal invariants hold.")] 371 | #[ensures(self.per_class_info.len() > old(crate::class::max_id()), 372 | "There exists an entry for the max class id when the function was called.")] 373 | #[cold] 374 | fn grow(&mut self) { 375 | let max_id = crate::class::max_id(); 376 | if self.per_class_info.len() > max_id { 377 | return; 378 | } 379 | 380 | assert!(max_id <= u32::MAX as usize); 381 | self.ensure_per_class_length(max_id + 1); 382 | 383 | while self.per_class_info.len() <= max_id { 384 | let id = NonZeroU32::new(self.per_class_info.len() as u32); 385 | let info = id.and_then(|id| Class::from_id(id).map(|class| class.info())); 386 | 387 | self.per_class_info.push(Info { 388 | info, 389 | cache: LocalMagazineCache::Nothing, 390 | }); 391 | } 392 | 393 | unsafe { 394 | // We want to pass `per_class.len()`, despite it being 395 | // longer than `per_class_info`: the extra elements will 396 | // correctly trigger a slow path, so this is safe, and we 397 | // want to concentrate all slow path conditionals to the 398 | // same branch, for predictability. We can't get rid of 399 | // the "magazine is exhausted" condition, so let's make 400 | // the "array is too short" branch as unlikely as possible. 401 | slitter__cache_register(self.per_class.as_mut_ptr(), self.per_class.len()); 402 | } 403 | } 404 | 405 | #[invariant(self.check_rep_or_err().is_ok(), "Internal invariants hold.")] 406 | #[ensures(ret.is_some() -> 407 | debug_allocation_map::can_be_allocated(class, ret.as_ref().unwrap().get()).is_ok(), 408 | "Successful allocations must be from the correct class, and not double allocate.")] 409 | #[ensures(ret.is_some() -> debug_type_map::is_class(class, ret.as_ref().unwrap()).is_ok(), 410 | "Successful allocations must match the class of the address range.")] 411 | #[ensures(ret.is_some() -> 412 | press::check_allocation(class, ret.as_ref().unwrap().get().as_ptr() as usize).is_ok(), 413 | "Sucessful allocations must have the allocation metadata set correctly.")] 414 | #[inline(always)] 415 | fn allocate_slow(&mut self, class: Class) -> Option { 416 | let index = class.id().get() as usize; 417 | 418 | if self.per_class_info.len() <= index { 419 | self.grow(); 420 | } 421 | 422 | // per_class.len() >= per_class_info.len() 423 | let mag = &mut self.per_class[index].alloc; 424 | if let Some(alloc) = mag.get() { 425 | return Some(alloc); 426 | } 427 | 428 | let info = &mut self.per_class_info[index]; 429 | info.info 430 | .expect("must have class info") 431 | .refill_magazine(mag, &mut info.cache) 432 | } 433 | 434 | /// Attempts to return an allocation for `class`. Consumes from 435 | /// the cache if possible, and hits the Class(Info)'s slow path 436 | /// otherwise. 437 | #[invariant(self.check_rep_or_err().is_ok(), "Internal invariants hold.")] 438 | #[ensures(ret.is_some() -> 439 | debug_allocation_map::can_be_allocated(class, ret.as_ref().unwrap().get()).is_ok(), 440 | "Successful allocations must be from the correct class, and not double allocate.")] 441 | #[ensures(ret.is_some() -> debug_type_map::is_class(class, ret.as_ref().unwrap()).is_ok(), 442 | "Successful allocations must match the class of the address range.")] 443 | #[ensures(ret.is_some() -> 444 | press::check_allocation(class, ret.as_ref().unwrap().get().as_ptr() as usize).is_ok(), 445 | "Sucessful allocations must have the allocation metadata set correctly.")] 446 | #[inline(always)] 447 | fn allocate(&mut self, class: Class) -> Option { 448 | if cfg!(feature = "c_fast_path") { 449 | extern "C" { 450 | fn slitter_allocate(class: Class) -> Option; 451 | } 452 | 453 | unsafe { slitter_allocate(class) } 454 | } else { 455 | self.allocate_slow(class) 456 | } 457 | } 458 | 459 | #[invariant(self.check_rep_or_err().is_ok(), "Internal invariants hold.")] 460 | #[requires(debug_allocation_map::has_been_released(class, block.get()).is_ok(), 461 | "A released block for `class` must have been marked as such.")] 462 | #[requires(debug_type_map::is_class(class, &block).is_ok(), 463 | "Deallocated blocks must match the class of the address range.")] 464 | #[requires(press::check_allocation(class, block.get().as_ptr() as usize).is_ok(), 465 | "Deallocated block must have the allocation metadata set correctly.")] 466 | #[inline(always)] 467 | fn release_slow(&mut self, class: Class, block: LinearRef) { 468 | press::check_allocation(class, block.get().as_ptr() as usize) 469 | .expect("deallocated address should match allocation class"); 470 | 471 | let index = class.id().get() as usize; 472 | 473 | if self.per_class_info.len() <= index { 474 | assert!(index < u32::MAX as usize); 475 | self.grow(); 476 | } 477 | 478 | // per_class.len() >= per_class_info.len() 479 | let mag = &mut self.per_class[index].release; 480 | // We prefer to cache freshly deallocated objects, for 481 | // temporal locality. 482 | if let Some(spill) = mag.put(block) { 483 | let info = &mut self.per_class_info[index]; 484 | info.info 485 | .expect("must have class info") 486 | .clear_magazine(mag, &mut info.cache, spill); 487 | } 488 | } 489 | 490 | /// Marks `block`, an allocation for `class`, ready for reuse. 491 | /// Pushes to the cache if possible, and hits the Class(Info)'s 492 | /// slow path otherwise. 493 | #[invariant(self.check_rep_or_err().is_ok(), "Internal invariants hold.")] 494 | #[requires(debug_allocation_map::has_been_released(class, block.get()).is_ok(), 495 | "A released block for `class` must have been marked as such.")] 496 | #[requires(debug_type_map::is_class(class, &block).is_ok(), 497 | "Deallocated blocks must match the class of the address range.")] 498 | #[requires(press::check_allocation(class, block.get().as_ptr() as usize).is_ok(), 499 | "Deallocated block must have the allocation metadata set correctly.")] 500 | #[inline(always)] 501 | fn release(&mut self, class: Class, block: LinearRef) { 502 | if cfg!(feature = "c_fast_path") { 503 | extern "C" { 504 | fn slitter_release(class: Class, block: LinearRef); 505 | } 506 | 507 | unsafe { slitter_release(class, block) } 508 | } else { 509 | self.release_slow(class, block) 510 | } 511 | } 512 | } 513 | -------------------------------------------------------------------------------- /src/class.rs: -------------------------------------------------------------------------------- 1 | //! A Slitter allocation class represent a set of type-stable objects 2 | //! that all have the same size (Slitter never overwrites an 3 | //! allocation with internal metadata, even once freed). Allocation 4 | //! and deallocation calls must have matching `Class` structs, even if 5 | //! objects from different classes have the same size: the Slitter 6 | //! code may check this invariant to help detect bugs, and callers may 7 | //! rely on type stability. 8 | #[cfg(any( 9 | all(test, feature = "check_contracts_in_tests"), 10 | feature = "check_contracts" 11 | ))] 12 | use contracts::*; 13 | #[cfg(not(any( 14 | all(test, feature = "check_contracts_in_tests"), 15 | feature = "check_contracts" 16 | )))] 17 | use disabled_contracts::*; 18 | 19 | use std::alloc::Layout; 20 | use std::ffi::CStr; 21 | use std::num::NonZeroU32; 22 | use std::os::raw::c_char; 23 | 24 | use crate::magazine_stack::MagazineStack; 25 | use crate::press::Press; 26 | 27 | /// External callers interact with slitter allocation classes via this 28 | /// opaque Class struct. 29 | #[repr(C)] 30 | #[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] 31 | pub struct Class { 32 | id: NonZeroU32, 33 | } 34 | 35 | /// When created, a class is configured with an object size, and an 36 | /// optional name. 37 | pub struct ClassConfig { 38 | pub name: Option, 39 | pub layout: Layout, 40 | pub zero_init: bool, 41 | pub mapper_name: Option, 42 | } 43 | 44 | /// The extern "C" interface uses this version of `ClassConfig`. 45 | #[repr(C)] 46 | pub struct ForeignClassConfig { 47 | name: *const c_char, 48 | size: usize, 49 | zero_init: bool, 50 | mapper_name: *const c_char, 51 | } 52 | 53 | /// Slitter stores internal information about configured classes with 54 | /// this Info struct. 55 | pub(crate) struct ClassInfo { 56 | #[allow(dead_code)] 57 | pub name: Option, 58 | pub layout: Layout, 59 | 60 | // The Class will allocate and release magazines via this Rack. 61 | pub rack: &'static crate::rack::Rack, 62 | 63 | // Fully populated magazines go in in `full_mags`. 64 | pub full_mags: MagazineStack, 65 | 66 | // Partially populated, but non-empty, magazines go in `partial_mags`. 67 | // Empty magazines go back to the `Rack`. 68 | pub partial_mags: MagazineStack, 69 | 70 | // Use this `Press` to allocate new objects. 71 | pub press: Press, 72 | 73 | #[allow(dead_code)] 74 | pub id: Class, 75 | 76 | // Whether allocations should be zero-filled. 77 | pub zero_init: bool, 78 | } 79 | 80 | impl ClassConfig { 81 | /// Attempts to convert a `ForeignClassConfig` pointer to a native 82 | /// `ClassConfig`. 83 | /// 84 | /// # Safety 85 | /// 86 | /// This function assumes `config_ptr` is NULL or valid. 87 | pub unsafe fn from_c(config_ptr: *const ForeignClassConfig) -> Option { 88 | // Attempts to convert a C string to an optional String. 89 | fn to_nullable_str(ptr: *const c_char) -> Result, std::str::Utf8Error> { 90 | if ptr.is_null() { 91 | Ok(None) 92 | } else { 93 | Ok(Some(unsafe { CStr::from_ptr(ptr) }.to_str()?.to_owned())) 94 | } 95 | } 96 | 97 | if config_ptr.is_null() { 98 | return None; 99 | } 100 | 101 | let config: &ForeignClassConfig = &*config_ptr; 102 | let layout = Layout::from_size_align(config.size.max(1), /*align=*/ 8).ok()?; 103 | Some(ClassConfig { 104 | name: to_nullable_str(config.name).ok()?, 105 | layout, 106 | zero_init: config.zero_init, 107 | mapper_name: to_nullable_str(config.mapper_name).ok()?, 108 | }) 109 | } 110 | } 111 | 112 | lazy_static::lazy_static! { 113 | // TODO(lock): this lock is never taken on a fast path. 114 | static ref CLASSES: std::sync::Mutex> = Default::default(); 115 | } 116 | 117 | pub fn max_id() -> usize { 118 | let guard = CLASSES.lock().unwrap(); 119 | guard.len() 120 | } 121 | 122 | impl Class { 123 | /// Attempts to create a new allocation class for `config`. 124 | /// 125 | /// On success, there is a corresponding `ClassInfo` struct at 126 | /// `ret - 1` in the global `CLASSES` array. 127 | #[ensures(ret.is_ok() -> 128 | CLASSES.lock().unwrap().get(ret.unwrap().id.get() as usize - 1).map(|info| info.id) == Some(ret.unwrap()), 129 | "On success, the class is at id - 1 in the global array of ClassInfo")] 130 | #[ensures(ret.is_ok() -> 131 | Class::from_id(ret.unwrap().id) == Some(ret.unwrap()), 132 | "On success, we can instantiate `Class` from the NonZeroU32 id.")] 133 | #[ensures(ret.is_ok() -> 134 | std::ptr::eq(ret.unwrap().info(), CLASSES.lock().unwrap()[ret.unwrap().id.get() as usize - 1]), 135 | "On success, the class's info matches the entry in the array.")] 136 | pub fn new(config: ClassConfig) -> Result { 137 | let mut classes = CLASSES.lock().unwrap(); 138 | 139 | let next_id = classes.len() + 1; 140 | if next_id > u32::MAX as usize { 141 | return Err("too many slitter allocation classes"); 142 | } 143 | 144 | let id = Class { 145 | id: NonZeroU32::new(next_id as u32).expect("next_id is positive"), 146 | }; 147 | 148 | let layout = config.layout.pad_to_align(); 149 | 150 | let info = Box::leak(Box::new(ClassInfo { 151 | name: config.name, 152 | layout, 153 | rack: crate::rack::get_default_rack(), 154 | full_mags: MagazineStack::new(), 155 | partial_mags: MagazineStack::new(), 156 | press: Press::new(id, layout, config.mapper_name.as_deref())?, 157 | id, 158 | zero_init: config.zero_init, 159 | })); 160 | classes.push(info); 161 | Ok(id) 162 | } 163 | 164 | /// Returns a `Class` struct for `id` if such a class exists. 165 | /// 166 | /// On success, this operation can be inverted by calling `id()`. 167 | #[ensures(ret.is_none() -> CLASSES.lock().unwrap().iter().all(|info| info.id.id != id), 168 | "`from_id` only fails if there is no registered `ClassInfo` with that id.")] 169 | #[ensures(ret.is_some() -> CLASSES.lock().unwrap()[id.get() as usize - 1].id == ret.unwrap(), 170 | "On success, the class's info is at id - 1 in the global array of info.")] 171 | #[ensures(ret.is_some() -> ret.unwrap().id == id, 172 | "On success, the return value's id matches the argument.")] 173 | pub(crate) fn from_id(id: NonZeroU32) -> Option { 174 | let guard = CLASSES.lock().unwrap(); 175 | if id.get() as usize <= guard.len() { 176 | Some(Class { id }) 177 | } else { 178 | None 179 | } 180 | } 181 | 182 | /// Returns the `Class`'s underlying `NonZeroU32` id. 183 | /// 184 | /// This operation is the inverse of `Class::from_id`. 185 | #[ensures(Class::from_id(ret) == Some(self), 186 | "We can recover the same `Class` with `Class::from_id`.")] 187 | #[inline] 188 | pub(crate) fn id(self) -> NonZeroU32 { 189 | self.id 190 | } 191 | 192 | /// Returns the global `ClassInfo` for this `Class`. 193 | #[ensures(ret.id == self)] 194 | pub(crate) fn info(self) -> &'static ClassInfo { 195 | let guard = CLASSES.lock().unwrap(); 196 | 197 | (*guard) 198 | .get(self.id.get() as usize - 1) 199 | .expect("Class structs are only build for valid ids") 200 | } 201 | } 202 | 203 | #[cfg(test)] 204 | mod test { 205 | use proptest::bool; 206 | use proptest::collection::vec; 207 | use proptest::prelude::*; 208 | use proptest::sample; 209 | use std::alloc::Layout; 210 | use std::collections::VecDeque; 211 | use std::ffi::c_void; 212 | use std::ptr::NonNull; 213 | 214 | use crate::Class; 215 | use crate::ClassConfig; 216 | 217 | #[test] 218 | fn smoke_test() { 219 | let class = Class::new(ClassConfig { 220 | name: Some("alloc_smoke".into()), 221 | layout: Layout::from_size_align(8, 8).expect("layout should build"), 222 | zero_init: true, 223 | mapper_name: None, 224 | }) 225 | .expect("Class should build"); 226 | 227 | let p0 = class.allocate().expect("Should allocate"); 228 | let p1 = class.allocate().expect("Should allocate"); 229 | 230 | class.release(p0); 231 | 232 | let p2 = class.allocate().expect("Should allocate"); 233 | 234 | class.release(p2); 235 | class.release(p1); 236 | } 237 | 238 | // Keep allocating / deallocating from the same class. This 239 | // should help us trigger magazine refilling logic. 240 | #[test] 241 | fn back_to_back() { 242 | let class = Class::new(ClassConfig { 243 | name: Some("alloc_push_pop".into()), 244 | layout: Layout::from_size_align(8, 8).expect("layout should build"), 245 | zero_init: true, 246 | mapper_name: None, 247 | }) 248 | .expect("Class should build"); 249 | 250 | let size = 8; 251 | 252 | for _ in 0..100 { 253 | let allocated = class.allocate().expect("Should allocate"); 254 | 255 | let ptr = allocated.as_ptr() as *mut u8; 256 | // Fresh allocations should always be zero-filled. 257 | assert_eq!(unsafe { std::ptr::read(ptr) }, 0); 258 | assert_eq!(unsafe { std::ptr::read(ptr.add(size - 1)) }, 0); 259 | 260 | // Let's now write to our allocation before releasing. 261 | unsafe { std::ptr::write(ptr, 42u8) }; 262 | unsafe { std::ptr::write(ptr.add(size - 1), 42u8) }; 263 | 264 | class.release(allocated); 265 | } 266 | } 267 | 268 | // Allocate and deallocate from the same class, in batches. 269 | #[test] 270 | fn n_back_to_back() { 271 | let class = Class::new(ClassConfig { 272 | name: Some("alloc_push_pop".into()), 273 | layout: Layout::from_size_align(8, 8).expect("layout should build"), 274 | zero_init: true, 275 | mapper_name: None, 276 | }) 277 | .expect("Class should build"); 278 | 279 | let size = 8; 280 | 281 | for count in 1..128 { 282 | let mut allocations = Vec::new(); 283 | 284 | for _ in 0..count { 285 | let allocated = class.allocate().expect("Should allocate"); 286 | 287 | let ptr = allocated.as_ptr() as *mut u8; 288 | // Fresh allocations should always be zero-filled. 289 | assert_eq!(unsafe { std::ptr::read(ptr) }, 0); 290 | assert_eq!(unsafe { std::ptr::read(ptr.add(size - 1)) }, 0); 291 | 292 | // Let's now write to our allocation before releasing. 293 | unsafe { std::ptr::write(ptr, 42u8) }; 294 | unsafe { std::ptr::write(ptr.add(size - 1), 42u8) }; 295 | 296 | allocations.push(allocated); 297 | } 298 | 299 | for allocation in allocations { 300 | class.release(allocation); 301 | } 302 | } 303 | } 304 | 305 | // Returns true iff that `new` isn't in `current`. 306 | fn check_new_allocation(current: &[NonNull], new: NonNull) -> bool { 307 | current.iter().all(|x| x.as_ptr() != new.as_ptr()) 308 | } 309 | 310 | proptest! { 311 | // Bulk allocate, then deallocate and re-allocate in random-ish order. 312 | #[test] 313 | fn random_order(indices in vec(0..20usize, 1..50)) { 314 | use std::collections::HashSet; 315 | 316 | let class = Class::new(ClassConfig { 317 | name: Some("random".into()), 318 | layout: Layout::from_size_align(8, 8).expect("layout should build"), 319 | zero_init: false, 320 | mapper_name: None, 321 | }) 322 | .expect("Class should build"); 323 | 324 | // If a slot is None, we will allocate in there the next 325 | // time we hit it. If it holds a `NonNull`, we will 326 | // instead consume and free its contents. 327 | // 328 | // Let the vec leak because we do not want to drop its 329 | // contents on panic. 330 | let slots: &mut Vec>> = Box::leak(Box::new(Vec::new())); 331 | 332 | // Initialise with 20 allocations. 333 | slots.resize_with(20, || class.allocate()); 334 | 335 | // Make sure all the allocations are unique. 336 | prop_assert!(slots.len() == 337 | slots 338 | .iter() 339 | .map(|x| x.expect("alloc should succeed").as_ptr()) 340 | .collect::>() 341 | .len()); 342 | for index in indices.iter().cloned() { 343 | if let Some(alloc) = slots[index].take() { 344 | class.release(alloc); 345 | } else { 346 | let new_alloc = class.allocate(); 347 | 348 | prop_assert!(new_alloc.is_some()); 349 | 350 | let fresh = slots.iter().all(|x| { 351 | match x { 352 | Some(p) => p.as_ptr() != new_alloc.unwrap().as_ptr(), 353 | None => true, 354 | } 355 | }); 356 | prop_assert!(fresh); 357 | slots[index] = new_alloc; 358 | } 359 | } 360 | 361 | for slot in slots.iter_mut() { 362 | if let Some(freed) = slot.take() { 363 | class.release(freed); 364 | } 365 | } 366 | 367 | // Reacquire the vector to avoid leaking on success. 368 | unsafe { Box::from_raw(slots as *mut _); } 369 | } 370 | 371 | // Allocate and deallocate in random-ish order from two classes. 372 | #[test] 373 | fn random_order_two_classes(indices in vec((0..10usize, 0..2usize), 1..50)) { 374 | let classes = vec![ 375 | Class::new(ClassConfig { 376 | name: Some("random_class_1".into()), 377 | layout: Layout::from_size_align(8, 8).expect("layout should build"), 378 | zero_init: true, 379 | mapper_name: None, 380 | }).expect("Class should build"), 381 | Class::new(ClassConfig { 382 | name: Some("random_class_2".into()), 383 | layout: Layout::from_size_align(16, 8).expect("layout should build"), 384 | zero_init: false, 385 | mapper_name: None, 386 | }).expect("Class should build"), 387 | ]; 388 | 389 | // If a slot is None, we will allocate in there the next 390 | // time we hit it. If it holds a `NonNull`, we will 391 | // instead consume and free its contents. 392 | let slots: &mut Vec, Class)>> = Box::leak(Box::new(Vec::new())); 393 | 394 | slots.resize(20, None); 395 | for (index, class_id) in indices.iter().cloned() { 396 | if let Some((alloc, class)) = slots[index].take() { 397 | unsafe { std::ptr::write_bytes(alloc.as_ptr() as *mut u8, 42, 1); } 398 | class.release(alloc); 399 | } else { 400 | let class = classes[class_id]; 401 | let new_alloc = class.allocate(); 402 | 403 | prop_assert!(new_alloc.is_some()); 404 | if class_id == 0 { 405 | prop_assert_eq!(unsafe { std::ptr::read(new_alloc.as_ref().unwrap().as_ptr() as *const u8) }, 0); 406 | } 407 | 408 | let fresh = slots.iter().all(|x| { 409 | match x { 410 | Some((p, _)) => p.as_ptr() != new_alloc.unwrap().as_ptr(), 411 | None => true, 412 | } 413 | }); 414 | prop_assert!(fresh); 415 | 416 | slots[index] = Some((new_alloc.unwrap(), class)); 417 | } 418 | } 419 | 420 | for slot in slots.iter_mut() { 421 | if let Some((freed, class)) = slot.take() { 422 | class.release(freed); 423 | } 424 | } 425 | 426 | unsafe { Box::from_raw(slots as *mut _); } 427 | } 428 | 429 | // Check that we can correctly allocate and deallocate in stack order. 430 | #[test] 431 | fn lifo(push_pop in vec(bool::ANY, 2..50)) { 432 | let class = Class::new(ClassConfig { 433 | name: Some("lifo".into()), 434 | layout: Layout::from_size_align(8, 8).expect("layout should build"), 435 | zero_init: true, 436 | mapper_name: None, 437 | }) 438 | .expect("Class should build"); 439 | 440 | let stack: &mut Vec> = Box::leak(Box::new(Vec::new())); 441 | 442 | for alloc in push_pop.iter().cloned() { 443 | if alloc { 444 | let new_alloc = class.allocate(); 445 | 446 | prop_assert_ne!(new_alloc, None); 447 | let block = new_alloc.unwrap(); 448 | 449 | prop_assert!(check_new_allocation(&stack, block)); 450 | prop_assert_eq!(unsafe { std::ptr::read(block.as_ptr() as *const u8) }, 0); 451 | 452 | stack.push(block); 453 | } else if let Some(freed) = stack.pop() { 454 | unsafe { std::ptr::write_bytes(freed.as_ptr() as *mut u8, 42, 1); } 455 | class.release(freed); 456 | } 457 | } 458 | 459 | while let Some(freed) = stack.pop() { 460 | class.release(freed); 461 | } 462 | 463 | unsafe { Box::from_raw(stack as *mut _); } 464 | } 465 | 466 | // Check that we can correctly allocate and deallocate in queue order. 467 | #[test] 468 | fn fifo(push_pop in vec(bool::ANY, 2..50)) { 469 | let class = Class::new(ClassConfig { 470 | name: Some("lifo".into()), 471 | layout: Layout::from_size_align(8, 8).expect("layout should build"), 472 | zero_init: false, 473 | mapper_name: None, 474 | }) 475 | .expect("Class should build"); 476 | 477 | let queue: &mut VecDeque> = Box::leak(Box::new(VecDeque::new())); 478 | 479 | for alloc in push_pop.iter().cloned() { 480 | if alloc { 481 | let new_alloc = class.allocate(); 482 | 483 | prop_assert_ne!(new_alloc, None); 484 | let block = new_alloc.unwrap(); 485 | 486 | prop_assert!(check_new_allocation(queue.make_contiguous(), block)); 487 | queue.push_back(block); 488 | } else if let Some(freed) = queue.pop_front() { 489 | class.release(freed); 490 | } 491 | } 492 | 493 | while let Some(freed) = queue.pop_back() { 494 | class.release(freed); 495 | } 496 | 497 | unsafe { Box::from_raw(queue as *mut _); } 498 | } 499 | 500 | // Check that we can correctly allocate and deallocate in FIFO or LIFO order. 501 | // 502 | // 0 means allocate, -1 frees from the front, and 1 freeds from back. 503 | #[test] 504 | fn biendian(actions in vec(sample::select(vec![-1, 0, 1]), 2..50)) { 505 | let class = Class::new(ClassConfig { 506 | name: Some("lifo".into()), 507 | layout: Layout::from_size_align(8, 8).expect("layout should build"), 508 | zero_init: true, 509 | mapper_name: None, 510 | }) 511 | .expect("Class should build"); 512 | 513 | let queue: &mut VecDeque> = Box::leak(Box::new(VecDeque::new())); 514 | 515 | for action in actions.iter().cloned() { 516 | if action == 0 { 517 | let new_alloc = class.allocate(); 518 | 519 | prop_assert_ne!(new_alloc, None); 520 | let block = new_alloc.unwrap(); 521 | 522 | prop_assert!(check_new_allocation(queue.make_contiguous(), block)); 523 | prop_assert_eq!(unsafe { std::ptr::read(block.as_ptr() as *const u8) }, 0); 524 | queue.push_back(block); 525 | } else if action == -1 { 526 | if let Some(freed) = queue.pop_front() { 527 | unsafe { std::ptr::write_bytes(freed.as_ptr() as *mut u8, 42, 1); } 528 | class.release(freed); 529 | } 530 | } else if let Some(freed) = queue.pop_back() { 531 | unsafe { std::ptr::write_bytes(freed.as_ptr() as *mut u8, 42, 1); } 532 | class.release(freed); 533 | } 534 | } 535 | 536 | while let Some(freed) = queue.pop_back() { 537 | class.release(freed); 538 | } 539 | 540 | unsafe { Box::from_raw(queue as *mut _); } 541 | } 542 | } 543 | } 544 | -------------------------------------------------------------------------------- /src/debug_allocation_map.rs: -------------------------------------------------------------------------------- 1 | //! This module tracks the internal status of allocated objects in 2 | //! debug builds. 3 | use std::collections::HashMap; 4 | use std::ffi::c_void; 5 | use std::ptr::NonNull; 6 | use std::sync::Mutex; 7 | 8 | use crate::Class; 9 | 10 | struct AllocationInfo { 11 | class: Class, 12 | live: bool, // True if owned by the mutator 13 | } 14 | 15 | lazy_static::lazy_static! { 16 | static ref ALLOCATION_STATE_MAP: Mutex> = Default::default(); 17 | } 18 | 19 | /// Confirms that it makes sense to return this allocation to the mutator. 20 | pub fn can_be_allocated(class: Class, alloc: &NonNull) -> Result<(), &'static str> { 21 | let alignment = class.info().layout.align(); 22 | 23 | if (alloc.as_ptr() as usize % alignment) != 0 { 24 | return Err("misaligned address"); 25 | } 26 | 27 | let map = ALLOCATION_STATE_MAP.lock().unwrap(); 28 | 29 | if let Some(info) = map.get(&(alloc.as_ptr() as usize)) { 30 | if info.class != class { 31 | return Err("class mismatch"); 32 | } 33 | 34 | if info.live { 35 | return Err("double allocation"); 36 | } 37 | } 38 | 39 | Ok(()) 40 | } 41 | 42 | /// Marks this allocation as returned to the mutator. 43 | pub fn mark_allocated(class: Class, alloc: &NonNull) -> Result<(), &'static str> { 44 | let info = class.info(); 45 | let alignment = info.layout.align(); 46 | 47 | if (alloc.as_ptr() as usize % alignment) != 0 { 48 | return Err("misaligned address"); 49 | } 50 | 51 | // Confirm that `alloc` is all zero if necessary. 52 | if info.zero_init { 53 | let ptr = alloc.as_ptr() as *const u8; 54 | 55 | for i in 0..info.layout.size() { 56 | if unsafe { std::ptr::read(ptr.add(i)) } != 0 { 57 | return Err("non zero-filled allocation"); 58 | } 59 | } 60 | } 61 | 62 | let mut map = ALLOCATION_STATE_MAP.lock().unwrap(); 63 | let mut info = map 64 | .entry(alloc.as_ptr() as usize) 65 | .or_insert(AllocationInfo { class, live: false }); 66 | 67 | if info.class != class { 68 | return Err("class mismatch"); 69 | } 70 | 71 | if info.live { 72 | return Err("double allocation"); 73 | } 74 | 75 | info.live = true; 76 | Ok(()) 77 | } 78 | 79 | /// Marks this allocation as released by the mutator. 80 | pub fn mark_released(class: Class, alloc: &NonNull) -> Result<(), &'static str> { 81 | let mut map = ALLOCATION_STATE_MAP.lock().unwrap(); 82 | let mut info = map 83 | .get_mut(&(alloc.as_ptr() as usize)) 84 | .ok_or("Released unknown address")?; 85 | 86 | if info.class != class { 87 | return Err("class mismatch"); 88 | } 89 | 90 | if !info.live { 91 | return Err("double free"); 92 | } 93 | 94 | info.live = false; 95 | Ok(()) 96 | } 97 | 98 | /// Confirms that the allocation has been released by the mutator 99 | pub fn has_been_released(class: Class, alloc: &NonNull) -> Result<(), &'static str> { 100 | let map = ALLOCATION_STATE_MAP.lock().unwrap(); 101 | let info = map 102 | .get(&(alloc.as_ptr() as usize)) 103 | .ok_or("Released unknown address")?; 104 | 105 | if info.class != class { 106 | return Err("class mismatch"); 107 | } 108 | 109 | if info.live { 110 | return Err("released a live allocation"); 111 | } 112 | 113 | Ok(()) 114 | } 115 | -------------------------------------------------------------------------------- /src/debug_arange_map.rs: -------------------------------------------------------------------------------- 1 | //! This module tracks metadata about mapped address ranges in debug 2 | //! builds. 3 | use std::collections::BTreeMap; 4 | use std::sync::Mutex; 5 | 6 | #[derive(Clone, Copy)] 7 | struct Range { 8 | begin: usize, 9 | size: usize, 10 | } 11 | 12 | #[derive(Clone, Copy)] 13 | struct AddressRange { 14 | /// The range of address space reserved. 15 | begin: usize, 16 | size: usize, 17 | 18 | /// If populated, the *one* metadata range completely inside this 19 | /// reserved range. 20 | metadata: Option, 21 | 22 | /// If populated, the one data range completely inside this 23 | /// reserved range. 24 | data: Option, 25 | } 26 | 27 | lazy_static::lazy_static! { 28 | static ref ADDRESS_RANGE_MAP: Mutex> = Default::default(); 29 | } 30 | 31 | /// Returns the address range associated with the highest key less 32 | /// than or equal to `ptr`. 33 | fn predecessor(ptr: usize) -> Option { 34 | let map = ADDRESS_RANGE_MAP.lock().unwrap(); 35 | map.range(0..=ptr).last().map(|x| *x.1) 36 | } 37 | 38 | /// Registers a new address range. It must not overlap with any other 39 | /// registered range. 40 | pub fn reserve_range(begin: usize, size: usize) -> Result<(), &'static str> { 41 | if usize::MAX - begin < size { 42 | return Err("Address is too high."); 43 | } 44 | 45 | let mut map = ADDRESS_RANGE_MAP.lock().unwrap(); 46 | 47 | // Make sure nothing overlaps with the new range. 48 | for (_, info) in map.range(0..=(begin + size)).rev() { 49 | // We're walking allocations from the top down. If the 50 | // current allocation is too high, keep looking. 51 | if info.begin >= begin + size { 52 | continue; 53 | } 54 | 55 | // If the current allocation is too low, stop. 56 | if begin >= info.begin + info.size { 57 | break; 58 | } 59 | 60 | //return Err("Found address range"); 61 | } 62 | 63 | map.insert( 64 | begin, 65 | AddressRange { 66 | begin, 67 | size, 68 | metadata: None, 69 | data: None, 70 | }, 71 | ); 72 | Ok(()) 73 | } 74 | 75 | pub fn releasable_range(begin: usize, size: usize) -> Result<(), &'static str> { 76 | if size > usize::MAX - begin { 77 | return Err("Range too large"); 78 | } 79 | 80 | let reserved = predecessor(begin).ok_or("Parent range not found")?; 81 | 82 | if begin >= reserved.begin + reserved.size { 83 | return Err("Parent range too short for begin"); 84 | } 85 | 86 | if begin + size > reserved.begin + reserved.size { 87 | return Err("Parent range too short for size"); 88 | } 89 | 90 | if reserved.begin == begin && reserved.size == size { 91 | return Ok(()); 92 | } 93 | 94 | if let Some(meta) = reserved.metadata { 95 | if !(begin >= meta.begin + meta.size || begin + size <= meta.begin) { 96 | return Err("Released range overlaps with metadata region"); 97 | } 98 | } 99 | 100 | if let Some(data) = reserved.data { 101 | if !(begin >= data.begin + data.size || begin + size <= data.begin) { 102 | return Err("Released range overlaps with data region"); 103 | } 104 | } 105 | 106 | if reserved.begin == begin { 107 | return Ok(()); 108 | } 109 | 110 | if reserved.begin + reserved.size == begin + size { 111 | return Ok(()); 112 | } 113 | 114 | Err("Released range is in the middle of the reservation") 115 | } 116 | 117 | /// Unregisters a fragment of a pre-existing address range. The 118 | /// fragment must be at either end of the registered range. 119 | /// 120 | /// Unless the range is fully released, its data or metadata must not 121 | /// overlap with the released range. 122 | pub fn release_range(begin: usize, size: usize) -> Result<(), &'static str> { 123 | if size > usize::MAX - begin { 124 | return Err("Range too large"); 125 | } 126 | 127 | let reserved = predecessor(begin).ok_or("Parent range not found")?; 128 | 129 | if begin >= reserved.begin + reserved.size { 130 | return Err("Parent range too short for begin"); 131 | } 132 | 133 | if begin + size > reserved.begin + reserved.size { 134 | return Err("Parent range too short for size"); 135 | } 136 | 137 | let mut map = ADDRESS_RANGE_MAP.lock().unwrap(); 138 | 139 | if reserved.begin == begin && reserved.size == size { 140 | map.remove(&begin); 141 | return Ok(()); 142 | } 143 | 144 | if let Some(meta) = reserved.metadata { 145 | if !(begin >= meta.begin + meta.size || begin + size <= meta.begin) { 146 | return Err("Released range overlaps with metadata region"); 147 | } 148 | } 149 | 150 | if let Some(data) = reserved.data { 151 | if !(begin >= data.begin + data.size || begin + size <= data.begin) { 152 | return Err("Released range overlaps with data region"); 153 | } 154 | } 155 | 156 | if reserved.begin == begin { 157 | map.remove(&begin); 158 | assert!(size < reserved.size); 159 | map.insert( 160 | begin + size, 161 | AddressRange { 162 | begin: begin + size, 163 | size: reserved.size - size, 164 | metadata: reserved.metadata, 165 | data: reserved.data, 166 | }, 167 | ); 168 | return Ok(()); 169 | } 170 | 171 | if reserved.begin + reserved.size == begin + size { 172 | let entry: &mut _ = map 173 | .get_mut(&reserved.begin) 174 | .ok_or("Parent range not found on second lookup")?; 175 | 176 | assert!(size < entry.size); 177 | entry.size -= size; 178 | return Ok(()); 179 | } 180 | 181 | Err("Released range is in the middle of the reservation") 182 | } 183 | 184 | pub fn can_mark_metadata(begin: usize, size: usize) -> Result { 185 | if size > usize::MAX - begin { 186 | return Err("Range too large"); 187 | } 188 | 189 | let reserved = predecessor(begin).ok_or("Parent range not found")?; 190 | if begin < reserved.begin { 191 | return Err("Metadata address too low"); 192 | } 193 | 194 | if begin + size > reserved.begin + reserved.size { 195 | return Err("Metadata address too high"); 196 | } 197 | 198 | if reserved.metadata.is_some() { 199 | return Err("Metadata registered twice"); 200 | } 201 | 202 | if let Some(data) = reserved.data { 203 | if begin + size <= data.begin { 204 | return Ok(reserved.begin); 205 | } 206 | 207 | if begin >= data.begin + data.size { 208 | return Ok(reserved.begin); 209 | } 210 | 211 | return Err("Metadata range overlaps with data"); 212 | } 213 | 214 | Ok(reserved.begin) 215 | } 216 | 217 | /// Marks a new metadata subrange in a previously reserved range. 218 | pub fn mark_metadata(begin: usize, size: usize) -> Result<(), &'static str> { 219 | let reservation_begin = can_mark_metadata(begin, size)?; 220 | 221 | let mut map = ADDRESS_RANGE_MAP.lock().unwrap(); 222 | let entry: &mut _ = map 223 | .get_mut(&reservation_begin) 224 | .ok_or("Parent range not found on second lookup")?; 225 | 226 | if entry.metadata.is_some() { 227 | return Err("Metadata registered twice"); 228 | } 229 | 230 | entry.metadata = Some(Range { begin, size }); 231 | Ok(()) 232 | } 233 | 234 | pub fn can_mark_data(begin: usize, size: usize) -> Result { 235 | if size > usize::MAX - begin { 236 | return Err("Range too large"); 237 | } 238 | 239 | let reserved = predecessor(begin).ok_or("Parent range not found")?; 240 | if begin < reserved.begin { 241 | return Err("Data address too low"); 242 | } 243 | 244 | if begin + size > reserved.begin + reserved.size { 245 | return Err("Data address too high"); 246 | } 247 | 248 | if reserved.data.is_some() { 249 | return Err("Data registered twice"); 250 | } 251 | 252 | if let Some(data) = reserved.data { 253 | if begin + size <= data.begin { 254 | return Ok(reserved.begin); 255 | } 256 | 257 | if begin >= data.begin + data.size { 258 | return Ok(reserved.begin); 259 | } 260 | 261 | return Err("Data range overlaps with metadata"); 262 | } 263 | 264 | Ok(reserved.begin) 265 | } 266 | 267 | /// Marks a new data subrange in a previously reserved range. 268 | pub fn mark_data(begin: usize, size: usize) -> Result<(), &'static str> { 269 | let reservation_begin = can_mark_data(begin, size)?; 270 | 271 | let mut map = ADDRESS_RANGE_MAP.lock().unwrap(); 272 | let entry: &mut _ = map 273 | .get_mut(&reservation_begin) 274 | .ok_or("Parent range not found on second lookup")?; 275 | 276 | if entry.data.is_some() { 277 | return Err("Data registered twice"); 278 | } 279 | 280 | entry.data = Some(Range { begin, size }); 281 | Ok(()) 282 | } 283 | 284 | /// Returns Ok if the range is fully in a metadata region. 285 | pub fn is_metadata(begin: usize, size: usize) -> Result<(), &'static str> { 286 | if size > usize::MAX - begin { 287 | return Err("Range too large"); 288 | } 289 | 290 | let reserved = predecessor(begin).ok_or("Parent range not found")?; 291 | let range = reserved.metadata.ok_or("Parent range has no metadata")?; 292 | 293 | if begin < range.begin { 294 | return Err("Address below metadata"); 295 | } 296 | 297 | if begin + size > range.begin + range.size { 298 | return Err("Address above metadata"); 299 | } 300 | 301 | Ok(()) 302 | } 303 | 304 | /// Returns Ok if the range is fully in a data region. 305 | pub fn is_data(begin: usize, size: usize) -> Result<(), &'static str> { 306 | if size > usize::MAX - begin { 307 | return Err("Range too large"); 308 | } 309 | 310 | let reserved = predecessor(begin).ok_or("Parent range not found")?; 311 | let range = reserved.data.ok_or("Parent range has no metadata")?; 312 | if begin < range.begin { 313 | return Err("Address below data"); 314 | } 315 | 316 | if begin + size > range.begin + range.size { 317 | return Err("Address above data"); 318 | } 319 | 320 | Ok(()) 321 | } 322 | -------------------------------------------------------------------------------- /src/debug_type_map.rs: -------------------------------------------------------------------------------- 1 | //! This module tracks the type of allocated addresses in debug builds. 2 | use std::collections::BTreeMap; 3 | use std::ffi::c_void; 4 | use std::ptr::NonNull; 5 | use std::sync::Mutex; 6 | 7 | use crate::linear_ref::LinearRef; 8 | use crate::Class; 9 | 10 | struct TypeInfo { 11 | begin: usize, 12 | size: usize, 13 | class: Class, 14 | } 15 | 16 | lazy_static::lazy_static! { 17 | static ref ALLOCATION_TYPE_MAP: Mutex> = Default::default(); 18 | } 19 | 20 | /// Ensures this allocation is fresh and associates it with `class`. 21 | pub fn associate_class(class: Class, alloc: usize) -> Result<(), &'static str> { 22 | let info = class.info(); 23 | let size = info.layout.size(); 24 | let alignment = info.layout.align(); 25 | 26 | if usize::MAX - alloc < size { 27 | return Err("Address is too high."); 28 | } 29 | 30 | if (alloc % alignment) != 0 { 31 | return Err("Address is misaligned."); 32 | } 33 | 34 | let mut map = ALLOCATION_TYPE_MAP.lock().unwrap(); 35 | 36 | // Make sure nothing overlaps with the allocation. 37 | for (_, info) in map.range(0..=(alloc + size)).rev() { 38 | // We're walking allocations from the top down. If the 39 | // current allocation is too high, keep looking. 40 | if info.begin >= alloc + size { 41 | continue; 42 | } 43 | 44 | // If the current allocation is too low, stop. 45 | if alloc >= info.begin + info.size { 46 | break; 47 | } 48 | 49 | return Err("Found overlapping allocation"); 50 | } 51 | 52 | map.insert( 53 | alloc, 54 | TypeInfo { 55 | begin: alloc, 56 | size, 57 | class, 58 | }, 59 | ); 60 | Ok(()) 61 | } 62 | 63 | /// Checks whether the `alloc`ation is of type `class`. 64 | pub fn ptr_is_class(class: Class, alloc: &NonNull) -> Result<(), &'static str> { 65 | let begin = alloc.as_ptr() as usize; 66 | let map = ALLOCATION_TYPE_MAP.lock().unwrap(); 67 | 68 | let entry = map.get(&begin).ok_or("Allocation not found")?; 69 | if entry.class != class { 70 | return Err("Allocation class mismatch"); 71 | } 72 | 73 | Ok(()) 74 | } 75 | 76 | /// Checks whether the `alloc`ation is of type `class`. 77 | pub fn is_class(class: Class, alloc: &LinearRef) -> Result<(), &'static str> { 78 | ptr_is_class(class, alloc.get()) 79 | } 80 | -------------------------------------------------------------------------------- /src/file_backed_mapper.rs: -------------------------------------------------------------------------------- 1 | //! The file-backed mapper ensures object are allocated in shared file 2 | //! mappings of private temporary files. This lets the operating 3 | //! system eagerly swap out cold data when under memory pressure. 4 | #[cfg(any( 5 | all(test, feature = "check_contracts_in_tests"), 6 | feature = "check_contracts" 7 | ))] 8 | use contracts::*; 9 | #[cfg(not(any( 10 | all(test, feature = "check_contracts_in_tests"), 11 | feature = "check_contracts" 12 | )))] 13 | use disabled_contracts::*; 14 | 15 | use std::ffi::c_void; 16 | use std::fs::File; 17 | use std::path::PathBuf; 18 | use std::ptr::NonNull; 19 | use std::sync::Mutex; 20 | 21 | use crate::Mapper; 22 | 23 | #[derive(Debug)] 24 | pub struct FileBackedMapper {} 25 | 26 | lazy_static::lazy_static! { 27 | static ref FILE_BACKED_PATH: Mutex> = Default::default(); 28 | } 29 | 30 | /// Updates the parent directory for the file-backed mapper's 31 | /// temporary files to `path`. 32 | /// 33 | /// A value of `None` restores the default (`TMPDIR`), and 34 | /// `Some(":memory")` forces regular file mappings. 35 | pub fn set_file_backed_slab_directory(path: Option) { 36 | let mut global_path = FILE_BACKED_PATH.lock().unwrap(); 37 | 38 | *global_path = path; 39 | } 40 | 41 | /// Returns a temporary File in `FILE_BACKED_PATH`, or in the 42 | /// global `TMPDIR`. If the file is None, the mapper should 43 | /// instead use a regular anonymous memory mapping. 44 | /// 45 | /// TODO: return a `std::io::Result>`. 46 | fn get_temp_file() -> Result, i32> { 47 | let path = FILE_BACKED_PATH.lock().unwrap(); 48 | 49 | match &*path { 50 | Some(dir) if dir.to_str() == Some(":memory:") => Ok(None), 51 | Some(dir) => tempfile::tempfile_in(dir).map(Some), 52 | None => tempfile::tempfile().map(Some), 53 | } 54 | .map_err(|e| e.raw_os_error().unwrap_or(0)) 55 | } 56 | 57 | #[contract_trait] 58 | impl Mapper for FileBackedMapper { 59 | fn page_size(&self) -> usize { 60 | crate::map::page_size() 61 | } 62 | 63 | fn reserve( 64 | &self, 65 | desired_size: usize, 66 | _data_size: usize, 67 | _prefix: usize, 68 | _suffix: usize, 69 | ) -> Result<(NonNull, usize), i32> { 70 | let region: NonNull = crate::map::reserve_region(desired_size)?; 71 | Ok((region, desired_size)) 72 | } 73 | 74 | fn release(&self, base: NonNull, size: usize) -> Result<(), i32> { 75 | crate::map::release_region(base, size) 76 | } 77 | 78 | fn allocate_meta(&self, base: NonNull, size: usize) -> Result<(), i32> { 79 | crate::map::allocate_region(base, size) 80 | } 81 | 82 | fn allocate_data(&self, base: NonNull, size: usize) -> Result<(), i32> { 83 | let tempfile = get_temp_file()?; 84 | 85 | match tempfile { 86 | Some(file) => crate::map::allocate_file_region(file, base, size), 87 | None => crate::map::allocate_region(base, size), 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/individual.rs: -------------------------------------------------------------------------------- 1 | //! This module services individual allocation and deallocation calls, 2 | //! i.e., the majority of public calls into Slitter. 3 | #[cfg(any( 4 | all(test, feature = "check_contracts_in_tests"), 5 | feature = "check_contracts" 6 | ))] 7 | use contracts::*; 8 | #[cfg(not(any( 9 | all(test, feature = "check_contracts_in_tests"), 10 | feature = "check_contracts" 11 | )))] 12 | use disabled_contracts::*; 13 | 14 | use std::ffi::c_void; 15 | use std::ptr::NonNull; 16 | 17 | #[cfg(any( 18 | all(test, feature = "check_contracts_in_tests"), 19 | feature = "check_contracts" 20 | ))] 21 | use crate::debug_allocation_map; 22 | #[cfg(any( 23 | all(test, feature = "check_contracts_in_tests"), 24 | feature = "check_contracts" 25 | ))] 26 | use crate::debug_type_map; 27 | #[cfg(any( 28 | all(test, feature = "check_contracts_in_tests"), 29 | feature = "check_contracts" 30 | ))] 31 | use crate::press; 32 | 33 | use crate::cache; 34 | use crate::class::Class; 35 | use crate::class::ClassInfo; 36 | use crate::linear_ref::LinearRef; 37 | use crate::magazine::LocalMagazineCache; 38 | 39 | impl Class { 40 | /// Attempts to return a newly allocated object for this `Class`. 41 | #[ensures(ret.is_some() -> 42 | debug_allocation_map::mark_allocated(self, ret.as_ref().unwrap()).is_ok(), 43 | "Successful allocations match the class and avoid double-allocation.")] 44 | #[ensures(ret.is_some() -> debug_type_map::ptr_is_class(self, ret.as_ref().unwrap()).is_ok(), 45 | "Successful allocations come from an address of the correct class.")] 46 | #[ensures(ret.is_some() -> press::check_allocation(self, ret.as_ref().unwrap().as_ptr() as usize).is_ok(), 47 | "Sucessful allocations must have the allocation metadata set correctly.")] 48 | #[inline(always)] 49 | pub fn allocate(self) -> Option> { 50 | cache::allocate(self).map(|x| x.convert_to_non_null()) 51 | } 52 | 53 | /// Marks an object returned by `allocate` as ready for reuse. 54 | #[requires(debug_allocation_map::mark_released(self, &block).is_ok(), 55 | "Released blocks must match the class and not double-free.")] 56 | #[requires(debug_type_map::ptr_is_class(self, &block).is_ok(), 57 | "Released blocks come from an address of the correct class.")] 58 | #[inline(always)] 59 | pub fn release(self, block: NonNull) { 60 | cache::release(self, LinearRef::new(block)); 61 | } 62 | } 63 | 64 | impl ClassInfo { 65 | /// The `cache` calls into this slow path when its thread-local 66 | /// storage is being deinitialised. 67 | #[ensures(ret.is_some() -> 68 | debug_allocation_map::can_be_allocated(self.id, ret.as_ref().unwrap().get()).is_ok(), 69 | "Successful allocations are fresh, or match the class and avoid double-allocation.")] 70 | #[ensures(ret.is_some() -> 71 | debug_type_map::is_class(self.id, ret.as_ref().unwrap()).is_ok(), 72 | "Successful allocations come from an address of the correct class.")] 73 | #[ensures(ret.is_some() -> 74 | press::check_allocation(self.id, ret.as_ref().unwrap().get().as_ptr() as usize).is_ok(), 75 | "Sucessful allocations must have the allocation metadata set correctly.")] 76 | #[inline(never)] 77 | pub(crate) fn allocate_slow(&self) -> Option { 78 | let mut empty_cache = LocalMagazineCache::Nothing; 79 | if let Some(mut mag) = self.get_cached_magazine(&mut empty_cache) { 80 | let allocated = mag.get(); 81 | assert!(allocated.is_some()); 82 | 83 | self.release_magazine(mag, None); 84 | allocated 85 | } else { 86 | // We can assume the press always allocates zero-filled 87 | // objects: we require that the underlying mapper only 88 | // give us zero-filled memory. 89 | self.press.allocate_one_object() 90 | } 91 | } 92 | 93 | /// The `cache` calls into this slow path when its thread-local 94 | /// storage is being deinitialised. 95 | #[requires(debug_allocation_map::has_been_released(self.id, block.get()).is_ok(), 96 | "Slow-released blocks went through `Class::release`.")] 97 | #[requires(debug_type_map::is_class(self.id, &block).is_ok(), 98 | "Released blocks come from an address of the correct class.")] 99 | #[requires(press::check_allocation(self.id, block.get().as_ptr() as usize).is_ok(), 100 | "Deallocated block must have the allocation metadata set correctly.")] 101 | #[inline(never)] 102 | pub(crate) fn release_slow(&self, block: LinearRef) { 103 | let mut empty_cache = LocalMagazineCache::Nothing; 104 | let mut mag = self.allocate_non_full_magazine(&mut empty_cache); 105 | 106 | // Deallocation must succeed. 107 | assert_eq!(mag.put(block), None); 108 | self.release_magazine(mag, None); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | mod cache; 2 | mod class; 3 | mod file_backed_mapper; 4 | mod individual; 5 | mod linear_ref; 6 | mod magazine; 7 | mod magazine_impl; 8 | mod magazine_stack; 9 | mod map; 10 | mod mapper; 11 | mod mill; 12 | mod press; 13 | mod rack; 14 | 15 | #[cfg(any( 16 | all(test, feature = "check_contracts_in_tests"), 17 | feature = "check_contracts" 18 | ))] 19 | mod debug_allocation_map; 20 | #[cfg(any( 21 | all(test, feature = "check_contracts_in_tests"), 22 | feature = "check_contracts" 23 | ))] 24 | mod debug_arange_map; 25 | #[cfg(any( 26 | all(test, feature = "check_contracts_in_tests"), 27 | feature = "check_contracts" 28 | ))] 29 | mod debug_type_map; 30 | 31 | use std::os::raw::c_char; 32 | 33 | pub use class::Class; 34 | pub use class::ClassConfig; 35 | pub use class::ForeignClassConfig; 36 | pub use file_backed_mapper::set_file_backed_slab_directory; 37 | pub use mapper::register_mapper; 38 | pub use mapper::Mapper; 39 | 40 | /// Registers a new allocation class globally 41 | /// 42 | /// # Safety 43 | /// 44 | /// This function assumes `config_ptr` is NULL or valid. 45 | #[no_mangle] 46 | pub unsafe extern "C" fn slitter_class_register(config_ptr: *const ForeignClassConfig) -> Class { 47 | let config = ClassConfig::from_c(config_ptr).expect("slitter_class_config must be valid"); 48 | 49 | Class::new(config).expect("slitter class allocation should succeed") 50 | } 51 | 52 | /// Updates the directory for the file-backed slab's temporary files. 53 | /// 54 | /// NULL reverts to the default, and ":memory:" forces regular 55 | /// anonymous mappings. 56 | /// 57 | /// # Safety 58 | /// 59 | /// This function assumes `path` is NULL or valid. 60 | #[no_mangle] 61 | pub unsafe extern "C" fn slitter_set_file_backed_slab_directory(path: *const c_char) { 62 | use std::ffi::CStr; 63 | 64 | if path.is_null() { 65 | set_file_backed_slab_directory(None); 66 | return; 67 | } 68 | 69 | let path_str = CStr::from_ptr(path) 70 | .to_str() 71 | .expect("path must be valid") 72 | .to_owned(); 73 | set_file_backed_slab_directory(Some(path_str.into())); 74 | } 75 | 76 | // TODO: we would like to re-export `slitter_allocate` and 77 | // `slitter_release`, but cargo won't let us do that. We 78 | // can however generate a static archive, which will let 79 | // the caller grab the C-side definition as needed. 80 | -------------------------------------------------------------------------------- /src/linear_ref.rs: -------------------------------------------------------------------------------- 1 | //! A `LinearRef` is a `NonNull` that can't be copied or 2 | //! cloned. We use it internally in Slitter to make it harder to 3 | //! accidentally duplicate allocations. 4 | #[cfg(any( 5 | all(test, feature = "check_contracts_in_tests"), 6 | feature = "check_contracts" 7 | ))] 8 | use contracts::*; 9 | #[cfg(not(any( 10 | all(test, feature = "check_contracts_in_tests"), 11 | feature = "check_contracts" 12 | )))] 13 | use disabled_contracts::*; 14 | 15 | use std::ffi::c_void; 16 | use std::ptr::NonNull; 17 | 18 | #[derive(Debug)] 19 | #[repr(transparent)] 20 | pub struct LinearRef { 21 | inner: NonNull, 22 | } 23 | 24 | /// `LinearRef` wrap allocations to help Slitter avoid duplicating 25 | /// allocations. 26 | impl LinearRef { 27 | /// Creates a new `LinearRef` from a `NonNull`. That `inner` 28 | /// `NonNull` must be the unique reference to that address. 29 | /// 30 | /// This function should only be used when directly interacting 31 | /// with external code (e.g., callers, the system allocator, or 32 | /// newly mapped pages). 33 | #[allow(clippy::assertions_on_constants)] 34 | #[requires(true, "`inner` must be unique (check manually)")] 35 | #[inline(always)] 36 | pub fn new(inner: NonNull) -> Self { 37 | Self { inner } 38 | } 39 | 40 | /// Converts a `LinearRef` to a `NonNull`. 41 | /// 42 | /// This function should only be used when directly interacting 43 | /// with external code (e.g., when returning an allocation to a 44 | /// caller). 45 | #[inline(always)] 46 | pub fn convert_to_non_null(self) -> NonNull { 47 | #[allow(clippy::let_and_return)] 48 | let ret = self.inner; 49 | 50 | #[cfg(any( 51 | all(test, feature = "check_contracts_in_tests"), 52 | feature = "check_contracts" 53 | ))] 54 | std::mem::forget(self); 55 | ret 56 | } 57 | 58 | /// Returns a `LinearRef` for an arbitrary non-zero integer 59 | #[cfg(test)] 60 | pub fn from_address(address: usize) -> Self { 61 | Self::new(NonNull::new(address as *mut c_void).expect("should not be zero")) 62 | } 63 | 64 | /// Only used for safety checks: returns a reference to the 65 | /// underlying `NonNull`. 66 | pub(crate) fn get(&self) -> &NonNull { 67 | &self.inner 68 | } 69 | } 70 | 71 | #[cfg(any( 72 | all(test, feature = "check_contracts_in_tests"), 73 | feature = "check_contracts" 74 | ))] 75 | impl Drop for LinearRef { 76 | #[allow(clippy::assertions_on_constants)] 77 | #[requires(false, "LinearRef should never be dropped.")] 78 | fn drop(&mut self) {} 79 | } 80 | 81 | impl PartialEq for LinearRef { 82 | fn eq(&self, other: &Self) -> bool { 83 | self.inner.as_ptr() == other.inner.as_ptr() 84 | } 85 | } 86 | 87 | impl Eq for LinearRef {} 88 | 89 | // It's safe to send LinearRef, because linearity means there's only 90 | // one reference to the underlying address, and thus only one thread 91 | // at a time has access to the data. 92 | unsafe impl Send for LinearRef {} 93 | -------------------------------------------------------------------------------- /src/magazine.rs: -------------------------------------------------------------------------------- 1 | //! The cache layer always allocates from and releases into small 2 | //! arrays of pointers to pre-allocated block. These small arrays are 3 | //! "magazines," and are themselves allocated and released by a 4 | //! "rack." 5 | #[cfg(any( 6 | all(test, feature = "check_contracts_in_tests"), 7 | feature = "check_contracts" 8 | ))] 9 | use contracts::*; 10 | #[cfg(not(any( 11 | all(test, feature = "check_contracts_in_tests"), 12 | feature = "check_contracts" 13 | )))] 14 | use disabled_contracts::*; 15 | 16 | use std::mem::MaybeUninit; 17 | 18 | #[cfg(any( 19 | all(test, feature = "check_contracts_in_tests"), 20 | feature = "check_contracts" 21 | ))] 22 | use crate::debug_allocation_map; 23 | #[cfg(any( 24 | all(test, feature = "check_contracts_in_tests"), 25 | feature = "check_contracts" 26 | ))] 27 | use crate::debug_type_map; 28 | #[cfg(any( 29 | all(test, feature = "check_contracts_in_tests"), 30 | feature = "check_contracts" 31 | ))] 32 | use crate::press; 33 | #[cfg(any( 34 | all(test, feature = "check_contracts_in_tests"), 35 | feature = "check_contracts" 36 | ))] 37 | use crate::Class; 38 | 39 | use crate::linear_ref::LinearRef; 40 | use crate::magazine_impl::MagazineImpl; 41 | 42 | /// A Magazine is a thin wrapper around MagazineImpl: the wrapping 43 | /// lets us impose a tighter contract on the interface used in the 44 | /// allocator, while keeping the internal implementation testable. 45 | /// 46 | /// A `PUSH_MAG: true` magazine can only grow, and a `PUSH_MAG: false` 47 | /// one can only shrink. 48 | /// 49 | /// The default for Push magazines is always full, and the default Pop 50 | /// magazine is always empty. 51 | #[derive(Default)] 52 | #[repr(transparent)] 53 | pub struct Magazine(pub(crate) MagazineImpl); 54 | 55 | pub type PushMagazine = Magazine; 56 | pub type PopMagazine = Magazine; 57 | 58 | /// Thread-local allocation caches also cache magazines locally. 59 | /// Buffering one magazine before pushing it to global freelists, 60 | /// helps avoid contention for common patterns like back-to-back 61 | /// allocation and deallocation. 62 | pub enum LocalMagazineCache { 63 | Nothing, 64 | Empty(PopMagazine), // Always an empty magazine with storage. 65 | Full(PushMagazine), // Always a full magazine with storage. 66 | } 67 | 68 | impl LocalMagazineCache { 69 | /// Stores `mag` in the cache, and returns the previously-cached 70 | /// magazine, if any. 71 | /// 72 | /// If `mag` cannot be cached, returns `mag`. 73 | pub fn populate( 74 | &mut self, 75 | mag: Magazine, 76 | ) -> Option> { 77 | use LocalMagazineCache::*; 78 | 79 | if !mag.has_storage() { 80 | return Some(mag); 81 | } 82 | 83 | let mut local; 84 | 85 | if mag.is_full() { 86 | let storage = mag.0.storage(); 87 | assert!(storage.is_some(), "Checked on entry"); 88 | local = LocalMagazineCache::Full(Magazine(MagazineImpl::new(storage))); 89 | } else if mag.is_empty() { 90 | let storage = mag.0.storage(); 91 | assert!(storage.is_some(), "Checked on entry"); 92 | local = LocalMagazineCache::Empty(Magazine(MagazineImpl::new(storage))); 93 | } else { 94 | return Some(mag); 95 | } 96 | 97 | std::mem::swap(self, &mut local); 98 | 99 | match local { 100 | Nothing => None, 101 | Empty(cached) => Some(Magazine(MagazineImpl::new(cached.0.storage()))), 102 | Full(cached) => Some(Magazine(MagazineImpl::new(cached.0.storage()))), 103 | } 104 | } 105 | 106 | /// Returns a full `PopMagazine` if one is cached. 107 | pub fn steal_full(&mut self) -> Option { 108 | use LocalMagazineCache::*; 109 | 110 | match self { 111 | Nothing => None, 112 | Empty(_) => None, 113 | Full(_) => { 114 | let mut private = LocalMagazineCache::Nothing; 115 | 116 | std::mem::swap(&mut private, self); 117 | let storage = if let Full(mag) = private { 118 | mag.0.storage()? 119 | } else { 120 | panic!("std::mem::swap changed enum"); 121 | }; 122 | 123 | assert_eq!( 124 | storage.num_allocated_slow, 125 | crate::magazine_impl::MAGAZINE_SIZE 126 | ); 127 | Some(Magazine(MagazineImpl::new(Some(storage)))) 128 | } 129 | } 130 | } 131 | 132 | /// Returns an empty `PushMagazine` if one is cached. 133 | pub fn steal_empty(&mut self) -> Option { 134 | use LocalMagazineCache::*; 135 | 136 | match self { 137 | Nothing => None, 138 | Full(_) => None, 139 | Empty(_) => { 140 | let mut private = LocalMagazineCache::Nothing; 141 | 142 | std::mem::swap(&mut private, self); 143 | let storage = if let Empty(mag) = private { 144 | mag.0.storage()? 145 | } else { 146 | panic!("std::mem::swap changed enum"); 147 | }; 148 | 149 | assert_eq!(storage.num_allocated_slow, 0); 150 | Some(Magazine(MagazineImpl::new(Some(storage)))) 151 | } 152 | } 153 | } 154 | } 155 | 156 | impl Magazine { 157 | /// Checks that current object's state is valid. 158 | /// 159 | /// If a class is provided, all allocations must match it. 160 | #[cfg(any( 161 | all(test, feature = "check_contracts_in_tests"), 162 | feature = "check_contracts" 163 | ))] 164 | pub fn check_rep(&self, maybe_class: Option) -> Result<(), &'static str> { 165 | if !self.0.check_rep() { 166 | return Err("MagazineImpl fails check_rep"); 167 | } 168 | 169 | // If we have an allocation class, the types must match. 170 | if let Some(class) = maybe_class { 171 | let info = class.info(); 172 | let zeroed_out = |alloc: &LinearRef| { 173 | let ptr = alloc.get().as_ptr() as *const u8; 174 | 175 | (0..info.layout.size()).all(|i| unsafe { std::ptr::read(ptr.add(i)) } == 0) 176 | }; 177 | 178 | for i in 0..self.0.len() { 179 | if let Some(alloc) = self.0.nth(i) { 180 | debug_allocation_map::can_be_allocated(class, alloc.get())?; 181 | debug_type_map::is_class(class, alloc)?; 182 | press::check_allocation(class, alloc.get().as_ptr() as usize)?; 183 | 184 | // If allocations are supposed to be zero-initialised, 185 | // everything in a pop mag should be zeroed out. 186 | if !PUSH_MAG && info.zero_init { 187 | if !zeroed_out(alloc) { 188 | return Err("Non-zero-initialised cached allocation"); 189 | } 190 | } 191 | } 192 | } 193 | } 194 | 195 | Ok(()) 196 | } 197 | 198 | /// Returns whether this magazine is backed by real storage, and 199 | /// thus has capacity. 200 | #[inline(always)] 201 | pub fn has_storage(&self) -> bool { 202 | self.0.has_storage() 203 | } 204 | 205 | #[inline(always)] 206 | pub fn is_full(&self) -> bool { 207 | self.0.is_full() 208 | } 209 | 210 | #[inline(always)] 211 | pub fn is_empty(&self) -> bool { 212 | self.0.is_empty() 213 | } 214 | } 215 | 216 | impl Magazine { 217 | /// Attempts to put an unused block back in the magazine. 218 | /// 219 | /// Returns that unused block on failure. 220 | #[invariant(self.check_rep(None).is_ok())] 221 | #[inline(always)] 222 | pub fn put(&mut self, freed: LinearRef) -> Option { 223 | self.0.put(freed) 224 | } 225 | } 226 | 227 | impl Magazine { 228 | /// Attempts to get an unused block from the magazine. 229 | #[invariant(self.check_rep(None).is_ok())] 230 | #[inline(always)] 231 | pub fn get(&mut self) -> Option { 232 | self.0.get() 233 | } 234 | 235 | /// Returns a slice for the used slots in the magazine 236 | #[inline(always)] 237 | fn get_populated(&self) -> &[MaybeUninit] { 238 | self.0.get_populated() 239 | } 240 | 241 | /// Returns a slice for the unused slots in the magazine 242 | #[inline(always)] 243 | fn get_unpopulated(&mut self) -> &mut [MaybeUninit] { 244 | self.0.get_unpopulated() 245 | } 246 | 247 | /// Marks the first `count` unused slots in the magazine as now populated. 248 | #[invariant(self.check_rep(None).is_ok())] 249 | #[inline(always)] 250 | fn commit_populated(&mut self, count: usize) { 251 | self.0.commit_populated(count) 252 | } 253 | } 254 | 255 | impl crate::class::ClassInfo { 256 | /// Returns a cached magazine; it is never empty. 257 | #[ensures(ret.is_some() -> !ret.as_ref().unwrap().is_empty(), 258 | "On success, the magazine is non-empty.")] 259 | #[ensures(ret.is_some() -> 260 | ret.as_ref().unwrap().check_rep(Some(self.id)).is_ok(), 261 | "Returned magazine makes sense for class.")] 262 | #[inline] 263 | pub(crate) fn get_cached_magazine( 264 | &self, 265 | cache: &mut LocalMagazineCache, 266 | ) -> Option { 267 | // Pop from partial magazines first, because we'd prefer to 268 | // have 0 partial mag. 269 | let ret = cache 270 | .steal_full() 271 | .or_else(|| self.partial_mags.try_pop()) 272 | .or_else(|| self.full_mags.pop())?; 273 | 274 | if self.zero_init { 275 | for allocation in ret.get_populated() { 276 | unsafe { 277 | let alloc = &*allocation.as_ptr(); 278 | std::ptr::write_bytes(alloc.get().as_ptr() as *mut u8, 0, self.layout.size()); 279 | } 280 | } 281 | } 282 | 283 | Some(ret) 284 | } 285 | 286 | /// Returns a magazine; it may be partially populated or empty. 287 | #[ensures(!ret.is_full(), "The returned magazine is never empty.")] 288 | #[ensures(ret.check_rep(Some(self.id)).is_ok(), 289 | "Returned magazine makes sense for class.")] 290 | #[inline] 291 | pub(crate) fn allocate_non_full_magazine( 292 | &self, 293 | cache: &mut LocalMagazineCache, 294 | ) -> PushMagazine { 295 | cache 296 | .steal_empty() 297 | .or_else(|| self.partial_mags.try_pop()) 298 | .unwrap_or_else(|| self.rack.allocate_empty_magazine()) 299 | } 300 | 301 | /// Attempts to return one allocation and to refill `mag`. 302 | /// 303 | /// When the return value is not `None` (i.e., not an OOM), `mag` 304 | /// is usually non-empty on exit; in the common case, `mag` is 305 | /// one allocation (the return value) short of full. 306 | #[invariant(mag.check_rep(Some(self.id)).is_ok(), 307 | "Magazine must match `self`.")] 308 | #[requires(mag.is_empty(), 309 | "Magazine must be empty on entry.")] 310 | #[ensures(ret.is_none() -> mag.is_empty(), 311 | "Allocation never fails when the magazine is non-empty.")] 312 | #[ensures(ret.is_some() -> 313 | debug_allocation_map::can_be_allocated(self.id, ret.as_ref().unwrap().get()).is_ok(), 314 | "Successful allocations are not in use.")] 315 | #[ensures(ret.is_some() -> 316 | debug_type_map::is_class(self.id, ret.as_ref().unwrap()).is_ok(), 317 | "Successful allocations come from an address of the correct class.")] 318 | #[ensures(ret.is_some() -> 319 | press::check_allocation(self.id, ret.as_ref().unwrap().get().as_ptr() as usize).is_ok(), 320 | "Sucessful allocations must have the allocation metadata set correctly.")] 321 | #[inline(never)] 322 | pub(crate) fn refill_magazine( 323 | &self, 324 | mag: &mut PopMagazine, 325 | cache: &mut LocalMagazineCache, 326 | ) -> Option { 327 | if let Some(mut new_mag) = self.get_cached_magazine(cache) { 328 | assert!(!new_mag.is_empty()); 329 | 330 | let allocated = new_mag.get(); 331 | std::mem::swap(&mut new_mag, mag); 332 | self.release_magazine(new_mag, Some(cache)); 333 | 334 | return allocated; 335 | } 336 | 337 | // Make sure we have capacity for `allocate_many_objects()` to 338 | // do something useful. 339 | if !mag.has_storage() { 340 | // We only enter this branch at most once per thread per 341 | // allocation class: the thread cache starts with a dummy 342 | // magazine, and we upgrade to a real one here. 343 | let mut new_mag = self.rack.allocate_empty_magazine(); 344 | std::mem::swap(&mut new_mag, mag); 345 | 346 | self.release_magazine(new_mag, Some(cache)); 347 | } 348 | 349 | let (count, allocated) = self.press.allocate_many_objects(mag.get_unpopulated()); 350 | mag.commit_populated(count); 351 | allocated 352 | } 353 | 354 | /// Acquires ownership of `spilled` and all cached allocations from 355 | /// the magazine, and removes some allocations from `mag`. 356 | /// 357 | /// On exit, `spilled` is in a magazine, and `mag` is usually not 358 | /// full; in the common case, `mag` only contains `spilled`. 359 | #[invariant(mag.check_rep(Some(self.id)).is_ok(), 360 | "Magazine must match `self`.")] 361 | #[requires(debug_allocation_map::has_been_released(self.id, spilled.get()).is_ok(), 362 | "A released block for `class` must have been marked as such.")] 363 | #[requires(debug_type_map::is_class(self.id, &spilled).is_ok(), 364 | "Deallocated blocks must match the class of the address range.")] 365 | #[requires(press::check_allocation(self.id, spilled.get().as_ptr() as usize).is_ok(), 366 | "Deallocated block must have the allocation metadata set correctly.")] 367 | #[inline(never)] 368 | pub(crate) fn clear_magazine( 369 | &self, 370 | mag: &mut PushMagazine, 371 | cache: &mut LocalMagazineCache, 372 | spilled: LinearRef, 373 | ) { 374 | let mut new_mag = self.allocate_non_full_magazine(cache); 375 | 376 | assert!(!new_mag.is_full()); 377 | assert_eq!(new_mag.put(spilled), None); 378 | 379 | std::mem::swap(&mut new_mag, mag); 380 | self.release_magazine(new_mag, Some(cache)); 381 | } 382 | 383 | /// Acquires ownership of `mag` and its cached allocations. 384 | #[requires(mag.check_rep(Some(self.id)).is_ok(), 385 | "Magazine must match `self`.")] 386 | #[inline(never)] 387 | pub(crate) fn release_magazine( 388 | &self, 389 | mut mag: Magazine, 390 | maybe_cache: Option<&mut LocalMagazineCache>, 391 | ) { 392 | if let Some(cache) = maybe_cache { 393 | match cache.populate(mag) { 394 | Some(new_mag) => mag = new_mag, 395 | None => return, 396 | } 397 | } 398 | 399 | if mag.is_empty() { 400 | self.rack.release_empty_magazine(mag); 401 | } else if mag.is_full() { 402 | self.full_mags.push(mag); 403 | } else { 404 | self.partial_mags.push(mag); 405 | } 406 | } 407 | } 408 | -------------------------------------------------------------------------------- /src/magazine_impl.rs: -------------------------------------------------------------------------------- 1 | //! The base `MagazineImpl` handles the pushing and popping of 2 | //! arbitrary pointers to inline uninit storage. It does not impose 3 | //! strong contracts; that's the responsibility of its `Magazine` 4 | //! wrapper struct. 5 | #[cfg(any( 6 | all(test, feature = "check_contracts_in_tests"), 7 | feature = "check_contracts" 8 | ))] 9 | use contracts::*; 10 | #[cfg(not(any( 11 | all(test, feature = "check_contracts_in_tests"), 12 | feature = "check_contracts" 13 | )))] 14 | use disabled_contracts::*; 15 | 16 | #[cfg(any( 17 | all(test, feature = "check_contracts_in_tests"), 18 | feature = "check_contracts" 19 | ))] 20 | use std::ffi::c_void; 21 | 22 | use std::mem::MaybeUninit; 23 | use std::ptr::NonNull; 24 | 25 | use crate::linear_ref::LinearRef; 26 | 27 | #[cfg(not(feature = "test_only_small_constants"))] 28 | pub const MAGAZINE_SIZE: u32 = 30; 29 | 30 | #[cfg(feature = "test_only_small_constants")] 31 | pub const MAGAZINE_SIZE: u32 = 6; 32 | 33 | /// The `MagazineStorage` is the heap-allocated storage for a 34 | /// magazine. 35 | /// 36 | /// The same struct is available in C as `struct magazine_storage`, in 37 | /// `mag.h`. 38 | #[repr(C)] 39 | pub struct MagazineStorage { 40 | allocations: [MaybeUninit; MAGAZINE_SIZE as usize], 41 | 42 | /// Single linked list linkage. 43 | pub(crate) link: Option>, 44 | 45 | /// The `allocations` array is populated from the bottom up; the 46 | /// first `num_allocated_slow` indices have values, and the 47 | /// remainder are uninitialised. 48 | /// 49 | /// This field may not be accurate when wrapped in a `MagazineImpl`. 50 | pub(crate) num_allocated_slow: u32, 51 | } 52 | 53 | /// The `MagazineImpl` is the actual implementation for the storage. 54 | /// This split lets us cache information inline. 55 | /// 56 | /// The same struct is available in C as `struct magazine`, in 57 | /// `mag.h`. 58 | /// 59 | /// If `PUSH_MAG` is true, we have a push-only magazine. If it's 60 | /// false, we have a pop-only magazine. 61 | #[repr(C)] 62 | pub struct MagazineImpl { 63 | /// "Pop" (PUSH_MAG = false) magazines decrement the `top_of_stack` 64 | /// from MAGAZINE_SIZE down to 0. 65 | /// 66 | /// "Push" (PUSH_MAG = true) magazines increment the `top_of_stack` 67 | /// from `-MAGAZINE_SIZE` up to 0. 68 | /// 69 | /// The backing storate expects the opposite direction than the 70 | /// Push strategy, so we must convert when going in and out of the 71 | /// raw `MagazineStorage` representation. 72 | top_of_stack: isize, 73 | 74 | /// Always populated when `top_of_stack != 0`. 75 | inner: Option<&'static mut MagazineStorage>, 76 | } 77 | 78 | impl MagazineImpl { 79 | /// Wraps `maybe_inner` in an impl. If `maybe_inner` is None, the 80 | /// a push magazine is initialised to full, and a pop to empty. 81 | // Disabled precondition: lifetimes are too hard for contracts. 82 | // #[requires(inner.link.is_none())] 83 | #[inline(always)] 84 | pub fn new(maybe_inner: Option<&'static mut MagazineStorage>) -> Self { 85 | if let Some(inner) = maybe_inner { 86 | #[cfg(any( 87 | all(test, feature = "check_contracts_in_tests"), 88 | feature = "check_contracts" 89 | ))] 90 | assert!(inner.link.is_none()); 91 | 92 | if PUSH_MAG { 93 | Self { 94 | top_of_stack: inner.num_allocated_slow as isize - MAGAZINE_SIZE as isize, 95 | inner: Some(inner), 96 | } 97 | } else { 98 | Self { 99 | top_of_stack: inner.num_allocated_slow as isize, 100 | inner: Some(inner), 101 | } 102 | } 103 | } else { 104 | Self { 105 | top_of_stack: 0, 106 | inner: None, 107 | } 108 | } 109 | } 110 | 111 | /// Returns whether this magazine is backed by real storage, and 112 | /// thus has capacity. 113 | #[inline(always)] 114 | pub fn has_storage(&self) -> bool { 115 | self.inner.is_some() 116 | } 117 | 118 | // Disabled postcondition: lifetimes are too hard for contracts. 119 | // #[requires(self.check_rep())] 120 | // #[ensures(ret.link.is_none())] 121 | #[inline(always)] 122 | pub fn storage(self) -> Option<&'static mut MagazineStorage> { 123 | #[cfg(any( 124 | all(test, feature = "check_contracts_in_tests"), 125 | feature = "check_contracts" 126 | ))] 127 | assert!(self.check_rep()); 128 | 129 | let inner = self.inner?; 130 | if PUSH_MAG { 131 | inner.num_allocated_slow = (MAGAZINE_SIZE as isize + self.top_of_stack) as u32; 132 | } else { 133 | inner.num_allocated_slow = self.top_of_stack as u32; 134 | } 135 | 136 | #[cfg(any( 137 | all(test, feature = "check_contracts_in_tests"), 138 | feature = "check_contracts" 139 | ))] 140 | assert!(inner.link.is_none()); 141 | Some(inner) 142 | } 143 | 144 | #[invariant(self.check_rep())] 145 | #[inline] 146 | pub fn is_full(&self) -> bool { 147 | if PUSH_MAG { 148 | self.top_of_stack == 0 149 | } else { 150 | self.top_of_stack == MAGAZINE_SIZE as isize 151 | } 152 | } 153 | 154 | #[invariant(self.check_rep())] 155 | #[inline] 156 | pub fn is_empty(&self) -> bool { 157 | if PUSH_MAG { 158 | self.top_of_stack == -(MAGAZINE_SIZE as isize) 159 | } else { 160 | self.top_of_stack == 0 161 | } 162 | } 163 | 164 | /// Returns the number of elements in the magazine. 165 | #[cfg(any(test, feature = "check_contracts"))] 166 | pub fn len(&self) -> usize { 167 | if PUSH_MAG { 168 | (self.top_of_stack + MAGAZINE_SIZE as isize) as usize 169 | } else { 170 | self.top_of_stack as usize 171 | } 172 | } 173 | 174 | /// Returns a reference to the element at `index`. 175 | /// 176 | /// Calling this with an index that has no valid value 177 | /// will cause undefined behafiour. 178 | #[cfg(any( 179 | all(test, feature = "check_contracts_in_tests"), 180 | feature = "check_contracts" 181 | ))] 182 | pub fn nth(&self, index: usize) -> Option<&LinearRef> { 183 | Some(unsafe { &*self.inner.as_ref()?.allocations[index].as_ptr() }) 184 | } 185 | 186 | /// Checks that the current object's state is valid. 187 | #[cfg(any( 188 | all(test, feature = "check_contracts_in_tests"), 189 | feature = "check_contracts" 190 | ))] 191 | pub(crate) fn check_rep(&self) -> bool { 192 | let inner; 193 | 194 | if let Some(storage) = &self.inner { 195 | inner = storage; 196 | } else { 197 | // Missing storage is only allowed when `top_of_stack == 198 | // 0`. 199 | return self.top_of_stack == 0; 200 | }; 201 | 202 | // The storage should never be in a linked stack. 203 | if inner.link.is_some() { 204 | return false; 205 | } 206 | 207 | if PUSH_MAG { 208 | // The top of stack index should be in [-MAGAZINE_SIZE, 0] 209 | // for Push magazine. 210 | if self.top_of_stack < -(MAGAZINE_SIZE as isize) || self.top_of_stack > 0 { 211 | return false; 212 | } 213 | } else { 214 | // The top of stack index should be in [0, MAGAZINE_SIZE] 215 | // for Pop magazines. 216 | if self.top_of_stack < 0 || self.top_of_stack > MAGAZINE_SIZE as isize { 217 | return false; 218 | } 219 | } 220 | 221 | // Everything before `allocated` must be populated, and thus 222 | // non-NULL. Everything at or after `allocated` is garbage 223 | // and must not be read. 224 | inner 225 | .allocations 226 | .iter() 227 | .take(self.len()) 228 | .all(|entry| !entry.as_ptr().is_null()) 229 | } 230 | } 231 | 232 | impl Default for MagazineImpl { 233 | #[inline(always)] 234 | fn default() -> Self { 235 | Self::new(None) 236 | } 237 | } 238 | 239 | // Logic for "push" magazines. 240 | impl MagazineImpl { 241 | /// Converts to a Pop magazine 242 | #[cfg(test)] 243 | pub fn into_pop(self) -> MagazineImpl { 244 | MagazineImpl::new(self.storage()) 245 | } 246 | 247 | /// Attempts to put an unused block back in the magazine. 248 | /// 249 | /// Returns that unused block on failure. 250 | #[invariant(self.check_rep())] 251 | #[ensures(ret.is_none() -> self.top_of_stack == old(self.top_of_stack) + 1, 252 | "We add one element on success.")] 253 | #[ensures(ret.is_some() -> self.top_of_stack == old(self.top_of_stack), 254 | "We don't change the stack on failure.")] 255 | #[ensures(ret.is_some() -> old(freed.get().as_ptr()) == ret.as_ref().unwrap().get().as_ptr(), 256 | "On failure, we return `freed`.")] 257 | #[ensures(ret.is_none() -> old(freed.get().as_ptr()) == self.peek(), 258 | "On success, `freed` is in the magazine.")] 259 | #[ensures(old(self.is_full()) == ret.is_some(), 260 | "We only fail to push to full magazines.")] 261 | #[inline(always)] 262 | pub fn put(&mut self, freed: LinearRef) -> Option { 263 | if cfg!(feature = "c_fast_path") { 264 | extern "C" { 265 | fn slitter__magazine_put( 266 | mag: &mut MagazineImpl, 267 | freed: LinearRef, 268 | ) -> Option; 269 | } 270 | 271 | return unsafe { slitter__magazine_put(self, freed) }; 272 | } 273 | 274 | let index = self.top_of_stack; 275 | if index == 0 { 276 | return Some(freed); 277 | } 278 | 279 | self.top_of_stack += 1; 280 | unsafe { 281 | self.inner 282 | .as_mut() 283 | .expect("non-zero top_of_stack must have a storage") 284 | .allocations[(MAGAZINE_SIZE as isize + index) as usize] 285 | .as_mut_ptr() 286 | .write(freed); 287 | } 288 | None 289 | } 290 | 291 | /// Contract-only: returns the pointer at the top of the stack, of NULL if none. 292 | #[cfg(any( 293 | all(test, feature = "check_contracts_in_tests"), 294 | feature = "check_contracts" 295 | ))] 296 | fn peek(&self) -> *mut c_void { 297 | if self.top_of_stack == -(MAGAZINE_SIZE as isize) { 298 | std::ptr::null::() as *mut _ 299 | } else { 300 | unsafe { 301 | self.inner 302 | .as_ref() 303 | .expect("non-zero top_of_stack must have a storage") 304 | .allocations[(MAGAZINE_SIZE as isize + self.top_of_stack) as usize - 1] 305 | .as_ptr() 306 | .as_ref() 307 | } 308 | .unwrap() 309 | .get() 310 | .as_ptr() 311 | } 312 | } 313 | } 314 | 315 | // Functions that only exist on "pop" magazines. 316 | impl MagazineImpl { 317 | /// Converts to a Push magazine 318 | #[cfg(test)] 319 | pub fn into_push(self) -> MagazineImpl { 320 | MagazineImpl::new(self.storage()) 321 | } 322 | 323 | /// Attempts to get an unused block from the magazine. 324 | #[invariant(self.check_rep(), "Representation makes sense.")] 325 | #[ensures(old(self.is_empty()) == ret.is_none(), 326 | "We only fail to pop from empty magazines.")] 327 | #[ensures(ret.is_none() -> self.top_of_stack == old(self.top_of_stack), 328 | "We don't change the stack size on failure.")] 329 | #[ensures(ret.is_some() -> self.top_of_stack == old(self.top_of_stack) - 1, 330 | "Must remove one element on success.")] 331 | #[ensures(ret.is_some() -> ret.as_ref().unwrap().get().as_ptr() == old(self.peek()), 332 | "Must return the top of stack on success.")] 333 | #[inline(always)] 334 | pub fn get(&mut self) -> Option { 335 | if cfg!(feature = "c_fast_path") { 336 | extern "C" { 337 | fn slitter__magazine_get(mag: &mut MagazineImpl) -> Option; 338 | } 339 | 340 | return unsafe { slitter__magazine_get(self) }; 341 | } 342 | 343 | if self.top_of_stack == 0 { 344 | return None; 345 | } 346 | 347 | self.top_of_stack -= 1; 348 | let mut old = MaybeUninit::uninit(); 349 | std::mem::swap( 350 | &mut old, 351 | &mut self 352 | .inner 353 | .as_mut() 354 | .expect("non-zero top_of_stack must have a storage") 355 | .allocations[self.top_of_stack as usize], 356 | ); 357 | Some(unsafe { old.assume_init() }) 358 | } 359 | 360 | /// Returns a slice for the used slots in the magazine 361 | // No invariant: they confuse the borrow checker. 362 | #[inline(always)] 363 | pub fn get_populated(&self) -> &[MaybeUninit] { 364 | if let Some(inner) = &self.inner { 365 | &inner.allocations[0..self.top_of_stack as usize] 366 | } else { 367 | &[] 368 | } 369 | } 370 | 371 | /// Returns a slice for the unused slots in the magazine 372 | // No invariant: they confuse the borrow checker. 373 | #[inline(always)] 374 | pub fn get_unpopulated(&mut self) -> &mut [MaybeUninit] { 375 | if let Some(inner) = &mut self.inner { 376 | &mut inner.allocations[self.top_of_stack as usize..] 377 | } else { 378 | &mut [] 379 | } 380 | } 381 | 382 | /// Marks the first `count` unused slots in the magazine as now populated. 383 | #[invariant(self.check_rep())] 384 | #[requires(count <= MAGAZINE_SIZE as usize - self.top_of_stack as usize)] 385 | #[inline(always)] 386 | pub fn commit_populated(&mut self, count: usize) { 387 | self.top_of_stack += count as isize; 388 | } 389 | 390 | /// Contract-only: returns the pointer at the top of the stack, of NULL if none. 391 | #[cfg(any( 392 | all(test, feature = "check_contracts_in_tests"), 393 | feature = "check_contracts" 394 | ))] 395 | fn peek(&self) -> *mut c_void { 396 | if self.top_of_stack == 0 { 397 | std::ptr::null::() as *mut _ 398 | } else { 399 | unsafe { 400 | self.inner 401 | .as_ref() 402 | .expect("non-zero top_of_stack must have a storage") 403 | .allocations[self.top_of_stack as usize - 1] 404 | .as_ptr() 405 | .as_ref() 406 | } 407 | .unwrap() 408 | .get() 409 | .as_ptr() 410 | } 411 | } 412 | } 413 | 414 | impl Default for MagazineStorage { 415 | fn default() -> Self { 416 | // Proof that MagazineImpl its constituents are FFI-safe. 417 | #[allow(dead_code)] 418 | extern "C" fn unused( 419 | _mag: MagazineStorage, 420 | _ref: Option, 421 | _link: Option>, 422 | ) { 423 | } 424 | 425 | Self { 426 | // Safe to leave this as garbage: we never read past 427 | // `num_allocated_slow`. 428 | allocations: unsafe { MaybeUninit::uninit().assume_init() }, 429 | link: None, 430 | num_allocated_slow: 0, 431 | } 432 | } 433 | } 434 | 435 | /// We should only drop empty magazines. 436 | impl Drop for MagazineStorage { 437 | #[requires(self.num_allocated_slow == 0, 438 | "Only empty magazines can be dropped.")] 439 | fn drop(&mut self) {} 440 | } 441 | 442 | #[test] 443 | fn smoke_test_magazine() { 444 | let rack = crate::rack::get_default_rack(); 445 | let mut mag = rack.allocate_empty_magazine::().0; 446 | 447 | // Getting an empty magazine should return None 448 | assert_eq!(mag.get(), None); // mag: [] 449 | 450 | // And getting it again should still return None. 451 | assert_eq!(mag.get(), None); // mag: [] 452 | 453 | let mut mag2 = mag.into_push(); 454 | 455 | assert_eq!(mag2.put(LinearRef::from_address(1)), None); // mag: [1] 456 | assert_eq!(mag2.put(LinearRef::from_address(2)), None); // mag: [1, 2] 457 | 458 | let mut mag3 = mag2.into_pop(); 459 | { 460 | let popped = mag3.get().expect("should have a value"); // mag: [1] 461 | 462 | assert_eq!(popped.get().as_ptr() as usize, 2); 463 | std::mem::forget(popped); 464 | } 465 | 466 | let mut mag4 = mag3.into_push(); 467 | assert_eq!(mag4.put(LinearRef::from_address(3)), None); // mag: [1, 3] 468 | 469 | let mut mag5 = mag4.into_pop(); 470 | { 471 | let popped = mag5.get().expect("should have a value"); 472 | 473 | assert_eq!(popped.get().as_ptr() as usize, 3); // mag: [1] 474 | std::mem::forget(popped); 475 | } 476 | 477 | { 478 | let popped = mag5.get().expect("should have a value"); 479 | 480 | assert_eq!(popped.get().as_ptr() as usize, 1); // mag: [] 481 | std::mem::forget(popped); 482 | } 483 | 484 | rack.release_empty_magazine(crate::magazine::Magazine(mag5)); 485 | } 486 | 487 | #[test] 488 | fn magazine_fill_up() { 489 | let rack = crate::rack::get_default_rack(); 490 | let mut mag = rack.allocate_empty_magazine::().0; 491 | 492 | // Fill up the magazine. 493 | for i in 1..=MAGAZINE_SIZE as usize { 494 | assert_eq!(mag.len(), i - 1); 495 | assert_eq!(mag.put(LinearRef::from_address(i)), None); 496 | assert_eq!(mag.len(), i); 497 | } 498 | 499 | // This insert should fail 500 | let failed_insert = mag 501 | .put(LinearRef::from_address(usize::MAX)) 502 | .expect("should fail"); 503 | assert_eq!(failed_insert.get().as_ptr() as usize, usize::MAX); 504 | std::mem::forget(failed_insert); 505 | 506 | assert_eq!(mag.len(), MAGAZINE_SIZE as usize); 507 | 508 | let mut pop_mag = mag.into_pop(); 509 | 510 | // We should pop in LIFO order. 511 | for i in (1..=MAGAZINE_SIZE as usize).rev() { 512 | assert_eq!(pop_mag.len(), i); 513 | let popped = pop_mag.get().expect("has value"); 514 | assert_eq!(popped.get().as_ptr() as usize, i as usize); 515 | std::mem::forget(popped); 516 | 517 | assert_eq!(pop_mag.len(), i - 1); 518 | } 519 | 520 | // And now the magazine should be empty. 521 | assert_eq!(pop_mag.len(), 0); 522 | // So all subsequent `get()` calls will return None. 523 | assert_eq!(pop_mag.get(), None); 524 | assert_eq!(pop_mag.get(), None); 525 | assert_eq!(pop_mag.len(), 0); 526 | 527 | rack.release_empty_magazine(crate::magazine::Magazine(pop_mag)); 528 | } 529 | -------------------------------------------------------------------------------- /src/magazine_stack.rs: -------------------------------------------------------------------------------- 1 | //! A `MagazineStack` is a thread-safe single-linked intrusive stack 2 | //! of magazines. 3 | #[cfg(any( 4 | all(test, feature = "check_contracts_in_tests"), 5 | feature = "check_contracts" 6 | ))] 7 | use contracts::*; 8 | #[cfg(not(any( 9 | all(test, feature = "check_contracts_in_tests"), 10 | feature = "check_contracts" 11 | )))] 12 | use disabled_contracts::*; 13 | 14 | use std::mem::MaybeUninit; 15 | use std::ptr::NonNull; 16 | use std::sync::atomic::AtomicPtr; 17 | use std::sync::atomic::AtomicUsize; 18 | use std::sync::atomic::Ordering; 19 | 20 | use crate::magazine::Magazine; 21 | use crate::magazine_impl::MagazineImpl; 22 | use crate::magazine_impl::MagazineStorage; 23 | 24 | /// A `MagazineStack` is a single-linked stack with a generation 25 | /// counter to protect against ABA. We do not have to worry 26 | /// about reclamation races because `MagazineStorage` are immortal: 27 | /// `Rack`s never free them, and simply cache empty magazines in 28 | /// a `MagazineStack`. 29 | #[repr(C)] 30 | #[repr(align(16))] 31 | pub struct MagazineStack { 32 | top_of_stack: AtomicPtr, 33 | generation: AtomicUsize, 34 | } 35 | 36 | // These are declared in stack.h 37 | extern "C" { 38 | fn slitter__stack_push(stack: &MagazineStack, mag: NonNull); 39 | fn slitter__stack_pop(stack: &MagazineStack, out_mag: *mut NonNull) -> bool; 40 | fn slitter__stack_try_pop( 41 | stack: &MagazineStack, 42 | out_mag: *mut NonNull, 43 | ) -> bool; 44 | } 45 | 46 | impl MagazineStack { 47 | pub fn new() -> Self { 48 | Self { 49 | top_of_stack: Default::default(), 50 | generation: AtomicUsize::new(0), 51 | } 52 | } 53 | 54 | #[requires(mag.check_rep(None).is_ok(), 55 | "Magazine must make sense.")] 56 | #[inline(always)] 57 | pub fn push(&self, mag: Magazine) { 58 | if let Some(storage) = mag.0.storage() { 59 | unsafe { slitter__stack_push(&self, storage.into()) } 60 | } 61 | } 62 | 63 | #[ensures(ret.is_some() -> 64 | ret.as_ref().unwrap().check_rep(None).is_ok(), 65 | "Magazine should make sense.")] 66 | #[inline(always)] 67 | pub fn pop(&self) -> Option> { 68 | if self.top_of_stack.load(Ordering::Relaxed).is_null() { 69 | return None; 70 | } 71 | 72 | let mut dst: MaybeUninit> = MaybeUninit::uninit(); 73 | if unsafe { slitter__stack_pop(&self, dst.as_mut_ptr()) } { 74 | // If `stack_pop` returns true, `dst` must contain a valid owning pointer 75 | // to a `MagazineStorage`. 76 | let storage = unsafe { &mut *dst.assume_init().as_ptr() }; 77 | Some(Magazine(MagazineImpl::new(Some(storage)))) 78 | } else { 79 | None 80 | } 81 | } 82 | 83 | #[ensures(ret.is_some() -> 84 | ret.as_ref().unwrap().check_rep(None).is_ok(), 85 | "Magazine should make sense.")] 86 | #[inline(always)] 87 | pub fn try_pop(&self) -> Option> { 88 | if self.top_of_stack.load(Ordering::Relaxed).is_null() { 89 | return None; 90 | } 91 | 92 | let mut dst: MaybeUninit> = MaybeUninit::uninit(); 93 | if unsafe { slitter__stack_try_pop(&self, dst.as_mut_ptr()) } { 94 | // If `stack_pop` returns true, `dst` must contain a valid owning pointer 95 | // to a `MagazineStorage`. 96 | let storage = unsafe { &mut *dst.assume_init().as_ptr() }; 97 | Some(Magazine(MagazineImpl::new(Some(storage)))) 98 | } else { 99 | None 100 | } 101 | } 102 | } 103 | 104 | // MagazineStack is safe to `Send` because we convert `NonNull` 105 | // to/from mutable references around the lock. 106 | unsafe impl Send for MagazineStack {} 107 | unsafe impl Sync for MagazineStack {} 108 | 109 | #[test] 110 | fn magazine_stack_smoke_test() { 111 | let rack = crate::rack::get_default_rack(); 112 | let stack = MagazineStack::new(); 113 | 114 | // Push/pop shouldn't care about the magazines' polarity. 115 | stack.push(rack.allocate_empty_magazine::()); 116 | stack.push(rack.allocate_empty_magazine::()); 117 | 118 | assert!(stack.pop::().is_some()); 119 | 120 | stack.push(rack.allocate_empty_magazine::()); 121 | assert!(stack.pop::().is_some()); 122 | assert!(stack.pop::().is_some()); 123 | 124 | assert!(stack.pop::().is_none()); 125 | } 126 | -------------------------------------------------------------------------------- /src/map.rs: -------------------------------------------------------------------------------- 1 | //! Rust bindings for the support code in C that calls out to mmap. 2 | //! 3 | //! TODO: wrap strerror_r usefully. 4 | use std::ptr::NonNull; 5 | use std::{ffi::c_void, fs::File}; 6 | 7 | // These helpers are declared in `c/map.h`. 8 | extern "C" { 9 | fn slitter__page_size() -> i64; 10 | fn slitter__reserve_region(size: usize, OUT_errno: *mut i32) -> Option>; 11 | fn slitter__release_region(base: NonNull, size: usize) -> i32; 12 | fn slitter__allocate_region(base: NonNull, size: usize) -> i32; 13 | fn slitter__allocate_fd_region( 14 | fd: i32, 15 | offset: usize, 16 | base: NonNull, 17 | size: usize, 18 | ) -> i32; 19 | } 20 | 21 | fn page_size_or_die() -> usize { 22 | let ret = unsafe { slitter__page_size() }; 23 | 24 | if ret <= 0 { 25 | panic!("Unable to find page_size: errno={}", -ret); 26 | } 27 | 28 | ret as usize 29 | } 30 | 31 | lazy_static::lazy_static! { 32 | static ref PAGE_SIZE: usize = page_size_or_die(); 33 | } 34 | 35 | /// Returns the system page size. 36 | #[inline] 37 | pub fn page_size() -> usize { 38 | *PAGE_SIZE 39 | } 40 | 41 | /// Attempts to reserve an *address space* region of `size` bytes. 42 | /// 43 | /// The `size` argument must be a multiple of the page size. 44 | pub fn reserve_region(size: usize) -> Result, i32> { 45 | let mut errno: i32 = 0; 46 | 47 | assert!( 48 | size > 0 && (size % page_size()) == 0, 49 | "Bad region size={} page_size={}", 50 | size, 51 | page_size() 52 | ); 53 | 54 | if let Some(base) = unsafe { slitter__reserve_region(size, &mut errno) } { 55 | Ok(base) 56 | } else { 57 | Err(errno) 58 | } 59 | } 60 | 61 | /// Releases a region of `size` bytes starting at `base`. 62 | /// 63 | /// The size argument must be a multiple of the page size. 64 | pub fn release_region(base: NonNull, size: usize) -> Result<(), i32> { 65 | if size == 0 { 66 | return Ok(()); 67 | } 68 | 69 | assert!( 70 | (size % page_size()) == 0, 71 | "Bad region size={} page_size={}", 72 | size, 73 | page_size() 74 | ); 75 | 76 | let ret = unsafe { slitter__release_region(base, size) }; 77 | 78 | if ret == 0 { 79 | Ok(()) 80 | } else { 81 | Err(-ret) 82 | } 83 | } 84 | 85 | /// Backs a region of `size` bytes starting at `base` with 86 | /// (demand-faulted) memory. 87 | /// 88 | /// The size argument must be a multiple of the page size. 89 | pub fn allocate_region(base: NonNull, size: usize) -> Result<(), i32> { 90 | if size == 0 { 91 | return Ok(()); 92 | } 93 | 94 | assert!( 95 | (size % page_size()) == 0, 96 | "Bad region size={} page_size={}", 97 | size, 98 | page_size() 99 | ); 100 | 101 | let ret = unsafe { slitter__allocate_region(base, size) }; 102 | 103 | if ret == 0 { 104 | Ok(()) 105 | } else { 106 | Err(-ret) 107 | } 108 | } 109 | 110 | /// Backs a region of `size` bytes starting at `base` with 111 | /// (demand-faulted) memory from `file`. The `file` must be empty on 112 | /// entry. 113 | /// 114 | /// The size argument must be a multiple of the page size. 115 | pub fn allocate_file_region(file: File, base: NonNull, size: usize) -> Result<(), i32> { 116 | use std::os::unix::io::FromRawFd; 117 | use std::os::unix::io::IntoRawFd; 118 | 119 | assert_eq!( 120 | file.metadata().expect("has metadata").len(), 121 | 0, 122 | "The file must be empty on entry" 123 | ); 124 | 125 | if size == 0 { 126 | return Ok(()); 127 | } 128 | 129 | assert!( 130 | (size % page_size()) == 0, 131 | "Bad region size={} page_size={}", 132 | size, 133 | page_size() 134 | ); 135 | 136 | file.set_len(size as u64).map_err(|_| 0)?; 137 | let fd = file.into_raw_fd(); 138 | let ret = unsafe { slitter__allocate_fd_region(fd, 0, base, size) }; 139 | 140 | // Make sure to drop the file before returning. 141 | unsafe { File::from_raw_fd(fd) }; 142 | 143 | if ret == 0 { 144 | Ok(()) 145 | } else { 146 | Err(-ret) 147 | } 148 | } 149 | 150 | #[test] 151 | fn test_page_size() { 152 | assert_ne!(page_size(), 0); 153 | 154 | // We only develop on platforms with 4K pages. 155 | assert_eq!(page_size(), 4096); 156 | } 157 | 158 | // Simulate a data + metadata allocation workflow: overallocate, trim 159 | // the slop, and ask for real memory in some of the remaining space. 160 | #[test] 161 | fn smoke_test() { 162 | let region_size = 1usize << 21; 163 | let mut base = reserve_region(3 * region_size).expect("reserve should succeed"); 164 | 165 | assert!(region_size > 3 * page_size()); 166 | 167 | // We overallocated `base` by 3x. Drop the bottom and top 168 | // `region_size` bytes from the range. 169 | release_region(base, region_size).expect("should release the bottom slop"); 170 | base = NonNull::new((base.as_ptr() as usize + region_size) as *mut c_void) 171 | .expect("Should be non-null"); 172 | 173 | let top_slop = NonNull::new((base.as_ptr() as usize + region_size) as *mut c_void) 174 | .expect("Should be non-null"); 175 | release_region(top_slop, region_size).expect("should release the top slop"); 176 | 177 | // Conceptually split the region in three ranges: a one-page 178 | // region at the base, a guard page just after, and the rest. 179 | let bottom = base; // one page 180 | let _guard = NonNull::new((base.as_ptr() as usize + page_size()) as *mut c_void) 181 | .expect("Should be non-null"); 182 | let remainder = NonNull::new((base.as_ptr() as usize + 2 * page_size()) as *mut c_void) 183 | .expect("Should be non-null"); 184 | 185 | // Start by allocating the bottom and remainder regions. 186 | allocate_region(bottom, page_size()).expect("should allocate bottom"); 187 | allocate_region(remainder, region_size - 2 * page_size()).expect("should allocate remainder"); 188 | 189 | // And now release everything. 190 | release_region(base, region_size).expect("should release everything"); 191 | } 192 | -------------------------------------------------------------------------------- /src/mapper.rs: -------------------------------------------------------------------------------- 1 | //! A `Mapper` is responsible for acquiring address space and backing 2 | //! memory from the operating system. Each `Mill` is parameterised on 3 | //! such a `Mapper`. 4 | #[cfg(any( 5 | all(test, feature = "check_contracts_in_tests"), 6 | feature = "check_contracts" 7 | ))] 8 | use contracts::*; 9 | #[cfg(not(any( 10 | all(test, feature = "check_contracts_in_tests"), 11 | feature = "check_contracts" 12 | )))] 13 | use disabled_contracts::*; 14 | 15 | use std::collections::HashMap; 16 | use std::ffi::c_void; 17 | use std::ptr::NonNull; 18 | use std::sync::Mutex; 19 | 20 | #[cfg(any( 21 | all(test, feature = "check_contracts_in_tests"), 22 | feature = "check_contracts" 23 | ))] 24 | use crate::debug_arange_map; 25 | 26 | pub use crate::mill::GUARD_PAGE_SIZE; 27 | 28 | #[allow(clippy::inline_fn_without_body)] 29 | #[contract_trait] 30 | pub trait Mapper: std::fmt::Debug + Sync { 31 | /// Returns the mapping granularity for this mapper. All calls 32 | /// into the mapper will align addresses and sizes to that page 33 | /// size. 34 | /// 35 | /// The page size must be constant for the lifetime of a process. 36 | #[ensures(ret > 0 && ret & (ret - 1) == 0, "page size must be a power of 2")] 37 | #[ensures(ret <= GUARD_PAGE_SIZE, "pages should be smaller than guard ranges")] 38 | fn page_size(&self) -> usize; 39 | 40 | /// Attempts to reserve a range of address space. On success, 41 | /// returns the address of the first byte in the reserved range, 42 | /// and the number of bytes actually reserved. Both values 43 | /// should be aligned to the `page_size()`. 44 | /// 45 | /// Any page-aligned allocation of `desired_size` bytes will 46 | /// suffice to satisfy the caller. However, the mapper may also 47 | /// try to do something smarter, knowing that its caller wants an 48 | /// range of `data_size` bytes aligned to `data_size`, with 49 | /// `prefix` bytes before that range, and `suffix` bytes after. 50 | /// 51 | /// The `data_size`, `prefix`, and `suffix` values may 52 | /// be misaligned with respect to the page size. 53 | #[requires(desired_size % self.page_size() == 0)] 54 | #[requires(desired_size > 0)] 55 | #[requires(data_size > 0)] 56 | #[ensures(ret.is_ok() -> debug_arange_map::reserve_range(ret.unwrap().0.as_ptr() as usize, ret.unwrap().1).is_ok())] 57 | #[ensures(ret.is_ok() -> ret.as_ref().unwrap().0.as_ptr() != std::ptr::null_mut(), 58 | "The mapped range never includes NULL")] 59 | #[ensures(ret.is_ok() -> ret.as_ref().unwrap().1 < usize::MAX - ret.as_ref().unwrap().0.as_ptr() as usize, 60 | "The mapped range never overflows")] 61 | fn reserve( 62 | &self, 63 | desired_size: usize, 64 | data_size: usize, 65 | prefix: usize, 66 | suffix: usize, 67 | ) -> Result<(NonNull, usize), i32>; 68 | 69 | /// Releases a page-aligned range that was previously obtained 70 | /// with a single call to `reserve`. The `release`d range is 71 | /// always a subset of a range that was returned by a single 72 | /// `reserve` call. 73 | #[requires(base.as_ptr() as usize % self.page_size() == 0)] 74 | #[requires(size % self.page_size() == 0)] 75 | #[requires(debug_arange_map::releasable_range(base.as_ptr() as usize, size).is_ok())] 76 | #[ensures(ret.is_ok() -> debug_arange_map::release_range(base.as_ptr() as usize, size).is_ok())] 77 | fn release(&self, base: NonNull, size: usize) -> Result<(), i32>; 78 | 79 | /// Prepares a page-aligned range of metadata for read and write 80 | /// access. The `allocate`d range is always a subset of a range 81 | /// that was returned by a single `reserve` call. 82 | /// 83 | /// On successful return, the range must be zero-filled. 84 | #[requires(debug_arange_map::can_mark_metadata(base.as_ptr() as usize, size).is_ok())] 85 | #[ensures(ret.is_ok() -> debug_arange_map::mark_metadata(base.as_ptr() as usize, size).is_ok())] 86 | fn allocate_meta(&self, base: NonNull, size: usize) -> Result<(), i32>; 87 | 88 | /// Prepares a page-aligned range of object data for read and 89 | /// write access. The `allocate`d range is always a subset of a 90 | /// range that was returned by a single `reserve` call. 91 | /// 92 | /// On successful return, the range must be zero-filled. 93 | #[requires(debug_arange_map::can_mark_data(base.as_ptr() as usize, size).is_ok())] 94 | #[ensures(ret.is_ok() -> debug_arange_map::mark_data(base.as_ptr() as usize, size).is_ok())] 95 | fn allocate_data(&self, base: NonNull, size: usize) -> Result<(), i32>; 96 | } 97 | 98 | #[derive(Debug)] 99 | struct DefaultMapper {} 100 | 101 | lazy_static::lazy_static! { 102 | static ref NAMED_MAPPERS: Mutex> = { 103 | let mut map: HashMap = HashMap::new(); 104 | 105 | map.insert("file".to_string(), Box::leak(Box::new(crate::file_backed_mapper::FileBackedMapper{}))); 106 | Mutex::new(map) 107 | }; 108 | } 109 | 110 | /// Upserts the mapper associated with `name`. 111 | pub fn register_mapper(name: String, mapper: &'static dyn Mapper) { 112 | let mut mappers = NAMED_MAPPERS.lock().unwrap(); 113 | 114 | mappers.insert(name, mapper); 115 | } 116 | 117 | /// Returns the mapper for the given `name`, if one exists, or the 118 | /// default mapper if `name` is `None`. 119 | /// 120 | /// # Errors 121 | /// 122 | /// Returns `Err` if no such mapper is defined. 123 | pub fn get_mapper(name: Option<&str>) -> Result<&'static dyn Mapper, &'static str> { 124 | lazy_static::lazy_static! { 125 | static ref DEFAULT_MAPPER: DefaultMapper = DefaultMapper{}; 126 | } 127 | 128 | match name { 129 | Some(key) => { 130 | let mappers = NAMED_MAPPERS.lock().unwrap(); 131 | 132 | Ok(*mappers.get(key).ok_or("Mapper not found")?) 133 | } 134 | None => Ok(&*DEFAULT_MAPPER), 135 | } 136 | } 137 | 138 | #[contract_trait] 139 | impl Mapper for DefaultMapper { 140 | fn page_size(&self) -> usize { 141 | crate::map::page_size() 142 | } 143 | 144 | fn reserve( 145 | &self, 146 | desired_size: usize, 147 | _data_size: usize, 148 | _prefix: usize, 149 | _suffix: usize, 150 | ) -> Result<(NonNull, usize), i32> { 151 | let region: NonNull = crate::map::reserve_region(desired_size)?; 152 | Ok((region, desired_size)) 153 | } 154 | 155 | fn release(&self, base: NonNull, size: usize) -> Result<(), i32> { 156 | crate::map::release_region(base, size) 157 | } 158 | 159 | fn allocate_meta(&self, base: NonNull, size: usize) -> Result<(), i32> { 160 | crate::map::allocate_region(base, size) 161 | } 162 | 163 | fn allocate_data(&self, base: NonNull, size: usize) -> Result<(), i32> { 164 | crate::map::allocate_region(base, size) 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/press.rs: -------------------------------------------------------------------------------- 1 | //! A `Press` creates new allocations for a given `Class`. The 2 | //! allocations must be such that the `Press` can also map valid 3 | //! addresses back to their `Class`. 4 | //! 5 | //! While each class gets its own press, the latter requirement means 6 | //! that the presses must all implement compatible metadata stashing 7 | //! schemes. This works because `Mill`s all use the same scheme. 8 | //! 9 | //! We enable mostly lock-free operations by guaranteeing that each 10 | //! span and corresponding metadata is immortal once allocated. 11 | #[cfg(any( 12 | all(test, feature = "check_contracts_in_tests"), 13 | feature = "check_contracts" 14 | ))] 15 | use contracts::*; 16 | #[cfg(not(any( 17 | all(test, feature = "check_contracts_in_tests"), 18 | feature = "check_contracts" 19 | )))] 20 | use disabled_contracts::*; 21 | 22 | use std::alloc::Layout; 23 | use std::ffi::c_void; 24 | use std::mem::MaybeUninit; 25 | use std::num::NonZeroUsize; 26 | use std::ptr::NonNull; 27 | use std::sync::atomic::AtomicPtr; 28 | use std::sync::atomic::AtomicUsize; 29 | use std::sync::atomic::Ordering; 30 | use std::sync::Mutex; 31 | 32 | #[cfg(any( 33 | all(test, feature = "check_contracts_in_tests"), 34 | feature = "check_contracts" 35 | ))] 36 | use crate::debug_allocation_map; 37 | #[cfg(any( 38 | all(test, feature = "check_contracts_in_tests"), 39 | feature = "check_contracts" 40 | ))] 41 | use crate::debug_arange_map; 42 | #[cfg(any( 43 | all(test, feature = "check_contracts_in_tests"), 44 | feature = "check_contracts" 45 | ))] 46 | use crate::debug_type_map; 47 | 48 | use crate::linear_ref::LinearRef; 49 | use crate::mill; 50 | use crate::mill::Mill; 51 | use crate::mill::SpanMetadata; 52 | use crate::mill::MAX_SPAN_SIZE; 53 | use crate::Class; 54 | 55 | /// We batch-allocate at most this many elements at once. This limit 56 | /// makes it clear that a 64-bit counter will not wraparound. 57 | /// 58 | /// In practice, callers ask for one more than the magazine size, at 59 | /// most, and that's less than this limit. 60 | const MAX_ALLOCATION_BATCH: usize = 100; 61 | 62 | static_assertions::const_assert!( 63 | (crate::magazine_impl::MAGAZINE_SIZE as usize) < MAX_ALLOCATION_BATCH 64 | ); 65 | 66 | /// We don't guarantee alignment greater than this value. 67 | pub const MAX_OBJECT_ALIGNMENT: usize = 4096; 68 | 69 | static_assertions::const_assert!(MAX_OBJECT_ALIGNMENT <= mill::MAX_SPAN_SIZE); 70 | 71 | #[derive(Debug)] 72 | pub struct Press { 73 | /// The current span that services bump pointer allocation. 74 | bump: AtomicPtr, 75 | 76 | /// Writes to the bump itself (i.e., updating the `AtomicPtr` 77 | /// itself) go through this lock. 78 | mill: Mutex<&'static Mill>, 79 | layout: Layout, 80 | class: Class, 81 | } 82 | 83 | /// Returns Ok if the allocation `address` might have come from a `Press` for `class`. 84 | /// 85 | /// # Errors 86 | /// 87 | /// Returns Err if the address definitely did not come from that `class`. 88 | #[inline] 89 | pub fn check_allocation(class: Class, address: usize) -> Result<(), &'static str> { 90 | let meta_ptr = SpanMetadata::from_allocation_address(address); 91 | 92 | let meta = unsafe { meta_ptr.as_mut() }.ok_or("Derived a bad metadata address")?; 93 | if meta.class_id != Some(class.id()) { 94 | Err("Incorrect class id") 95 | } else { 96 | Ok(()) 97 | } 98 | } 99 | 100 | impl Press { 101 | /// Returns a fresh `Press` for an object `class` with that object 102 | /// `layout`, and the underlying mapper `mapper_name` (`None` for 103 | /// the default `Mapper` / `Mill`). 104 | /// 105 | /// All presses with the same `mapper_name` share the same `Mill`. 106 | /// 107 | /// # Errors 108 | /// 109 | /// Returns `Err` when the layout violates the allocator's constraints, 110 | /// or no mapper can be found for `mapper_name`. 111 | pub fn new( 112 | class: Class, 113 | mut layout: Layout, 114 | mapper_name: Option<&str>, 115 | ) -> Result { 116 | if layout.align() > MAX_OBJECT_ALIGNMENT { 117 | return Err("slitter only supports alignment up to 4 KB"); 118 | } 119 | 120 | layout = layout.pad_to_align(); 121 | assert_eq!(layout.size() % layout.align(), 0); 122 | 123 | if layout.size() > MAX_SPAN_SIZE / 2 { 124 | Err("Class elements too large (after alignment)") 125 | } else { 126 | Ok(Self { 127 | bump: Default::default(), 128 | mill: Mutex::new(mill::get_mill(mapper_name)?), 129 | layout, 130 | class, 131 | }) 132 | } 133 | } 134 | 135 | /// Associates the `count` allocations starting at `begin` with `self.class`. 136 | #[cfg(any( 137 | all(test, feature = "check_contracts_in_tests"), 138 | feature = "check_contracts" 139 | ))] 140 | fn associate_range(&self, begin: usize, count: usize) -> Result<(), &'static str> { 141 | for i in 0..count { 142 | debug_type_map::associate_class(self.class, begin + i * self.layout.size())?; 143 | } 144 | 145 | Ok(()) 146 | } 147 | 148 | /// Checks if the `count` allocations starting at `begin` are associated with `self.class`. 149 | #[cfg(any( 150 | all(test, feature = "check_contracts_in_tests"), 151 | feature = "check_contracts" 152 | ))] 153 | fn is_range_associated_and_free(&self, begin: usize, count: usize) -> Result<(), &'static str> { 154 | for i in 0..count { 155 | let address = NonNull::new((begin + i * self.layout.size()) as *mut c_void) 156 | .ok_or("allocated NULL pointer")?; 157 | debug_type_map::ptr_is_class(self.class, &address)?; 158 | debug_allocation_map::can_be_allocated(self.class, &address)?; 159 | } 160 | 161 | Ok(()) 162 | } 163 | 164 | /// Checks that all `count` allocations starting at `begin` are associated with `self.class`. 165 | #[cfg(any( 166 | all(test, feature = "check_contracts_in_tests"), 167 | feature = "check_contracts" 168 | ))] 169 | fn check_allocation_range(&self, begin: usize, count: usize) -> Result<(), &'static str> { 170 | for i in 0..count { 171 | check_allocation(self.class, begin + i * self.layout.size())?; 172 | } 173 | 174 | Ok(()) 175 | } 176 | 177 | /// Attempts to allocate up to `max_count` consecutive object by 178 | /// bumping the metadata pointer. 179 | /// 180 | /// Returns the address of the first object and the number of 181 | /// allocations on success. 182 | #[requires(debug_arange_map::is_metadata(meta as * mut SpanMetadata as usize, 183 | std::mem::size_of::()).is_ok(), 184 | "The `meta` reference must come from a metadata range.")] 185 | #[ensures(ret.is_some() -> ret.unwrap().1.get() <= max_count.get(), 186 | "We never return more than `max_count` allocations.")] 187 | #[ensures(ret.is_some() -> ret.unwrap().0.get() as usize % self.layout.align() == 0, 188 | "The base address is correctly aligned.")] 189 | #[ensures(ret.is_some() -> self.associate_range(ret.unwrap().0.get(), ret.unwrap().1.get()).is_ok(), 190 | "On success, it must be possible to associate the returned address with `self.class`.")] 191 | #[ensures(ret.is_some() -> 192 | debug_arange_map::is_data(ret.unwrap().0.get(), self.layout.size() * ret.unwrap().1.get()).is_ok(), 193 | "On success, the returned data must come from a data range.")] 194 | #[ensures(ret.is_some() -> self.check_allocation_range(ret.unwrap().0.get(), ret.unwrap().1.get()).is_ok(), 195 | "On success, the allocations must all have the class metadata set up.")] 196 | fn try_allocate_from_span( 197 | &self, 198 | meta: &mut SpanMetadata, 199 | max_count: NonZeroUsize, 200 | ) -> Option<(NonZeroUsize, NonZeroUsize)> { 201 | let desired = max_count.get().clamp(0, MAX_ALLOCATION_BATCH); 202 | let limit = meta.bump_limit as usize; 203 | 204 | let allocated_id = meta.bump_ptr.fetch_add(desired, Ordering::Relaxed); 205 | if allocated_id >= limit { 206 | return None; 207 | } 208 | 209 | // This is our actual allocation count: our allocated range 210 | // starts at `allocated_id`, and stops at `allocated_id + 211 | // desired` (that's how much we acquired), or at `limit` 212 | // if we acquired more than the bump limit. 213 | let actual = (limit - allocated_id).clamp(0, desired); 214 | 215 | // `meta.bump_ptr` is incremented atomically, so 216 | // we always return fresh addresses. 217 | // 218 | // XXX: This expression has to satisfy the `ensures` 219 | // postconditions; they're checked in 220 | // `assert_new_bump_is_safe`, including the alignment 221 | // of `span_begin`. 222 | Some(( 223 | NonZeroUsize::new(meta.span_begin + allocated_id * self.layout.size())?, 224 | NonZeroUsize::new(actual)?, 225 | )) 226 | } 227 | 228 | /// Asserts that every allocation in `bump` is valid for the 229 | /// allocation. 230 | #[cfg(any( 231 | all(test, feature = "check_contracts_in_tests"), 232 | feature = "check_contracts" 233 | ))] 234 | fn assert_new_bump_is_safe(&self, bump: *mut SpanMetadata) { 235 | assert!( 236 | debug_arange_map::is_metadata(bump as usize, std::mem::size_of::()) 237 | .is_ok() 238 | ); 239 | 240 | let meta = unsafe { bump.as_mut() }.expect("must be valid"); 241 | 242 | assert_eq!(meta.span_begin % self.layout.align(), 0); 243 | 244 | for i in 0..meta.bump_limit as usize { 245 | let address = meta.span_begin + i * self.layout.size(); 246 | assert!(debug_arange_map::is_data(address, self.layout.size()).is_ok()); 247 | assert!(check_allocation(self.class, address).is_ok()); 248 | } 249 | } 250 | 251 | #[cfg(not(any( 252 | all(test, feature = "check_contracts_in_tests"), 253 | feature = "check_contracts" 254 | )))] 255 | #[inline] 256 | fn assert_new_bump_is_safe(&self, _bump: *mut SpanMetadata) {} 257 | 258 | /// Attempts to replace our bump pointer with a new one. 259 | #[ensures(ret.is_ok() -> 260 | self.bump.load(Ordering::Relaxed) != old(self.bump.load(Ordering::Relaxed)), 261 | "On success, the bump Span has been updated.")] 262 | #[ensures(debug_arange_map::is_metadata(self.bump.load(Ordering::Relaxed) as usize, 263 | std::mem::size_of::()).is_ok(), 264 | "The bump struct must point to a valid metadata range.")] 265 | fn try_replace_span(&self, expected: *mut SpanMetadata) -> Result<(), i32> { 266 | if self.bump.load(Ordering::Relaxed) != expected { 267 | // Someone else made progress. 268 | 269 | return Ok(()); 270 | } 271 | 272 | let mill = self.mill.lock().unwrap(); 273 | // Check again with the lock held, before allocating a new span. 274 | if self.bump.load(Ordering::Relaxed) != expected { 275 | return Ok(()); 276 | } 277 | 278 | // Get a new span. It must have enough bytes for one 279 | // allocation, but will usually have more (the default desired 280 | // size, nearly 1 MB). 281 | let range = mill.get_span(self.layout.size(), None)?; 282 | let meta: &mut _ = range.meta; 283 | 284 | // We should have a fresh Metadata struct before claiming it as ours. 285 | assert_eq!(meta.class_id, None); 286 | meta.class_id = Some(self.class.id()); 287 | meta.bump_limit = (range.data_size / self.layout.size()) as u32; 288 | assert!( 289 | meta.bump_limit > 0, 290 | "layout.size > MAX_SPAN_SIZE, but we check for that in the constructor." 291 | ); 292 | meta.bump_ptr = AtomicUsize::new(0); 293 | meta.span_begin = range.data as usize; 294 | 295 | // Make sure allocations in the trail are properly marked as being ours. 296 | for trailing_meta in range.trail { 297 | // This Metadata struct must not already be allocated. 298 | assert_eq!(trailing_meta.class_id, None); 299 | trailing_meta.class_id = Some(self.class.id()); 300 | } 301 | 302 | // Publish the metadata for our fresh span. 303 | assert_eq!(self.bump.load(Ordering::Relaxed), expected); 304 | self.assert_new_bump_is_safe(meta); 305 | self.bump.store(meta, Ordering::Release); 306 | Ok(()) 307 | } 308 | 309 | /// Attempts to allocate up to `max_count` objects. Returns Ok() 310 | /// if we tried to allocate from the current bump region. 311 | /// 312 | /// On allocation success, returns Ok(Some(base_address, object_count)) 313 | /// 314 | /// # Errors 315 | /// 316 | /// Returns `Err` if we failed to grab a new bump region. 317 | #[ensures(ret.is_ok() && ret.unwrap().is_some() -> 318 | ret.unwrap().unwrap().1.get() <= max_count.get(), 319 | "We never overallocate.")] 320 | #[ensures(ret.is_ok() && ret.unwrap().is_some() -> 321 | self.is_range_associated_and_free(ret.unwrap().unwrap().0.get(), ret.unwrap().unwrap().1.get()).is_ok(), 322 | "Successful allocations are fresh, or match the class and avoid double-allocation.")] 323 | #[ensures(ret.is_ok() && ret.unwrap().is_some() -> 324 | self.check_allocation_range(ret.unwrap().unwrap().0.get(), ret.unwrap().unwrap().1.get()).is_ok(), 325 | "Sucessful allocations must have the allocation metadata set correctly.")] 326 | fn try_allocate_once( 327 | &self, 328 | max_count: NonZeroUsize, 329 | ) -> Result, i32> { 330 | let meta_ptr: *mut SpanMetadata = self.bump.load(Ordering::Acquire); 331 | 332 | if let Some(meta) = unsafe { meta_ptr.as_mut() } { 333 | if let Some(result) = self.try_allocate_from_span(meta, max_count) { 334 | return Ok(Some(result)); 335 | } 336 | } 337 | 338 | // Either we didn't find any span metadata, or bump 339 | // allocation failed. Either way, let's try to put 340 | // a new span in. 341 | self.try_replace_span(meta_ptr).map(|_| None) 342 | } 343 | 344 | /// Tries to allocate up to `max_count` objects. Only fails on OOM. 345 | #[ensures(ret.is_some() -> 346 | ret.unwrap().1.get() <= max_count.get(), 347 | "We never overallocate.")] 348 | #[ensures(ret.is_some() -> 349 | self.is_range_associated_and_free(ret.unwrap().0.get(), ret.unwrap().1.get()).is_ok(), 350 | "Successful allocations are fresh, or match the class and avoid double-allocation.")] 351 | #[ensures(ret.is_some() -> 352 | self.check_allocation_range(ret.unwrap().0.get(), ret.unwrap().1.get()).is_ok(), 353 | "Sucessful allocations must have the allocation metadata set correctly.")] 354 | fn try_allocate(&self, max_count: NonZeroUsize) -> Option<(NonZeroUsize, NonZeroUsize)> { 355 | loop { 356 | match self.try_allocate_once(max_count) { 357 | Err(_) => return None, // TODO: log 358 | Ok(Some(result)) => return Some(result), 359 | _ => continue, 360 | } 361 | } 362 | } 363 | 364 | #[ensures(ret.is_some() -> 365 | debug_allocation_map::can_be_allocated(self.class, ret.as_ref().unwrap().get()).is_ok(), 366 | "Successful allocations are fresh, or match the class and avoid double-allocation.")] 367 | #[ensures(ret.is_some() -> 368 | debug_type_map::is_class(self.class, ret.as_ref().unwrap()).is_ok(), 369 | "On success, the new allocation has the correct type.")] 370 | #[ensures(ret.is_some() -> 371 | check_allocation(self.class, ret.as_ref().unwrap().get().as_ptr() as usize).is_ok(), 372 | "Sucessful allocations must have the allocation metadata set correctly.")] 373 | pub fn allocate_one_object(&self) -> Option { 374 | let (address, _count) = self.try_allocate(NonZeroUsize::new(1).unwrap())?; 375 | 376 | debug_assert_eq!(_count.get(), 1); 377 | Some(LinearRef::new(unsafe { 378 | NonNull::new_unchecked(address.get() as *mut c_void) 379 | })) 380 | } 381 | 382 | /// Attempts to allocate multiple objects: first the second return 383 | /// value, and then as many elements in `dst` as possible. 384 | /// 385 | /// Returns the number of elements populated in `dst` (starting 386 | /// at low indices), and an allocated object if possible. 387 | #[ensures(ret.1.is_some() -> 388 | debug_allocation_map::can_be_allocated(self.class, ret.1.as_ref().unwrap().get()).is_ok(), 389 | "Successful allocations are fresh, or match the class and avoid double-allocation.")] 390 | #[ensures(ret.1.is_some() -> 391 | debug_type_map::is_class(self.class, ret.1.as_ref().unwrap()).is_ok(), 392 | "On success, the new allocation has the correct type.")] 393 | #[ensures(ret.1.is_some() -> 394 | check_allocation(self.class, ret.1.as_ref().unwrap().get().as_ptr() as usize).is_ok(), 395 | "Sucessful allocations must have the allocation metadata set correctly.")] 396 | #[ensures(ret.1.is_none() -> ret.0 == 0, 397 | "We always try to satisfy the return value first.")] 398 | // We don't check `dst` because the contract expression would be 399 | // unsafe, but it's the same as `ret.1.is_some()` for all 400 | // populated elements. 401 | // 402 | // We do check the same invariants in the target `Magazine` via 403 | // `ClassInfo::refill_magazine`. 404 | pub fn allocate_many_objects( 405 | &self, 406 | dst: &mut [MaybeUninit], 407 | ) -> (usize, Option) { 408 | let elsize = self.layout.size(); 409 | 410 | match self.try_allocate(NonZeroUsize::new(dst.len() + 1).expect("Should not overflow")) { 411 | Some((base, count)) => { 412 | let mut address = base.get(); 413 | 414 | // Acquires the next element from `base[0..count]`. 415 | let mut get_ref = || { 416 | let ret = 417 | LinearRef::new(unsafe { NonNull::new_unchecked(address as *mut c_void) }); 418 | 419 | address += elsize; 420 | ret 421 | }; 422 | 423 | let ret = Some(get_ref()); 424 | 425 | let mut populated = 0; 426 | for uninit in dst.iter_mut().take(count.get() - 1) { 427 | unsafe { uninit.as_mut_ptr().write(get_ref()) }; 428 | populated += 1; 429 | } 430 | 431 | debug_assert!(populated <= count.get()); 432 | (populated, ret) 433 | } 434 | None => (0, None), 435 | } 436 | } 437 | } 438 | -------------------------------------------------------------------------------- /src/rack.rs: -------------------------------------------------------------------------------- 1 | //! A `Rack` manages empty `Magazine`s: it allocates them as needed, 2 | //! and recycles unused empty ones. 3 | #[cfg(any( 4 | all(test, feature = "check_contracts_in_tests"), 5 | feature = "check_contracts" 6 | ))] 7 | use contracts::*; 8 | #[cfg(not(any( 9 | all(test, feature = "check_contracts_in_tests"), 10 | feature = "check_contracts" 11 | )))] 12 | use disabled_contracts::*; 13 | 14 | use crate::magazine::Magazine; 15 | use crate::magazine_impl::MagazineImpl; 16 | use crate::magazine_stack::MagazineStack; 17 | 18 | /// A `Rack` allocates and recycles empty magazines. 19 | pub struct Rack { 20 | freelist: MagazineStack, 21 | } 22 | 23 | impl Rack { 24 | pub fn new() -> Self { 25 | use crate::magazine_impl::MagazineStorage; 26 | use crate::magazine_impl::MAGAZINE_SIZE; 27 | 28 | extern "C" { 29 | fn slitter__magazine_size() -> usize; 30 | fn slitter__magazine_storage_sizeof() -> usize; 31 | fn slitter__magazine_sizeof() -> usize; 32 | } 33 | 34 | unsafe { 35 | assert_eq!(MAGAZINE_SIZE as usize, slitter__magazine_size()); 36 | assert_eq!( 37 | std::mem::size_of::(), 38 | slitter__magazine_storage_sizeof() 39 | ); 40 | assert_eq!( 41 | std::mem::size_of::>(), 42 | slitter__magazine_sizeof() 43 | ); 44 | assert_eq!( 45 | std::mem::size_of::>(), 46 | slitter__magazine_sizeof() 47 | ); 48 | assert_eq!( 49 | std::mem::size_of::>(), 50 | slitter__magazine_sizeof() 51 | ); 52 | assert_eq!( 53 | std::mem::size_of::>(), 54 | slitter__magazine_sizeof() 55 | ); 56 | } 57 | 58 | Self { 59 | freelist: MagazineStack::new(), 60 | } 61 | } 62 | } 63 | 64 | /// Returns a reference to the global default magazine rack. 65 | pub fn get_default_rack() -> &'static Rack { 66 | lazy_static::lazy_static! { static ref RACK: Rack = Rack::new(); }; 67 | 68 | &RACK 69 | } 70 | 71 | impl Rack { 72 | #[ensures(ret.has_storage() && ret.is_empty(), "Newly allocated magazines are empty.")] 73 | #[inline(always)] 74 | pub fn allocate_empty_magazine(&self) -> Magazine { 75 | self.freelist.pop().unwrap_or_else(|| { 76 | Magazine(MagazineImpl::new(Some(Box::leak(Box::new( 77 | Default::default(), 78 | ))))) 79 | }) 80 | } 81 | 82 | #[requires(!mag.has_storage() || mag.is_empty(), "Only empty magazines are released to the Rack.")] 83 | pub fn release_empty_magazine(&self, mag: Magazine) { 84 | // This function is only called during thread shutdown, and 85 | // things will really break if mag is actually non-empty. 86 | assert!(!mag.has_storage() || mag.is_empty()); 87 | self.freelist.push(mag); 88 | } 89 | } 90 | 91 | #[test] 92 | fn smoke_test_rack() { 93 | let rack = get_default_rack(); 94 | let mag = rack.allocate_empty_magazine::(); 95 | 96 | rack.release_empty_magazine(mag); 97 | } 98 | --------------------------------------------------------------------------------