├── .gitignore ├── info ├── elfmalloc-data │ ├── prod-cons-mem.png │ ├── prod-cons-tp.png │ ├── shbench-large-tp.png │ ├── shbench-small-tp.png │ ├── shbench-large-mem.png │ ├── shbench-small-mem.png │ ├── threadtest-large-mem.png │ ├── threadtest-large-tp.png │ ├── threadtest-small-mem.png │ └── threadtest-small-tp.png ├── bagpipes.md └── elfmalloc-performance.md ├── elfc ├── travis.sh ├── appveyor.sh ├── CHANGELOG.md ├── README.md ├── Cargo.toml └── src │ └── lib.rs ├── bsalloc ├── travis.sh ├── appveyor.sh ├── CHANGELOG.md ├── Cargo.toml ├── README.md └── src │ ├── lib.rs │ └── bsalloc.rs ├── git-hooks ├── README.md ├── install-hooks.sh └── pre-commit.sh ├── malloc-bind ├── travis.sh ├── appveyor.sh ├── README.md ├── Cargo.toml └── CHANGELOG.md ├── object-alloc ├── travis.sh ├── appveyor.sh ├── CHANGELOG.md ├── Cargo.toml ├── README.md └── src │ └── lib.rs ├── object-alloc-test ├── travis.sh ├── appveyor.sh ├── CHANGELOG.md ├── README.md ├── Cargo.toml └── src │ ├── lib.rs │ ├── leaky_alloc.rs │ └── types.rs ├── alloc-tls ├── CHANGELOG.md ├── Cargo.toml ├── README.md └── DESIGN.md ├── alloc-fmt ├── src │ └── bin │ │ ├── print.rs │ │ └── assert.rs ├── appveyor.sh ├── travis.sh ├── CHANGELOG.md ├── README.md └── Cargo.toml ├── bagpipe ├── appveyor.sh ├── travis.sh ├── CHANGELOG.md ├── Cargo.toml ├── README.md └── src │ └── bag.rs ├── mmap-alloc ├── appveyor.sh ├── README.md ├── travis.sh ├── Cargo.toml └── CHANGELOG.md ├── elfmalloc ├── appveyor.sh ├── src │ ├── alloc_type.rs │ ├── lib.rs │ ├── stats.rs │ ├── alloc_impl.rs │ ├── utils.rs │ ├── sources.rs │ └── bin │ │ └── bench_vec.rs ├── travis.sh ├── CHANGELOG.md ├── README.md └── Cargo.toml ├── slab-alloc ├── appveyor.sh ├── travis.sh ├── CHANGELOG.md ├── README.md ├── Cargo.toml └── src │ ├── aligned.rs │ ├── backing.rs │ ├── init.rs │ ├── large.rs │ └── tests.rs ├── LICENSE-MIT ├── .travis.yml ├── test-scripts └── check-copyright-comments.sh ├── appveyor.yml ├── README.md └── CONTRIBUTING.md /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /info/elfmalloc-data/prod-cons-mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ailisp/allocators-rs/master/info/elfmalloc-data/prod-cons-mem.png -------------------------------------------------------------------------------- /info/elfmalloc-data/prod-cons-tp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ailisp/allocators-rs/master/info/elfmalloc-data/prod-cons-tp.png -------------------------------------------------------------------------------- /info/elfmalloc-data/shbench-large-tp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ailisp/allocators-rs/master/info/elfmalloc-data/shbench-large-tp.png -------------------------------------------------------------------------------- /info/elfmalloc-data/shbench-small-tp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ailisp/allocators-rs/master/info/elfmalloc-data/shbench-small-tp.png -------------------------------------------------------------------------------- /info/elfmalloc-data/shbench-large-mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ailisp/allocators-rs/master/info/elfmalloc-data/shbench-large-mem.png -------------------------------------------------------------------------------- /info/elfmalloc-data/shbench-small-mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ailisp/allocators-rs/master/info/elfmalloc-data/shbench-small-mem.png -------------------------------------------------------------------------------- /info/elfmalloc-data/threadtest-large-mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ailisp/allocators-rs/master/info/elfmalloc-data/threadtest-large-mem.png -------------------------------------------------------------------------------- /info/elfmalloc-data/threadtest-large-tp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ailisp/allocators-rs/master/info/elfmalloc-data/threadtest-large-tp.png -------------------------------------------------------------------------------- /info/elfmalloc-data/threadtest-small-mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ailisp/allocators-rs/master/info/elfmalloc-data/threadtest-small-mem.png -------------------------------------------------------------------------------- /info/elfmalloc-data/threadtest-small-tp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ailisp/allocators-rs/master/info/elfmalloc-data/threadtest-small-tp.png -------------------------------------------------------------------------------- /elfc/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | travis-cargo --only nightly build 14 | # TODO: Test 15 | -------------------------------------------------------------------------------- /bsalloc/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | travis-cargo --only nightly build 14 | RUST_BACKTRACE=1 travis-cargo --only nightly test 15 | -------------------------------------------------------------------------------- /git-hooks/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Git Hooks 9 | 10 | This directory contains hooks which git will execute at various times. To 11 | install the hooks, run `./install-hooks.sh`. 12 | -------------------------------------------------------------------------------- /malloc-bind/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | travis-cargo --only nightly build 14 | RUST_BACKTRACE=1 travis-cargo --only nightly test 15 | -------------------------------------------------------------------------------- /object-alloc/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | travis-cargo --only nightly build 14 | RUST_BACKTRACE=1 travis-cargo --only nightly test 15 | -------------------------------------------------------------------------------- /elfc/appveyor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | if [ "$RUST_NIGHTLY" != "1" ]; then 14 | exit 0 15 | fi 16 | 17 | cargo build 18 | # TODO: Test 19 | -------------------------------------------------------------------------------- /object-alloc-test/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | travis-cargo --only nightly build 14 | RUST_BACKTRACE=1 travis-cargo --only nightly test 15 | -------------------------------------------------------------------------------- /bsalloc/appveyor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | if [ "$RUST_NIGHTLY" != "1" ]; then 14 | exit 0 15 | fi 16 | 17 | cargo build 18 | RUST_BACKTRACE=1 cargo test 19 | -------------------------------------------------------------------------------- /malloc-bind/appveyor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | if [ "$RUST_NIGHTLY" != "1" ]; then 14 | exit 0 15 | fi 16 | 17 | cargo build 18 | RUST_BACKTRACE=1 cargo test 19 | -------------------------------------------------------------------------------- /object-alloc/appveyor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | if [ "$RUST_NIGHTLY" != "1" ]; then 14 | exit 0 15 | fi 16 | 17 | cargo build 18 | RUST_BACKTRACE=1 cargo test 19 | -------------------------------------------------------------------------------- /object-alloc-test/appveyor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | if [ "$RUST_NIGHTLY" != "1" ]; then 14 | exit 0 15 | fi 16 | 17 | cargo build 18 | RUST_BACKTRACE=1 cargo test 19 | -------------------------------------------------------------------------------- /alloc-tls/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Changelog 9 | 10 | All notable changes to this project will be documented in this file. 11 | 12 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 13 | 14 | ## [Unreleased] 15 | -------------------------------------------------------------------------------- /alloc-fmt/src/bin/print.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | #[macro_use] 9 | extern crate alloc_fmt; 10 | 11 | fn main() { 12 | alloc_print!("alloc_print\n"); 13 | alloc_println!("alloc_println"); 14 | alloc_eprint!("alloc_eprint\n"); 15 | alloc_eprintln!("alloc_eprintln"); 16 | } 17 | -------------------------------------------------------------------------------- /bagpipe/appveyor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | cargo build 14 | RUST_BACKTRACE=1 cargo test 15 | for feature in check_empty_yq prime_schedules staggered_indexes huge_segments; do 16 | RUST_BACKTRACE=1 cargo test --features "$feature" 17 | done 18 | -------------------------------------------------------------------------------- /bagpipe/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | travis-cargo build 14 | RUST_BACKTRACE=1 travis-cargo test 15 | for feature in check_empty_yq prime_schedules staggered_indexes huge_segments; do 16 | RUST_BACKTRACE=1 travis-cargo test -- --features "$feature" 17 | done 18 | -------------------------------------------------------------------------------- /alloc-fmt/appveyor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | if [ "$RUST_NIGHTLY" != "1" ]; then 14 | exit 0 15 | fi 16 | 17 | cargo build 18 | RUST_BACKTRACE=1 cargo test 19 | RUST_BACKTRACE=1 cargo run --bin print 20 | RUST_BACKTRACE=1 cargo run --no-default-features --bin print 21 | -------------------------------------------------------------------------------- /mmap-alloc/appveyor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | if [ "$RUST_NIGHTLY" != "1" ]; then 14 | exit 0 15 | fi 16 | 17 | export RUST_TEST_THREADS=1 18 | 19 | cargo build 20 | for feature in '' large-align; do 21 | RUST_BACKTRACE=1 cargo test --features "$feature" 22 | done 23 | -------------------------------------------------------------------------------- /alloc-fmt/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | travis-cargo --only nightly build 14 | RUST_BACKTRACE=1 travis-cargo --only nightly test 15 | RUST_BACKTRACE=1 travis-cargo --only nightly run -- --bin print 16 | RUST_BACKTRACE=1 travis-cargo --only nightly run -- --no-default-features --bin print 17 | -------------------------------------------------------------------------------- /git-hooks/install-hooks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | # source: http://stackoverflow.com/a/957978/836390 11 | ROOT=$(git rev-parse --show-toplevel) || exit 1 12 | 13 | if [ "$ROOT" == "" ]; then 14 | echo "`git rev-parse --show-toplevel` returned empty root path" >&2 15 | exit 1 16 | fi 17 | 18 | cd $ROOT/.git/hooks || exit 1 19 | 20 | ln -s ../../git-hooks/pre-commit.sh pre-commit || exit 1 21 | -------------------------------------------------------------------------------- /elfmalloc/appveyor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | if [ "$RUST_NIGHTLY" != "1" ]; then 14 | exit 0 15 | fi 16 | 17 | # Skip elfmalloc tests until we can get them working. 18 | exit 0 19 | 20 | cargo build 21 | RUST_BACKTRACE=1 cargo test 22 | for feature in prime_schedules huge_segments no_lazy_region nightly; do 23 | RUST_BACKTRACE=1 cargo test --features "$feature" 24 | done 25 | -------------------------------------------------------------------------------- /elfmalloc/src/alloc_type.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | //! Global tags to indicate the allocation subsystem from which a pointer originates. 9 | //! 10 | //! The `AllocType` enum is used to mark pages in elfmalloc; see the `general` and `slag` modules 11 | //! for more on how this works. 12 | 13 | #[derive(Copy, Clone, Debug, PartialEq, Eq)] 14 | pub enum AllocType { 15 | SmallSlag, 16 | BigSlag, 17 | Large, 18 | } 19 | -------------------------------------------------------------------------------- /object-alloc-test/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Changelog 9 | 10 | All notable changes to this project will be documented in this file. 11 | 12 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 13 | 14 | ## [Unreleased] 15 | 16 | ### Added 17 | - Added this changelog 18 | 19 | ### Changed 20 | - Upgraded to new `ObjectAlloc` trait that uses `NonNull` instead 21 | of `*mut u8` 22 | -------------------------------------------------------------------------------- /alloc-fmt/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Changelog 9 | 10 | All notable changes to this project will be documented in this file. 11 | 12 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 13 | 14 | ## [Unreleased] 15 | 16 | ### Added 17 | - Added `alloc_panic` macro 18 | - Added `print-backtrace` feature 19 | - Added `AllocUnwrap` trait providing `alloc_unwrap` and `alloc_expect` 20 | methods with `impl`s for `Option` and `Result` 21 | -------------------------------------------------------------------------------- /object-alloc/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Changelog 9 | 10 | All notable changes to this project will be documented in this file. 11 | 12 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 13 | 14 | ## [Unreleased] 15 | 16 | ## 0.2.0 17 | 18 | ### Added 19 | - Added this changelog 20 | 21 | ### Changed 22 | - Switched from `*mut u8` to `NonNull` for pointer values 23 | - Switched from returning `Result`s to returning `Option`s 24 | -------------------------------------------------------------------------------- /object-alloc-test/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | object-alloc-test 9 | ================= 10 | 11 | [![Crates.io](https://img.shields.io/crates/v/object-alloc-test.svg)](https://crates.io/crates/object-alloc-test) 12 | [![Docs](https://docs.rs/object-alloc-test/badge.svg)](https://docs.rs/object-alloc-test) 13 | 14 | Tests for object allocators. This crate provides mechanisms to test implementations of the `ObjectAlloc` trait for correctness, including a comprehensive memory corruption tester. 15 | -------------------------------------------------------------------------------- /elfc/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Changelog 9 | 10 | All notable changes to this project will be documented in this file. 11 | 12 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 13 | 14 | ## [Unreleased] 15 | 16 | ### Added 17 | - Added this changelog 18 | 19 | ### Changed 20 | - Switched to using `malloc-bind` to provide C bindings 21 | 22 | ### Fixed 23 | - Fixed a bug caused by `sysconf` 0.3.0 that prevented compilation on Windows 24 | by upgrading to 0.3.1 25 | -------------------------------------------------------------------------------- /bsalloc/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Changelog 9 | 10 | All notable changes to this project will be documented in this file. 11 | 12 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 13 | 14 | ## [Unreleased] 15 | 16 | ### Added 17 | - Added this changelog 18 | 19 | ### Changed 20 | - Changed allocation routines to propagate mmap errors to the user instead of 21 | panicking 22 | 23 | ### Fixed 24 | - Fixed a Windows segfault bug by committing allocated memory on Windows 25 | -------------------------------------------------------------------------------- /alloc-fmt/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | alloc-fmt 9 | ========= 10 | 11 | `alloc-fmt` provides formatting and assertion macros similar to `println`, 12 | `eprintln`, `panic`, `assert`, `debug_assert`, etc which are safe for use in a 13 | global allocator. The standard library's formatting, panic, and assertion macros 14 | can allocate, meaning that if they are used in the implementation of a global 15 | allocator, it can cause infinite recursion. The macros in this crate do not 16 | allocate in order to avoid this problem. 17 | -------------------------------------------------------------------------------- /object-alloc/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | [package] 9 | name = "object-alloc" 10 | version = "0.2.0" 11 | authors = ["Joshua Liebow-Feeser "] 12 | license = "Apache-2.0/MIT" 13 | description = "An object allocator trait for Rust." 14 | 15 | keywords = ["allocator", "object", "caching"] 16 | categories = ["algorithms", "caching", "memory-management"] 17 | 18 | readme = "README.md" 19 | documentation = "https://docs.rs/object-alloc" 20 | repository = "https://github.com/ezrosent/allocators-rs/tree/master/object-alloc" 21 | 22 | exclude = ["appveyor.sh", "travis.sh"] 23 | -------------------------------------------------------------------------------- /mmap-alloc/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | mmap-alloc 9 | ========== 10 | 11 | [![Crates.io](https://img.shields.io/crates/v/mmap-alloc.svg)](https://crates.io/crates/mmap-alloc) 12 | [![Docs](https://docs.rs/mmap-alloc/badge.svg)](https://docs.rs/mmap-alloc) 13 | 14 | An allocator that is backed by directly mapping memory pages. 15 | 16 | The `MapAlloc` type defined by this crate implements the `Alloc` and `ObjectAlloc` traits by directly mapping memory pages from the kernel (`mmap`/`munmap` on POSIX systems and `VirtualAlloc`/`VirtualFree` on Windows). It also allows pages to be committed or uncommitted. 17 | -------------------------------------------------------------------------------- /git-hooks/pre-commit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | echo "Executing pre-commit hooks..." 11 | 12 | # source: http://stackoverflow.com/a/957978/836390 13 | ROOT="$(git rev-parse --show-toplevel)" || exit 1 14 | 15 | if [ "$ROOT" == "" ]; then 16 | echo "`git rev-parse --show-toplevel` returned empty root path" >&2 17 | exit 1 18 | fi 19 | 20 | cd "$ROOT" 21 | 22 | function die { 23 | if [ $# -eq 1 ]; then 24 | rm "$1" 25 | fi 26 | echo "commit aborted" >&1 27 | exit 1 28 | } 29 | 30 | echo " Running ./test-scripts/check-copyright-comments.sh..." 31 | ./test-scripts/check-copyright-comments.sh 32 | -------------------------------------------------------------------------------- /mmap-alloc/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | export RUST_TEST_THREADS=1 14 | 15 | travis-cargo --only nightly build 16 | # TODO: Figure out why test_map_panic_too_large results in SIGBUS 17 | # (e.g., see https://travis-ci.org/ezrosent/allocators-rs/jobs/291713981) 18 | # TODO: Remove -q and --verbose once the following issue is fixed: 19 | # https://github.com/huonw/travis-cargo/issues/75 20 | for feature in '' large-align; do 21 | RUST_BACKTRACE=1 travis-cargo -q --only nightly test -- --features "$feature" \ 22 | --verbose -- --skip test_map_panic_too_large 23 | done 24 | -------------------------------------------------------------------------------- /alloc-tls/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | [package] 9 | name = "alloc-tls" 10 | version = "0.2.0" 11 | authors = ["Joshua Liebow-Feeser "] 12 | license = "Apache-2.0/MIT" 13 | description = "Thread-local storage that is safe for use in implementing global allocators." 14 | 15 | keywords = ["thread-local", "allocator", "tls"] 16 | categories = ["memory-management", "os"] 17 | 18 | readme = "README.md" 19 | documentation = "https://docs.rs/alloc-tls" 20 | repository = "https://github.com/ezrosent/allocators-rs/tree/master/alloc-tls" 21 | 22 | [features] 23 | dylib = [] 24 | 25 | [dependencies] 26 | alloc-fmt = "0.2.0" 27 | -------------------------------------------------------------------------------- /bagpipe/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Changelog 9 | 10 | All notable changes to this project will be documented in this file. 11 | 12 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 13 | 14 | ## [Unreleased] 15 | 16 | ### Added 17 | - Added this changelog 18 | - Bagpipes can now call drop on their elements when they are dropped. This does 19 | not happen automatically, but there is a new trait to inject cleanup callbacks 20 | to `BagPipe` shutdown. 21 | 22 | ### Fixed 23 | - Fixed a bug where crossbeam TLS would remain uninitialized upon cloning a 24 | `BagPipe`, resulting in stack overflow in `elfmalloc`. 25 | -------------------------------------------------------------------------------- /malloc-bind/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | malloc-bind 9 | =========== 10 | 11 | [![Crates.io](https://img.shields.io/crates/v/malloc-bind.svg)](https://crates.io/crates/malloc-bind) 12 | [![Docs](https://docs.rs/malloc-bind/badge.svg)](https://docs.rs/malloc-bind) 13 | 14 | The `malloc-bind` crate provides bindings for the C `malloc` API. Given an implementation of the Rust `Alloc` trait, it produces implementations of the C `malloc` API (`malloc`, `free`, `realloc`, etc) and defines `extern "C"` functions that can be used to produce a C shared object file. 15 | 16 | ## Platform support 17 | 18 | `malloc-bind` currently supports Linux, Mac, and Windows. 19 | -------------------------------------------------------------------------------- /slab-alloc/appveyor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | if [ "$RUST_NIGHTLY" != "1" ]; then 14 | exit 0 15 | fi 16 | 17 | cargo build 18 | # Use opt-level=3 because the corruption tests take a long time, and the 19 | # optimized build saves more time in test execution than it adds in compilation 20 | # time. 21 | RUSTFLAGS='-C opt-level=3' RUST_BACKTRACE=1 cargo test 22 | # TODO: Once no-std and no-os work, add those features. 23 | for feature in build-ignored-tests use-stdlib-hashmap no-coloring hashmap-no-resize hashmap-no-coalesce; do 24 | RUSTFLAGS='-C opt-level=3' RUST_BACKTRACE=1 cargo test --features "$feature" 25 | done 26 | -------------------------------------------------------------------------------- /elfc/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | elfc 9 | ==== 10 | 11 | This crate uses [`malloc-bind`](https://crates.io/crates/malloc-bind) to provide 12 | an implementation of the C allocation API (`malloc`, `free`, etc) based on the 13 | [`elfmalloc`](https://crates.io/crates/elfmalloc) allocator. It compiles to a 14 | dynamic library (`.so` on Linux and `.dylib` on Mac) that can be loaded using 15 | the `LD_PRELOAD` or `DYLD_INSERT_LIBRARIES` environment variables (on Linux or 16 | Mac respectively) or using `dlopen`. 17 | 18 | elfmalloc is still in early alpha, and some platforms are only minimally 19 | supported. For details on what's working and what isn't, see the elfmalloc 20 | README. 21 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any 2 | person obtaining a copy of this software and associated 3 | documentation files (the "Software"), to deal in the 4 | Software without restriction, including without 5 | limitation the rights to use, copy, modify, merge, 6 | publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software 8 | is furnished to do so, subject to the following 9 | conditions: 10 | 11 | The above copyright notice and this permission notice 12 | shall be included in all copies or substantial portions 13 | of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 16 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 17 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 18 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 19 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 22 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /elfmalloc/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | # Since elfmalloc tests use a lot of memory, we use RUST_TEST_THREADS=1 14 | # to force them to run sequentially in order to reduce the total memory 15 | # consumption and avoid a crash on Travis. 16 | 17 | travis-cargo --only nightly build 18 | RUST_TEST_THREADS=1 RUST_BACKTRACE=1 travis-cargo --only nightly test \ 19 | -- --features low-memory-tests 20 | for feature in \ 21 | prime_schedules \ 22 | huge_segments \ 23 | no_lazy_region \ 24 | local_cache \ 25 | magazine_layer \ 26 | c-api; do 27 | RUST_TEST_THREADS=1 RUST_BACKTRACE=1 travis-cargo --only nightly test \ 28 | -- --features "$feature low-memory-tests" 29 | done 30 | -------------------------------------------------------------------------------- /bsalloc/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | [package] 9 | name = "bsalloc" 10 | version = "0.1.0" 11 | authors = ["Eli Rosenthal "] 12 | license = "Apache-2.0/MIT" 13 | description = "A simple general-purpose allocator used to bootstrap other allocators." 14 | 15 | keywords = ["bootstrap", "allocator"] 16 | categories = ["concurrency", "memory-management", "no-std"] 17 | 18 | readme = "README.md" 19 | documentation = "https://docs.rs/bsalloc" 20 | repository = "https://github.com/ezrosent/allocators-rs/tree/master/bsalloc" 21 | 22 | exclude = ["appveyor.sh", "travis.sh"] 23 | 24 | [dependencies] 25 | alloc-fmt = { path = "../alloc-fmt" } 26 | lazy_static = { version = "1.0.0", features = ["spin_no_std"] } 27 | mmap-alloc = { path = "../mmap-alloc" } 28 | -------------------------------------------------------------------------------- /elfc/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | [package] 9 | name = "elfc" 10 | version = "0.1.0" 11 | authors = ["Eli Rosenthal "] 12 | license = "Apache-2.0/MIT" 13 | 14 | publish = false 15 | exclude = ["appveyor.sh", "travis.sh"] 16 | 17 | [profile.dev] 18 | panic = "abort" 19 | [profile.release] 20 | panic = "abort" 21 | 22 | [lib] 23 | name = "elfc" 24 | crate-type = ["cdylib"] 25 | 26 | [features] 27 | local_cache = ["elfmalloc/local_cache"] 28 | magazine_layer = ["elfmalloc/magazine_layer"] 29 | logging = ["elfmalloc/print_stats"] 30 | 31 | [dependencies] 32 | alloc-tls = { path = "../alloc-tls", features = ["dylib"] } 33 | elfmalloc = { path = "../elfmalloc", features = ["c-api"] } 34 | malloc-bind = { path = "../malloc-bind" } 35 | env_logger = "0.4.3" 36 | -------------------------------------------------------------------------------- /malloc-bind/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | [package] 9 | name = "malloc-bind" 10 | version = "0.1.0" 11 | authors = ["Joshua Liebow-Feeser "] 12 | license = "Apache-2.0/MIT" 13 | description = "A tool to implement the C malloc API using a Rust allocator." 14 | 15 | keywords = ["allocator", "malloc", "bindings", "C", "ffi"] 16 | categories = ["external-ffi-bindings", "memory-management", "no-std"] 17 | 18 | readme = "README.md" 19 | documentation = "https://docs.rs/malloc-bind" 20 | repository = "https://github.com/ezrosent/allocators-rs/tree/master/malloc-bind" 21 | 22 | exclude = ["appveyor.sh", "travis.sh"] 23 | 24 | [dependencies] 25 | errno = "0.2" 26 | lazy_static = { version = "1.0.0", features = ["spin_no_std"] } 27 | libc = "0.2" 28 | sysconf = ">= 0.3.3" 29 | -------------------------------------------------------------------------------- /slab-alloc/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | set -x 11 | set -e 12 | 13 | travis-cargo --only nightly build 14 | # Use opt-level=3 because the corruption tests take a long time, and the 15 | # optimized build saves more time in test execution than it adds in compilation 16 | # time. 17 | # TODO: Now that we have opt-level = 3 in the test profile in Cargo.toml, 18 | # is this necessary anymore? 19 | RUSTFLAGS='-C opt-level=3' RUST_BACKTRACE=1 travis-cargo --only nightly test 20 | # TODO: Once no-std and no-os work, add those features. 21 | for feature in build-ignored-tests use-stdlib-hashmap no-coloring hashmap-no-resize hashmap-no-coalesce; do 22 | RUSTFLAGS='-C opt-level=3' RUST_BACKTRACE=1 travis-cargo --only nightly test -- \ 23 | --features "$feature" 24 | done 25 | -------------------------------------------------------------------------------- /alloc-fmt/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | [package] 9 | name = "alloc-fmt" 10 | version = "0.2.0" 11 | authors = ["Joshua Liebow-Feeser "] 12 | license = "Apache-2.0/MIT" 13 | description = "Formatting utilities safe for use in an allocator." 14 | 15 | keywords = ["format", "print", "assert", "panic", "allocator"] 16 | categories = ["memory-management", "no-std"] 17 | 18 | readme = "README.md" 19 | documentation = "https://docs.rs/alloc-fmt" 20 | repository = "https://github.com/ezrosent/allocators-rs/tree/master/alloc-fmt" 21 | 22 | exclude = ["appveyor.sh", "travis.sh"] 23 | 24 | [features] 25 | default = ["print-backtrace"] 26 | 27 | print-backtrace = ["backtrace"] 28 | 29 | [dependencies] 30 | backtrace = { version = "0.3.3", optional = true } 31 | libc = "0.2" 32 | spin = "0.4.6" 33 | -------------------------------------------------------------------------------- /slab-alloc/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Changelog 9 | 10 | All notable changes to this project will be documented in this file. 11 | 12 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 13 | 14 | ## [Unreleased] 15 | 16 | ### Added 17 | - Added this changelog 18 | 19 | ### Changed 20 | - Upgraded to new `ObjectAlloc` and `UntypedObjectAlloc` traits that use 21 | `Option>` instead of `Result<*mut T, Exhausted>` 22 | - Switch from `*mut T` to `NonNull` or `Option>` for various 23 | internal pointers 24 | - Make `PtrHashMap` support any value type that is `Copy`, not just raw 25 | pointers 26 | 27 | ### Fixed 28 | - Fixed a bug that prevented compilation on 32-bit Windows 29 | - Fixed a bug caused by `sysconf` 0.3.0 that prevented compilation on Windows 30 | by upgrading to 0.3.1 31 | -------------------------------------------------------------------------------- /bagpipe/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | [package] 9 | name = "bagpipe" 10 | version = "0.1.0" 11 | authors = ["Eli Rosenthal "] 12 | license = "Apache-2.0/MIT" 13 | description = "A concurrent bag datastructure." 14 | 15 | keywords = ["bagpipe", "bag", "concurrent", "queue", "set"] 16 | categories = ["algorithms", "concurrency", "data-structures"] 17 | 18 | readme = "README.md" 19 | documentation = "https://docs.rs/bagpipe" 20 | repository = "https://github.com/ezrosent/allocators-rs/tree/master/bagpipe" 21 | 22 | exclude = ["appveyor.sh", "travis.sh"] 23 | 24 | [features] 25 | # TODO: Rename these features to use dashes instead of underscores 26 | default = ["check_empty_yq"] 27 | check_empty_yq = [] 28 | prime_schedules = [] 29 | staggered_indexes = [] 30 | huge_segments = [] 31 | 32 | [dependencies] 33 | crossbeam = "0.2" 34 | num_cpus = "1.5" 35 | -------------------------------------------------------------------------------- /object-alloc-test/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | [package] 9 | name = "object-alloc-test" 10 | version = "0.1.0" 11 | authors = ["Joshua Liebow-Feeser "] 12 | license = "Apache-2.0/MIT" 13 | description = "A suite of tests for object allocators." 14 | 15 | keywords = ["allocator", "object", "test", "memory", "corruption"] 16 | categories = ["caching", "memory-management"] 17 | 18 | readme = "README.md" 19 | documentation = "https://docs.rs/object-alloc-test" 20 | repository = "https://github.com/ezrosent/allocators-rs/tree/master/object-alloc-test" 21 | 22 | exclude = ["appveyor.sh", "travis.sh"] 23 | 24 | [dependencies] 25 | kernel32-sys = "0.2" 26 | lazy_static = "1.0.0" 27 | libc = "0.2" 28 | object-alloc = { path = "../object-alloc" } 29 | paste = "0.1" 30 | quickcheck = "0.4" 31 | rand = "0.3" 32 | twox-hash = "1.1" 33 | 34 | [dependencies.winapi] 35 | version = "0.3" 36 | features = ["basetsd", "psapi"] 37 | -------------------------------------------------------------------------------- /object-alloc/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | object-alloc 9 | ============ 10 | 11 | [![Crates.io](https://img.shields.io/crates/v/object-alloc.svg)](https://crates.io/crates/object-alloc) 12 | [![Docs](https://docs.rs/object-alloc/badge.svg)](https://docs.rs/object-alloc) 13 | 14 | Object allocators in Rust. This crate defines the `ObjectAlloc` trait and related types. An object allocator is an allocator which allocates and caches objects of a particular type, allowing for a number of performance improvements over general-purpose allocators that need to be able to service allocation requests of any size and alignment. Since all objects are of the same type, an object allocator can cache freed objects in a constructed state, and can thus allocate by re-using these cached objects. This allows object construction to be elided in certain circumstances, which can provide a further performance improvement. 15 | 16 | This crate only defines types - it does not contain any implementations of the `ObjectAlloc` trait. 17 | -------------------------------------------------------------------------------- /mmap-alloc/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | [package] 9 | name = "mmap-alloc" 10 | version = "0.2.0" 11 | authors = ["Joshua Liebow-Feeser "] 12 | license = "Apache-2.0/MIT" 13 | description = "A simple allocator backed by memory mappings." 14 | 15 | keywords = ["allocator", "mmap", "virtualalloc", "memory"] 16 | categories = ["memory-management", "no-std"] 17 | 18 | readme = "README.md" 19 | documentation = "https://docs.rs/mmap-alloc" 20 | repository = "https://github.com/ezrosent/allocators-rs/tree/master/mmap-alloc" 21 | 22 | exclude = ["appveyor.sh", "travis.sh"] 23 | 24 | [features] 25 | # Support alignments larger than a page size by mapping a larger region 26 | # and then unmapping all but a properly-aligned subset of the region. 27 | large-align = [] 28 | 29 | [dependencies] 30 | errno = "0.2" 31 | kernel32-sys = "0.2" 32 | # use no_std libc 33 | libc = { version = "0.2", default-features = false } 34 | object-alloc = { path = "../object-alloc" } 35 | sysconf = ">= 0.3.3" 36 | winapi = "0.2" 37 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | language: rust 9 | cache: cargo 10 | rust: 11 | - nightly 12 | - beta 13 | - stable 14 | os: 15 | - linux 16 | - osx 17 | # Load travis-cargo 18 | before_script: 19 | - | 20 | pip2 install 'travis-cargo<0.2' --user && 21 | export PATH=`python -m site --user-base`/bin:$PATH 22 | script: 23 | - | 24 | # Disable rm override of cd, pushd, and popd; 25 | # see https://github.com/travis-ci/travis-ci/issues/8703 26 | if [ "$TRAVIS_OS_NAME" == "osx" ]; then 27 | unset -f cd; 28 | unset -f pushd; 29 | unset -f popd; 30 | fi 31 | set -x; set -e; for script in */travis.sh; do 32 | dir=$(dirname "$script"); 33 | file=$(basename "$script"); 34 | pushd "$dir"; 35 | ./$file; 36 | popd; 37 | done 38 | - ./test-scripts/check-copyright-comments.sh 39 | 40 | env: 41 | global: 42 | # Override the default `--features unstable` used for the nightly branch 43 | - TRAVIS_CARGO_NIGHTLY_FEATURE="" 44 | -------------------------------------------------------------------------------- /bsalloc/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | bsalloc 9 | ======= 10 | 11 | [![Crates.io](https://img.shields.io/crates/v/bsalloc.svg)](https://crates.io/crates/bsalloc) 12 | [![Docs](https://docs.rs/bsalloc/badge.svg)](https://docs.rs/bsalloc) 13 | 14 | This crate implements a very simple global allocator. This is used to 15 | service heap allocations for dependencies of the dynamic allocators in 16 | the rest of the repo. In other words, this is used to **b**oot**s**trap 17 | the more efficient allocators in this repo without creating a needless 18 | dependency on another malloc implementation. 19 | 20 | ## Structure 21 | 22 | The allocator is a thin wrapper around `mmap`. Instead of simply calling 23 | `mmap` and `munmap` for dynamic allocations directly (this did slow 24 | things down in some microbenchmarks), we have two "size classes", 25 | each with a fixed-size global cache. These caches have a thread-safe 26 | interface that allows a thread to efficiently re-use memory freed by it or 27 | another thread. Naturally, allocations that are larger than the largest 28 | size class fall back on `mmap` directly. 29 | -------------------------------------------------------------------------------- /bagpipe/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | bagpipe 9 | ======= 10 | 11 | [![Crates.io](https://img.shields.io/crates/v/bagpipe.svg)](https://crates.io/crates/bagpipe) 12 | [![Docs](https://docs.rs/bagpipe/badge.svg)](https://docs.rs/bagpipe) 13 | 14 | A `Bagpipe` is an implementation of a concurrent pool data-structure. It 15 | is built on top of an implementation of a concurrent queue or stack 16 | data-structure. It shards requests for pushing and popping objects 17 | among a small set of queues or stacks, with load balancing performed in 18 | a manner that is low in coordination among threads. Contention is automatically 19 | managed by unrolling tight loops within the underlying queues and allowing 20 | threads to attempt pushing on another queue if a CAS failure occurs. 21 | 22 | ## Other Features 23 | 24 | This repo also includes implementations of concurrent queues such as 25 | the `FAAArrayQueue` and the `YangCrummeyQueue`. These are linearizable 26 | non-blocking multi-producer multi-consumer queues that may be of 27 | independent interest, as they scale better than the queues present in 28 | the `crossbeam` library (to my knowledge). 29 | -------------------------------------------------------------------------------- /object-alloc-test/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | #![feature(alloc)] 9 | #![feature(allocator_api)] 10 | #![feature(test)] 11 | 12 | extern crate alloc; 13 | #[cfg(unix)] 14 | #[macro_use] 15 | extern crate lazy_static; 16 | extern crate paste; 17 | extern crate object_alloc; 18 | 19 | pub mod corruption; 20 | pub mod leaky_alloc; 21 | pub mod types; 22 | 23 | /// Call a function once for each alignment. 24 | /// 25 | /// `foreach_align` calls `f` once for each valid alignment of `T` not greater than `max`. If `max` 26 | /// is greater than any valid alignment of `T`, `f` is called for each valid alignment of `T`, and 27 | /// `max` is ignored. 28 | /// 29 | /// `foreach_align` is useful for testing allocators whose behavior may be sensitive to requested 30 | /// alignment. 31 | pub fn foreach_align(f: F, max: usize) { 32 | use std::mem::{align_of, size_of}; 33 | 34 | let min_align = align_of::(); 35 | let size = size_of::(); 36 | let mut align = min_align; 37 | while align <= size && align <= max { 38 | if size % align == 0 { 39 | f(align); 40 | } 41 | align *= 2; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /slab-alloc/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | slab-alloc 9 | ========== 10 | 11 | [![Crates.io](https://img.shields.io/crates/v/slab-alloc.svg)](https://crates.io/crates/slab-alloc) 12 | [![Docs](https://docs.rs/slab-alloc/badge.svg)](https://docs.rs/slab-alloc) 13 | 14 | The slab allocator. This crate implements an allocator whose design is based on Jeff Bonwick's [The Slab Allocator: An Object-Caching Kernel Memory Allocator](http://www.usenix.org/publications/library/proceedings/bos94/full_papers/bonwick.ps). 15 | 16 | The slab allocator is an object allocator - it allocates and caches objects of a fixed type, and provides performance improvements over a general-purpose allocator. The allocator types in this crate implement the `ObjectAlloc` and `UntypedObjectAlloc` traits defined in the object-alloc crate. The slab allocator implemented in this crate is currently single-threaded and cannot be accessed concurrently. 17 | 18 | Like many object allocators, slab allocators cache objects in their constructed state, allowing expensive constructor calls to be elided in some circumstances. 19 | 20 | The slab allocator implemented in this crate is not currently `Sync` - it can only be used by a single thread at a time. 21 | -------------------------------------------------------------------------------- /alloc-tls/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | alloc-tls 9 | ========= 10 | 11 | `alloc-tls` provides the `alloc_thread_local!` macro, a near-drop-in replacement 12 | for the standard library's `thread_local!` macro that is safe for use in 13 | implementing a global allocator. 14 | 15 | Unlike `thread_local!`, `alloc_thread_local!` address the following issues 16 | unique to implementing a global allocator: 17 | - On platforms that support the `#[thread_local]` attribute, registering 18 | destructors for types that implement `Drop` requires allocation. When a 19 | thread-local is initialized from a call to an allocation function (`malloc`, 20 | `free`, etc), this causes reentrancy. `alloc_thread_local!` can detect this. 21 | - On Mac, it is not safe to access TLS while a dynamic library is being loaded. 22 | When implementing a Mac dynamic library that provides a global allocator, 23 | `alloc_thread_local!` can detect whether the library has been loaded or not, 24 | and can avoid using TLS if `malloc` or other similar calls are made from the 25 | loader itself during loading. 26 | 27 | Known limitations: 28 | - `alloc-tls` does not currently support platforms that do not support the 29 | `#[thread_local]` attribute. 30 | -------------------------------------------------------------------------------- /elfmalloc/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | #![feature(alloc)] 9 | #![feature(allocator_api)] 10 | #![cfg_attr(test, feature(test))] 11 | #![feature(thread_local_state)] 12 | #![feature(thread_local)] 13 | #![feature(const_fn)] 14 | #![feature(const_size_of)] 15 | #![feature(cfg_target_thread_local)] 16 | #![feature(core_intrinsics)] 17 | #![feature(const_ptr_null_mut)] 18 | extern crate alloc; 19 | extern crate bagpipe; 20 | extern crate mmap_alloc; 21 | extern crate num_cpus; 22 | extern crate sysconf; 23 | 24 | #[macro_use] 25 | extern crate alloc_fmt; 26 | // Linking in `bsalloc` causes it to be used as the global heap allocator. That is important when 27 | // using this as a basis for a `malloc` library, but it becomes a hindrance when using this crate 28 | // as a specialized allocator library. 29 | #[cfg(not(feature = "use_default_allocator"))] 30 | extern crate bsalloc; 31 | #[macro_use] 32 | extern crate lazy_static; 33 | #[macro_use] 34 | extern crate log; 35 | #[macro_use] 36 | extern crate alloc_tls; 37 | 38 | mod sources; 39 | mod alloc_type; 40 | mod utils; 41 | #[macro_use] 42 | mod stats; 43 | mod slag; 44 | pub mod frontends; 45 | pub mod general; 46 | 47 | pub mod alloc_impl; 48 | pub mod rust_alloc; 49 | pub mod vec_alloc; 50 | -------------------------------------------------------------------------------- /elfc/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | #![cfg_attr(any(feature = "logging", target_os = "macos"), feature(link_args))] 9 | #![cfg_attr(all(feature = "logging", target_os = "linux"), link_args = "-Wl,-init,init_log")] 10 | // On Mac, the C ABI prefixes all symbols with _. 11 | // Source: https://users.rust-lang.org/t/ld-preload-init-function-in-rust/12865/6 12 | // TODO: Consider switching to using the .mod_init_funcs (e.g., 13 | // #[link_secction = ".mod_init_funcs"]) as recommended here: 14 | // https://community.embarcadero.com/blogs/entry/mac-os-x-shared-library-initialization-5639 15 | #![cfg_attr(all(feature = "logging", target_os = "macos"), link_args = "-Wl,-init,_init")] 16 | #![cfg_attr(all(not(feature = "logging"), target_os = "macos"), link_args = "-Wl,-init,_dyld_init")] 17 | 18 | #[cfg(feature = "logging")] 19 | extern crate alloc_tls; 20 | extern crate elfmalloc; 21 | #[cfg(feature = "logging")] 22 | extern crate env_logger; 23 | #[macro_use] 24 | extern crate malloc_bind; 25 | use elfmalloc::alloc_impl::ElfMallocGlobal; 26 | 27 | define_malloc!(ElfMallocGlobal, ElfMallocGlobal); 28 | 29 | #[cfg(feature = "logging")] 30 | #[no_mangle] 31 | pub extern "C" fn init() { 32 | alloc_tls::dyld_init(); 33 | let _ = env_logger::init(); 34 | } 35 | -------------------------------------------------------------------------------- /elfmalloc/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Changelog 9 | 10 | All notable changes to this project will be documented in this file. 11 | 12 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 13 | 14 | ## [Unreleased] 15 | 16 | ### Added 17 | - Added this changelog 18 | - Implemented `Alloc` trait 19 | - Implemented `malloc-bind`'s `LayoutFinder` and `Malloc` traits 20 | - Added new metadata management mechanism that eliminates dependency on the 21 | `Creek` and overcommit more generally 22 | - Substantial refactoring of the code into separate traits/modules 23 | - Added experimental magazine/depot-style frontend. Currently still 24 | experimental; it is almost always slower than `LocalCache` and 25 | `MagazineCache` 26 | - Added `c-api` feature to optimize for the C malloc API 27 | 28 | ### Fixed 29 | - Fixed a bug preventing non-nightly builds from compiling 30 | - Fixed an integer multiplication overflow bug 31 | - Added workaround to avoid double-drop behavior in certain `malloc` workloads 32 | - Fixed "recursive `malloc`" bug caused by failing to initialize the `crossbeam` 33 | TLS early enough 34 | - Fixed recursive allocation bugs during printing, asserting, and panicking by 35 | switching to the macros from the `alloc-fmt` crate 36 | - Fixed recursive allocation bugs with TLS by switching to the `alloc-tls` crate 37 | -------------------------------------------------------------------------------- /slab-alloc/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | [package] 9 | name = "slab-alloc" 10 | version = "0.1.1" 11 | authors = ["Joshua Liebow-Feeser ", "Eli Rosenthal "] 12 | license = "Apache-2.0/MIT" 13 | description = "A fast, single-threaded object allocator." 14 | 15 | keywords = ["allocator", "cache", "object", "slab"] 16 | categories = ["algorithms", "caching", "memory-management", "no-std"] 17 | 18 | readme = "README.md" 19 | documentation = "https://docs.rs/slab-alloc" 20 | repository = "https://github.com/ezrosent/allocators-rs/tree/master/slab-alloc" 21 | 22 | exclude = ["appveyor.sh", "travis.sh"] 23 | 24 | [profile.test] 25 | opt-level = 3 26 | 27 | [features] 28 | # NOTE: no-std and no-os are not currently supported; see 29 | # https://github.com/ezrosent/allocators-rs/issues/2 for 30 | # progress towards no-std and no-os. 31 | default = ["std", "os"] 32 | std = ["os"] 33 | os = [] 34 | 35 | build-ignored-tests = [] 36 | use-stdlib-hashmap = [] 37 | no-coloring = [] 38 | hashmap-no-resize = [] 39 | hashmap-no-coalesce = [] 40 | 41 | [dependencies] 42 | lazy_static = { version = "1.0.0", features = ["spin_no_std"] } 43 | mmap-alloc = { path = "../mmap-alloc" } 44 | object-alloc = { path = "../object-alloc" } 45 | object-alloc-test = { path = "../object-alloc-test" } 46 | rand = "0.3" 47 | sysconf = ">= 0.3.3" 48 | -------------------------------------------------------------------------------- /elfmalloc/README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | elfmalloc 9 | ========= 10 | 11 | _Efficient dynamic memory allocation_ 12 | 13 | [![Crates.io](https://img.shields.io/crates/v/elfmalloc.svg)](https://crates.io/crates/elfmalloc) 14 | [![Docs](https://docs.rs/elfmalloc/badge.svg)](https://docs.rs/elfmalloc) 15 | 16 | This crate provides efficient multi-threaded heap allocation both on a 17 | per-object (i.e. fixed-size) or dynamic (i.e. `malloc`-like) basis. 18 | Most details are provided in the crate documentation. 19 | 20 | Note, if you link in this crate to a Rust project (e.g. to use object-specific 21 | allocators), you will want to set the `use_default_allocator` feature. Without 22 | this feature, all existing dynamic allocation requests from the rest of the 23 | project will be slower. 24 | 25 | To compile a dynamic library that can be loaded in existing C programs, use the 26 | [`elfc`](https://github.com/ezrosent/allocators-rs/tree/master/elfc) crate. 27 | 28 | ## Support 29 | 30 | elfmalloc is currently in early alpha. Support on 64-bit Linux is relatively 31 | complete, although there are certainly undiscovered bugs. 32-bit support is 32 | theoretically sound, but completely untested. We are working towards 64-bit Mac 33 | support, but even simple programs will currently encounter bugs or crashes on 34 | Mac. 35 | 36 | ## More Info 37 | 38 | - [Performance](https://github.com/ezrosent/allocators-rs/blob/master/info/elfmalloc-performance.md) 39 | - [Design](https://github.com/ezrosent/allocators-rs/blob/master/info/elfmalloc.md) 40 | -------------------------------------------------------------------------------- /malloc-bind/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Changelog 9 | 10 | All notable changes to this project will be documented in this file. 11 | 12 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 13 | 14 | ## [Unreleased] 15 | 16 | ### Added 17 | - Added this changelog 18 | - Added `_aligned_malloc` on Windows 19 | - Added documentation on the behavior of each of the functions in the C 20 | allocation API 21 | 22 | ### Changed 23 | - Put some `unlikely` in `if`s 24 | - Made `cfree` only compile on Linux 25 | - Made `posix_memalign` and `valloc` only compile on Linux and Mac 26 | - Made `LayoutFinder` and its methods unsafe 27 | - Made `Malloc` a trait rather than a struct, allowing methods to be overridden 28 | by implementors 29 | 30 | ### Removed 31 | - Removed the `reallocarray` function, as it was spuriously marked as being a 32 | Linux extension (it is actually an OpenBSD extension, and we don't currently 33 | support OpenBSD) 34 | 35 | ### Fixed 36 | - Fixed a bug caused by `sysconf` 0.3.0 that prevented compilation on Windows 37 | by upgrading to 0.3.1 38 | - Made it so that the correct lower bound on alignment is used on Windows and 39 | Mac 40 | - Set `errno` in places where it should have been set before 41 | - Made it so that 0-sized allocations only return `NULL` on Linux (they 42 | previously returned `NULL` on Mac and Windows as well, which is incorrect) 43 | - Check that `alignment` is a power of two in `aligned_alloc` 44 | - Changed macros to use absolute paths to types to avoid possible issues with 45 | scoping and imports 46 | -------------------------------------------------------------------------------- /bsalloc/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of 2 | // the README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | #![no_std] 9 | #![feature(allocator_api)] 10 | #![feature(alloc)] 11 | 12 | extern crate alloc; 13 | #[macro_use] 14 | extern crate alloc_fmt; 15 | #[macro_use] 16 | extern crate lazy_static; 17 | extern crate mmap_alloc; 18 | 19 | mod bsalloc; 20 | 21 | use alloc::alloc::{Alloc, GlobalAlloc, Layout}; 22 | use core::{cmp, ptr}; 23 | 24 | use bsalloc::{BsAlloc, ALLOC}; 25 | 26 | #[cfg(debug_assertions)] 27 | fn assert_nonoverlapping(r1: (usize, usize), r2: (usize, usize)) { 28 | let (_, b) = cmp::min(r1, r2); 29 | let (c, _) = cmp::max(r1, r2); 30 | alloc_assert!(c >= b); 31 | } 32 | 33 | #[cfg(not(test))] 34 | #[global_allocator] 35 | static GLOBAL: BsAlloc = BsAlloc; 36 | 37 | unsafe impl GlobalAlloc for BsAlloc { 38 | unsafe fn alloc(&self, l: Layout) -> *mut u8 { 39 | (*ALLOC).alloc(l.size()) 40 | } 41 | 42 | unsafe fn dealloc(&self, item: *mut u8, l: Layout) { 43 | (*ALLOC).free(item, l.size()) 44 | } 45 | 46 | unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { 47 | let old_size = layout.size(); 48 | // TODO(joshlf): To technically implement realloc correctly, we need to 49 | // preserve the alignment of the old allocation. To do this, we need to 50 | // round up new_size to a multiple of that alignment. 51 | let new_memory = self.alloc(Layout::from_size_align(new_size, 1).unwrap()); 52 | if new_memory.is_null() { 53 | return new_memory; 54 | } 55 | #[cfg(debug_assertions)] 56 | assert_nonoverlapping( 57 | (ptr as usize, ptr as usize + old_size), 58 | (new_memory as usize, new_memory as usize + new_size), 59 | ); 60 | ptr::copy_nonoverlapping(ptr, new_memory, cmp::min(old_size, new_size)); 61 | self.dealloc(ptr, layout); 62 | new_memory 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /elfmalloc/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | [package] 9 | name = "elfmalloc" 10 | version = "0.1.0" 11 | authors = ["Eli Rosenthal ", "Joshua Liebow-Feeser "] 12 | license = "Apache-2.0/MIT" 13 | description = "A fast, concurrent, general-purpose allocator." 14 | 15 | keywords = ["allocator"] 16 | categories = ["caching", "concurrency", "memory-management", "no-std"] 17 | 18 | readme = "README.md" 19 | documentation = "https://docs.rs/elfmalloc" 20 | repository = "https://github.com/ezrosent/allocators-rs/tree/master/elfmalloc" 21 | 22 | exclude = ["appveyor.sh", "travis.sh"] 23 | 24 | [[bin]] 25 | name = "bench_vec" 26 | path = "src/bin/bench_vec.rs" 27 | [[bin]] 28 | name = "bench" 29 | path = "src/bin/bench.rs" 30 | 31 | [features] 32 | default = [] 33 | # TODO: Rename these features to use dashes instead of underscores 34 | prime_schedules = ["bagpipe/prime_schedules"] 35 | huge_segments = ["bagpipe/huge_segments"] 36 | no_lazy_region = [] 37 | local_cache = [] 38 | use_default_allocator = [] 39 | print_stats = [] 40 | magazine_layer = [] 41 | # Limit the amount of memory used by tests to avoid running out. 42 | # This is used primarily in Travis. 43 | low-memory-tests = [] 44 | # Implement the C allocation API (malloc, free, etc) by implementing the 45 | # malloc-bind crate's Malloc trait. This feature will enable certain 46 | # optimizations that will make the C API faster but result in worse memory 47 | # usage for the Rust API (the Alloc trait). 48 | c-api = [] 49 | 50 | [dependencies] 51 | alloc-fmt = { path = "../alloc-fmt" } 52 | alloc-tls = { path = "../alloc-tls" } 53 | bagpipe = { path = "../bagpipe" } 54 | bsalloc = { path = "../bsalloc" } 55 | lazy_static = "1.0.0" 56 | libc = "0.2" 57 | log = "0.3.8" 58 | malloc-bind = { path = "../malloc-bind" } 59 | mmap-alloc = { path = "../mmap-alloc", features = ["large-align"] } 60 | num_cpus = "1.5" 61 | smallvec = "0.4.3" 62 | sysconf = ">= 0.3.3" 63 | 64 | [dev-dependencies] 65 | env_logger = "0.4.3" 66 | -------------------------------------------------------------------------------- /mmap-alloc/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Changelog 9 | 10 | All notable changes to this project will be documented in this file. 11 | 12 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). 13 | 14 | ## [Unreleased] 15 | 16 | ### Added 17 | - Added support for committing on Linux and Mac 18 | - Added support for configuring commit on allocation on Mac 19 | - Added support for alignments larger than a memory page (`large-align` 20 | feature) 21 | 22 | ### Changed 23 | - Upgraded to new `Alloc` trait that uses `Result, AllocErr>` 24 | instead of `Result<*mut u8, AllocErr>` 25 | - Upgraded to new `UntypedObjectAlloc` trait that uses `Option>` 26 | instead of `Result<*mut u8, Exhausted>` 27 | - Improved documentation on committed vs. uncommitted memory 28 | 29 | ## 0.2.0 30 | 31 | ### Added 32 | - Added this changelog 33 | - Added `commit` and `uncommit` methods on Windows 34 | - Added the ability to configure whether `alloc` commits memory 35 | - Added documentation about instruction cache incoherency 36 | - Added support for full Alloc API (`shrink_in_place`, `grow_in_place`, 37 | `realloc`) 38 | - In Linux, these functions can use `mremap` to grow/shrink beyond the size 39 | of a page 40 | - Added tests for memory permissions on Linux (by parsing `/proc//maps`) 41 | and Windows (by using the `VirtualQuery` function) 42 | 43 | ### Removed 44 | - Removed huge page support 45 | - Removed `commit` method on on Linux and Mac 46 | - Removed tests for `mmap`s at NULL on Linux and Mac, as it turns out they are 47 | guaranteed not to happen 48 | - Removed `test-no-std` feature 49 | 50 | ### Changed 51 | - Changed the way boolean configuration methods on `MapAllocBuilder` work 52 | (previously, for each configuration option, there was a pair of methods to 53 | enable and disable that option; now, there's a single method that accepts a 54 | boolean indicating whether the option should be enabled) 55 | 56 | ### Fixed 57 | - Fixed a bug that prevented compilation on 32-bit Windows 58 | - Fixed a bug caused by `sysconf` 0.3.0 that prevented compilation on Windows 59 | by upgrading to 0.3.1 60 | - Fixed a bug that failed to round allocations up correctly to the next multiple 61 | of the page size 62 | -------------------------------------------------------------------------------- /test-scripts/check-copyright-comments.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 4 | # README.md file at the top-level directory of this repository. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 7 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 8 | # copied, modified, or distributed except according to those terms. 9 | 10 | # modification_years 11 | function modification_years { 12 | # TODO(joshlf): I've determined that the 'git log' command is taking up the 13 | # majority of the execution time of this script. It's unclear whether there's 14 | # much opportunity for optimizing it, but it would be the primary target for 15 | # it. 16 | FIRST_LAST_YEARS=$(git log --pretty=format:"%ci" "$1" | awk 'NR==1; END{print}' | cut -d - -f 1 | tr '\n' ' ') 17 | FIRST_YEAR=$(echo "$FIRST_LAST_YEARS" | cut -d ' ' -f 2) 18 | LAST_YEAR=$(echo "$FIRST_LAST_YEARS" | cut -d ' ' -f 1) 19 | if [ "$FIRST_YEAR" == "$LAST_YEAR" ]; then 20 | echo "$FIRST_YEAR" 21 | else 22 | echo "${FIRST_YEAR}-${LAST_YEAR}" 23 | fi 24 | } 25 | 26 | # like check_comments, checks the th line for the comment 27 | # 28 | # check_comments_shell 29 | function check_comments_line { 30 | RET=0 31 | # NOTE: This will break if any of the files have spaces in them, 32 | # but the other alternative is to pipe the output of find into 33 | # 'while read file', which spawns a subshell, and thus prevents 34 | # us from setting the RET variable from inside the loop. 35 | 36 | for file in $(git ls-tree -r HEAD --name-only | grep "${1}$"); do 37 | # We're either looking for a comment of the form 'Copyright 2017' 38 | # or of the form 'Copyright 2017-2018'. 39 | YEARS=$(modification_years "$file") 40 | # Including " the authors." is important, or else "Copyright 2017-2018 the authors." 41 | # would be spuriously matched if the correct $YEARS was 2017. 42 | COMMENT="$2 Copyright $YEARS the authors." 43 | 44 | head -n "$3" "$file" | tail -n 1 | grep "$COMMENT" >/dev/null 45 | if [ $? -ne 0 ]; then 46 | echo "$file: missing or malformed copyright comment" >&2 47 | RET=1 48 | fi 49 | done 50 | 51 | return "$RET" 52 | } 53 | 54 | # check_comments 55 | function check_comments { 56 | check_comments_line "$1" "$2" 1 57 | } 58 | 59 | EXIT=0 60 | check_comments '.rs' '//' || EXIT=1 61 | check_comments '.md' ' 7 | 8 | Design of alloc-tls 9 | =================== 10 | 11 | `alloc-tls` provides the `alloc_thread_local!` macro, a near-drop-in replacement 12 | for the standard library's `thread_local!` macro that is safe for use in 13 | implementing a global allocator. 14 | 15 | Unlike `thread_local!`, `alloc_thread_local!` address the following issues 16 | unique to implementing a global allocator: 17 | - On platforms that support the `#[thread_local]` attribute, registering 18 | destructors for types that implement `Drop` requires allocation. When a 19 | thread-local is initialized from a call to an allocation function (`malloc`, 20 | `free`, etc), this causes reentrancy. `alloc_thread_local!` can detect this. 21 | - On Mac, it is not safe to access TLS while a dynamic library is being loaded. 22 | When implementing a Mac dynamic library that provides a global allocator, 23 | `alloc_thread_local!` can detect whether the library has been loaded or not, 24 | and can avoid using TLS if `malloc` or other similar calls are made from the 25 | loader itself during loading. 26 | 27 | # Reentrancy 28 | 29 | Reentrancy is addressed by expanding the number of states that a thread-local 30 | variable can be in. Variables defined using the standard library's 31 | `thread_local!` macro can be in one of three states - Uninitialized, Valid, and 32 | Destroyed. In contrast, variables defined using `alloc_thread_local!` can be in 33 | Uninitialized, Initializing (a new state), Initialized (equivalent to Valid), or 34 | Dropped (equivalent to Destroyed). When a variable is accessed in the 35 | Uninitialized state, it is moved into the Initializing state _before_ any 36 | destructors are registered. This way, if registering destructors causes 37 | allocation, any TLS access in that allocation will find the variable in the 38 | Initializing state, and will thus be able to detect the reentrancy. 39 | 40 | # Mac dynamic libraries 41 | 42 | On Mac, dynamic libraries can specify library constructors - functions that are 43 | called immediately after the library is loaded. When compiling a dynamic library 44 | for Mac, `alloc-tls` defines a global `DYLD_LOADED` variable that is initialized 45 | to false. A constructor is registered that sets it to true. When a TLS variable 46 | is accessed, if `DYLD_LOADED` is false, the access fails, leaving it up to the 47 | caller to use some slow path that doesn't rely on thread-local values. 48 | -------------------------------------------------------------------------------- /object-alloc-test/src/leaky_alloc.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | use std::ptr::NonNull; 9 | 10 | use alloc::alloc::{self, Alloc, AllocErr, Layout}; 11 | 12 | /// An allocator that only frees memory when it is dropped. 13 | /// 14 | /// `LeakyAlloc` is an allocator whose `dealloc` method doesn't actually free memory, but simply 15 | /// retains any dealloc'd memory until the `LeakyAlloc` itself is dropped. It is useful for 16 | /// testing. 17 | #[derive(Default)] 18 | pub struct LeakyAlloc { 19 | allocs: Vec, 20 | } 21 | 22 | struct Ptr { 23 | ptr: NonNull, 24 | layout: Layout, 25 | } 26 | 27 | impl Drop for Ptr { 28 | fn drop(&mut self) { 29 | unsafe { 30 | alloc::dealloc(self.ptr.as_ptr(), self.layout.clone()); 31 | } 32 | } 33 | } 34 | 35 | unsafe impl Alloc for LeakyAlloc { 36 | unsafe fn alloc(&mut self, layout: Layout) -> Result, AllocErr> { 37 | let ptr = NonNull::new(alloc::alloc(layout.clone())).ok_or(AllocErr)?; 38 | 39 | self.allocs.push(Ptr { ptr, layout }); 40 | Ok(ptr) 41 | } 42 | 43 | unsafe fn dealloc(&mut self, _: NonNull, _: Layout) {} 44 | } 45 | 46 | #[cfg(test)] 47 | mod tests { 48 | use alloc::alloc::{Alloc, AllocErr, Layout}; 49 | use object_alloc::ObjectAlloc; 50 | use std::marker::PhantomData; 51 | use std::ptr::{self, NonNull}; 52 | 53 | #[derive(Default)] 54 | struct LeakyObjectAlloc { 55 | alloc: super::LeakyAlloc, 56 | _marker: PhantomData, 57 | } 58 | 59 | unsafe impl ObjectAlloc for LeakyObjectAlloc { 60 | unsafe fn alloc(&mut self) -> Option> { 61 | let ptr = match Alloc::alloc(&mut self.alloc, Layout::new::()) { 62 | Ok(ptr) => ptr.cast(), 63 | Err(AllocErr) => return None, 64 | }; 65 | 66 | ptr::write(ptr.as_ptr(), T::default()); 67 | Some(ptr) 68 | } 69 | 70 | unsafe fn dealloc(&mut self, ptr: NonNull) { 71 | ::std::ptr::drop_in_place(ptr.as_ptr()); 72 | Alloc::dealloc(&mut self.alloc, ptr.cast(), Layout::new::()); 73 | } 74 | } 75 | 76 | #[test] 77 | fn test_memory_corruption() { 78 | use corruption::{CorruptionTesterDefault, TestBuilder}; 79 | TestBuilder::new(|| LeakyObjectAlloc::::default()).test(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /alloc-fmt/src/bin/assert.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | #[macro_use] 9 | extern crate alloc_fmt; 10 | use alloc_fmt::AllocUnwrap; 11 | 12 | #[cfg_attr(feature = "cargo-clippy", allow(cyclomatic_complexity))] 13 | fn main() { 14 | let arg = std::env::args().nth(1).expect("must provide an argument"); 15 | 16 | match arg.as_str() { 17 | "assert" => alloc_assert!(false && true), 18 | "assert_fmt" => alloc_assert!(false && true, "foo"), 19 | "assert_fmt_args" => alloc_assert!(false && true, "foo: {}", "bar"), 20 | "debug_assert" => alloc_debug_assert!(false && true), 21 | "debug_assert_fmt" => alloc_debug_assert!(false && true, "foo"), 22 | "debug_assert_fmt_args" => alloc_debug_assert!(false && true, "foo: {}", "bar"), 23 | "assert_eq" => alloc_assert_eq!(1 + 2, 1), 24 | "assert_eq_fmt" => alloc_assert_eq!(1 + 2, 1, "foo"), 25 | "assert_eq_fmt_args" => alloc_assert_eq!(1 + 2, 1, "foo: {}", "bar"), 26 | "debug_assert_eq" => alloc_debug_assert_eq!(1 + 2, 1), 27 | "debug_assert_eq_fmt" => alloc_debug_assert_eq!(1 + 2, 1, "foo"), 28 | "debug_assert_eq_fmt_args" => alloc_debug_assert_eq!(1 + 2, 1, "foo: {}", "bar"), 29 | "assert_ne" => alloc_assert_ne!(1 + 2, 3), 30 | "assert_ne_fmt" => alloc_assert_ne!(1 + 2, 3, "foo"), 31 | "assert_ne_fmt_args" => alloc_assert_ne!(1 + 2, 3, "foo: {}", "bar"), 32 | "debug_assert_ne" => alloc_debug_assert_ne!(1 + 2, 3), 33 | "debug_assert_ne_fmt" => alloc_debug_assert_ne!(1 + 2, 3, "foo"), 34 | "debug_assert_ne_fmt_args" => alloc_debug_assert_ne!(1 + 2, 3, "foo: {}", "bar"), 35 | "option_alloc_unwrap_success" => assert_eq!(Some(0).alloc_unwrap(), 0), 36 | "option_alloc_unwrap_failure" => assert_eq!((None as Option).alloc_unwrap(), 0), 37 | "option_alloc_expect_success" => assert_eq!(Some(0).alloc_expect("expect failed"), 0), 38 | "option_alloc_expect_failure" => { 39 | assert_eq!((None as Option).alloc_expect("expect failed"), 0) 40 | } 41 | "result_alloc_unwrap_success" => assert_eq!((Ok(0) as Result<_, ()>).alloc_unwrap(), 0), 42 | "result_alloc_unwrap_failure" => { 43 | assert_eq!((Err("") as Result).alloc_unwrap(), 0) 44 | } 45 | "result_alloc_expect_success" => { 46 | assert_eq!((Ok(0) as Result<_, ()>).alloc_expect("expect failed"), 0) 47 | } 48 | "result_alloc_expect_failure" => assert_eq!( 49 | (Err("") as Result).alloc_expect("expect failed"), 50 | 0 51 | ), 52 | _ => eprintln!("see source code for available arguments"), 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | # README.md file at the top-level directory of this repository. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | # the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | # copied, modified, or distributed except according to those terms. 7 | 8 | # Partially adapted from https://github.com/starkat99/appveyor-rust/blob/master/appveyor.yml 9 | 10 | # Description of environment variables: 11 | # CHANNEL - stable/beta/nightly 12 | # TARGET - target platform 13 | # CYGWIN_BASH - cygwin's bash binary 14 | # RUST_NIGHTLY - set to "1" on the nightly channel so that appveyor.sh scripts 15 | # can check for it 16 | 17 | environment: 18 | matrix: 19 | 20 | ### MSVC Toolchains ### 21 | 22 | # Stable 64-bit MSVC 23 | - CHANNEL: stable 24 | TARGET: x86_64-pc-windows-msvc 25 | CYGWIN_BASH: C:\Cygwin64\bin\bash 26 | # Stable 32-bit MSVC 27 | - CHANNEL: stable 28 | TARGET: i686-pc-windows-msvc 29 | CYGWIN_BASH: C:\Cygwin\bin\bash 30 | # Beta 64-bit MSVC 31 | - CHANNEL: beta 32 | TARGET: x86_64-pc-windows-msvc 33 | CYGWIN_BASH: C:\Cygwin64\bin\bash 34 | # Beta 32-bit MSVC 35 | - CHANNEL: beta 36 | TARGET: i686-pc-windows-msvc 37 | CYGWIN_BASH: C:\Cygwin\bin\bash 38 | # Nightly 64-bit MSVC 39 | - CHANNEL: nightly 40 | TARGET: x86_64-pc-windows-msvc 41 | CYGWIN_BASH: C:\Cygwin64\bin\bash 42 | RUST_NIGHTLY: 1 43 | # Nightly 32-bit MSVC 44 | - CHANNEL: nightly 45 | TARGET: i686-pc-windows-msvc 46 | CYGWIN_BASH: C:\Cygwin\bin\bash 47 | RUST_NIGHTLY: 1 48 | 49 | ### GNU Toolchains ### 50 | 51 | # Stable 64-bit GNU 52 | - CHANNEL: stable 53 | TARGET: x86_64-pc-windows-gnu 54 | CYGWIN_BASH: C:\Cygwin64\bin\bash 55 | # Stable 32-bit GNU 56 | - CHANNEL: stable 57 | TARGET: i686-pc-windows-gnu 58 | CYGWIN_BASH: C:\Cygwin\bin\bash 59 | # Beta 64-bit GNU 60 | - CHANNEL: beta 61 | TARGET: x86_64-pc-windows-gnu 62 | CYGWIN_BASH: C:\Cygwin64\bin\bash 63 | # Beta 32-bit GNU 64 | - CHANNEL: beta 65 | TARGET: i686-pc-windows-gnu 66 | CYGWIN_BASH: C:\Cygwin\bin\bash 67 | # Nightly 64-bit GNU 68 | - CHANNEL: nightly 69 | TARGET: x86_64-pc-windows-gnu 70 | CYGWIN_BASH: C:\Cygwin64\bin\bash 71 | RUST_NIGHTLY: 1 72 | # Nightly 32-bit GNU 73 | - CHANNEL: nightly 74 | TARGET: i686-pc-windows-gnu 75 | CYGWIN_BASH: C:\Cygwin\bin\bash 76 | RUST_NIGHTLY: 1 77 | 78 | # NOTE: The %CHANNEL% and %TARGET% environment variables are taken from the 79 | # build matrix defined above. 80 | install: 81 | - curl -fsS --retry 10 --retry-connrefused -o rustup-init.exe https://win.rustup.rs/ 82 | - rustup-init -yv --default-toolchain %CHANNEL% --default-host %TARGET% 83 | - set PATH=%PATH%;%USERPROFILE%\.cargo\bin 84 | - rustc -vV 85 | - cargo -vV 86 | 87 | # We build ourselves (in the test script) 88 | build: false 89 | 90 | test_script: 91 | - "%CYGWIN_BASH% -c 'set -x; set -e; for script in */appveyor.sh; do 92 | dir=$(dirname \"$script\"); 93 | file=$(basename \"$script\"); 94 | pushd \"$dir\"; 95 | ./$file; 96 | popd; 97 | done'" 98 | -------------------------------------------------------------------------------- /elfmalloc/src/stats.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | //! This module includes some lightweight utilities for tracing different allocation events. 9 | //! Like all global abstractions in use here, some hacks are required to get this to work correctly 10 | //! when this library is linked as part of a `malloc`. 11 | //! 12 | //! ## `THREAD_CTR` 13 | //! 14 | //! We use a global `AtomicUsize` to creat thread Ids rather than `thread::current().id()`. This is 15 | //! because the latter method panics somewhere when setting thread metadata during thread creation. 16 | //! This is presumably some ill-fated `malloc` dependency. 17 | //! 18 | //! ## `RefCell` 19 | //! 20 | //! While `RefCell` is probably the correct thing to use here regardless, we really must use some 21 | //! version of it. This is because the stat collection infrastructure can recursively call itself 22 | //! during thread initialization time because TLS can call `calloc`. This is similar to a problem 23 | //! encountered in the `general::global` in this crate. `RefCell`'s `try_borrow` gives us a guard 24 | //! against this recursion. 25 | 26 | type Num = i64; 27 | 28 | #[derive(Default, Debug, Clone, Copy)] 29 | /// Counters for various useful events. Primarily used for debugging and performance. 30 | pub struct AllocStats { 31 | /// Allocations from the owned `Slag` 32 | pub slag_alloc: Num, 33 | /// Allocations from a local cache 34 | pub cache_alloc: Num, 35 | 36 | /// Frees to a local cache. 37 | pub local_free: Num, 38 | /// Frees to a remote `Slag` 39 | pub remote_free: Num, 40 | /// Bulk frees to a remote `Slag` 41 | pub bulk_remote_free: Num, 42 | 43 | /// Transition a floating `Slag` to the available state 44 | pub transition_available: Num, 45 | /// Successfully transition an available `Slag` to the full state. 46 | pub transition_full: Num, 47 | 48 | /// Acquire an available `Slag` 49 | pub grabbed_available: Num, 50 | /// Acquire a dirty page 51 | pub grabbed_dirty: Num, 52 | /// Acquire a clean page 53 | pub grabbed_clean: Num, 54 | } 55 | 56 | #[cfg(feature = "print_stats")] 57 | pub mod state { 58 | use super::*; 59 | use std::cell::RefCell; 60 | use std::sync::atomic::{AtomicUsize, Ordering}; 61 | pub struct StatsHandle { 62 | pub stats: RefCell, 63 | pub thread_num: usize, 64 | } 65 | lazy_static! { 66 | pub static ref THREAD_CTR: AtomicUsize = AtomicUsize::new(0); 67 | } 68 | thread_local! { 69 | pub static LOCAL_STATS: StatsHandle = StatsHandle { 70 | stats: RefCell::new(AllocStats::default()), 71 | thread_num: THREAD_CTR.fetch_add(1, Ordering::Relaxed), 72 | }; 73 | } 74 | 75 | } 76 | 77 | macro_rules! trace_event { 78 | ($fld:tt) => { 79 | #[cfg(feature = "print_stats")] 80 | { 81 | let _ = $crate::stats::state::LOCAL_STATS.try_with(|sh| { 82 | if let Ok(mut _f_ref) = sh.stats.try_borrow_mut() { 83 | _f_ref.$fld += 1; 84 | let allocs = _f_ref.slag_alloc + _f_ref.cache_alloc; 85 | if (allocs % (1 << 22)) == 0 { 86 | trace!("thread {:2?} - {:?}", sh.thread_num, *_f_ref); 87 | } 88 | } 89 | }); 90 | } 91 | }; 92 | } 93 | -------------------------------------------------------------------------------- /slab-alloc/src/aligned.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | //! A slab that uses alignment to map objects to slabs. 9 | //! 10 | //! This module provides a slab implementation similar to Bonwick's original small slabs, but which 11 | //! relies on variable-size blocks of memory instead of fixed-size pages (page-sized blocks can be 12 | //! allocated, making Bonwick's original small slabs a special case of aligned slabs). 13 | //! 14 | //! The core idea behind aligned slabs, as with small slabs, is to back each slab with a chunk of 15 | //! memory whose alignment is equal to its size (thus the size is always a power of two). This 16 | //! makes it so that, given a pointer to an allocated object, figuring out which slab that object 17 | //! was allocated from is a trivial matter of masking off the insignificant lower-order bits, 18 | //! keeping only the bits needed to identify the containing slab. Compare this with the large slab 19 | //! algorithm, which requires keeping an allocator-global hash table mapping object pointers to 20 | //! their containing slabs. 21 | //! 22 | //! This implementation also differs from the original Bonwick algorithm in that it keeps track of 23 | //! available objects in a stack of pointers rather than in a linked list. For more details on this 24 | //! approach to keeping track of available objects, see the `stack` module. 25 | 26 | use alloc::alloc; 27 | use core::ptr::NonNull; 28 | use init::InitSystem; 29 | use object_alloc::UntypedObjectAlloc; 30 | use stack::{Layout, SlabHeader}; 31 | use {stack, OBJECTS_PER_SLAB, PAGE_SIZE}; 32 | 33 | pub struct ConfigData; 34 | 35 | impl stack::ConfigData for ConfigData { 36 | fn ptr_to_slab(&self, slab_size: usize, ptr: NonNull) -> NonNull { 37 | let slab = ptr.as_ptr() as usize & !(slab_size - 1); 38 | debug_assert_eq!(slab % slab_size, 0); 39 | unsafe { NonNull::new_unchecked(slab as *mut SlabHeader) } 40 | } 41 | } 42 | 43 | pub type System = stack::System; 44 | 45 | impl System { 46 | pub fn new(layout: alloc::Layout, alloc: A) -> Option> { 47 | if let Some((slab_layout, _)) = Layout::for_slab_size(layout, alloc.layout().size()) { 48 | Some(Self::from_config_data(ConfigData, slab_layout, alloc)) 49 | } else { 50 | None 51 | } 52 | } 53 | } 54 | 55 | pub fn backing_size_for(layout: &alloc::Layout) -> usize { 56 | struct PowerOfTwoIterator(usize); 57 | 58 | impl Iterator for PowerOfTwoIterator { 59 | type Item = usize; 60 | 61 | fn next(&mut self) -> Option { 62 | if (self.0 << 1) == 0 { 63 | // another double would overflow usize 64 | None 65 | } else { 66 | self.0 *= 2; 67 | Some(self.0) 68 | } 69 | } 70 | } 71 | 72 | let unused = |slab_size: usize| { 73 | Layout::for_slab_size(layout.clone(), slab_size) 74 | .map(|(layout, unused)| (layout.num_obj, unused)) 75 | }; 76 | 77 | // We guarantee that we never request aligned slabs smaller than a page (see the Documentation 78 | // on SlabAllocBuilder::build_backing), so we start off with an initial size of *PAGE_SIZE and 79 | // go up from there. 80 | ::util::size::choose_size(PowerOfTwoIterator(*PAGE_SIZE), unused, OBJECTS_PER_SLAB) 81 | } 82 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Allocators in Rust 9 | 10 | [![Build Status](https://travis-ci.org/ezrosent/allocators-rs.svg?branch=master)](https://travis-ci.org/ezrosent/allocators-rs) 11 | [![Build status](https://ci.appveyor.com/api/projects/status/github/ezrosent/allocators-rs?svg=true)](https://ci.appveyor.com/project/ezrosent/allocators-rs) 12 | 13 | _Looking for elfmalloc in particular? It's [here](https://github.com/ezrosent/allocators-rs/blob/master/elfmalloc)._ 14 | 15 | This repository encompasses a number of different crates. Some are general 16 | memory allocators, some are object allocators (allocators which allocate a 17 | specific type of object), and some are utility crates providing various 18 | features needed to implement these allocators. 19 | 20 | The [`info`](https://github.com/ezrosent/allocators-rs/blob/master/info) directory contains more detailed information, including performance 21 | measurements. Aside from `info`, all top-level directories are Rust crates. 22 | More detailed information on each crate can be found in the crate's `README.md`. 23 | 24 | ### Allocator crates 25 | 26 | | Crate | Description | 27 | |-------|-------------| 28 | | [`elfmalloc`](https://github.com/ezrosent/allocators-rs/blob/master/elfmalloc) | A general-purpose multi-threaded allocator providing both Rust `Alloc` and C `malloc` APIs | 29 | | [`slab-alloc`](https://github.com/ezrosent/allocators-rs/blob/master/slab-alloc) | An object-specific slab allocator in the same tradition as the original [slab allocator](https://www.usenix.org/legacy/publications/library/proceedings/bos94/full_papers/bonwick.a) by Jeff Bonwick | 30 | | [`bsalloc`](https://github.com/ezrosent/allocators-rs/blob/master/bsalloc) | A simple general-purpose "bootstrapping" allocator used in the implementation of other allocators | 31 | | [`mmap-alloc`](https://github.com/ezrosent/allocators-rs/blob/master/mmap-alloc) | An `Alloc` API which is a thin wrapper around `mmap` (on Unix) or `VirtualAlloc` (on Windows) | 32 | 33 | ### Utility crates 34 | 35 | | Crate | Description | 36 | |-------|-------------| 37 | | [`alloc-fmt`](https://github.com/ezrosent/allocators-rs/blob/master/alloc-fmt) | Allocation-safe formatting and debugging macros | 38 | | [`alloc-tls`](https://github.com/ezrosent/allocators-rs/blob/master/alloc-tls) | Allocation-safe thread-local storage | 39 | | [`bagpipe`](https://github.com/ezrosent/allocators-rs/blob/master/bagpipe) | Fast, concurrent data structures including queues and a weakly-ordered bag data structure ([design](https://github.com/ezrosent/allocators-rs/blob/master/info/bagpipes.md)) | 40 | | [`malloc-bind`](https://github.com/ezrosent/allocators-rs/blob/master/malloc-bind) | Bindings to allow a Rust `Alloc` to implement the C `malloc` API | 41 | | [`object-alloc`](https://github.com/ezrosent/allocators-rs/blob/master/object-alloc) | Traits representing type-specific variants of the `Alloc` trait | 42 | | [`object-alloc-test`](https://github.com/ezrosent/allocators-rs/blob/master/object-alloc-test) | A correctness test suite for object allocator implementations (uses the traits defined in `object-alloc`) | 43 | 44 | ## Contributing 45 | 46 | Interested in contributing? We'd love to have you! Check out [CONTRIBUTING.md](https://github.com/ezrosent/allocators-rs/blob/master/CONTRIBUTING.md). 47 | 48 | ## Copyright and license 49 | 50 | Copyrights in this project are retained by their contributors. No copyright 51 | assignment is required to contribute to this project. For a list of authors, see 52 | this repository's version control history. 53 | 54 | This project is dual-licensed under the [Apache 55 | 2.0](http://www.apache.org/licenses/LICENSE-2.0) license or the 56 | [MIT](http://opensource.org/licenses/MIT) license at your option. Copies of 57 | these licenses can be found in the `LICENSE-APACHE` and `LICENSE-MIT` files 58 | respectively. 59 | -------------------------------------------------------------------------------- /elfmalloc/src/alloc_impl.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | //! Implement various `Alloc`-specific traits. 9 | //! 10 | //! The reason why we do not do this directly is that Rust allocators have a different signature 11 | //! than the C/C++ `malloc` ecosystem. In particular, deallocations pass object size and alignment 12 | //! at the call site. 13 | //! 14 | //! This module also implements additional traits from the `malloc-bind` crate. 15 | 16 | extern crate alloc; 17 | #[cfg(feature = "c-api")] 18 | extern crate malloc_bind; 19 | #[cfg(feature = "c-api")] 20 | extern crate libc; 21 | use self::alloc::alloc::{Alloc, AllocErr, Layout}; 22 | #[cfg(feature = "c-api")] 23 | use self::malloc_bind::{LayoutFinder, Malloc, MIN_ALIGN}; 24 | use super::general::global; 25 | use std::mem; 26 | #[cfg(feature = "c-api")] 27 | use std::intrinsics::unlikely; 28 | 29 | #[cfg(feature = "c-api")] 30 | use self::libc::{size_t, c_void}; 31 | 32 | /// A zero-sized type used for implementing `Alloc` and `LayoutFinder` for the global instance of 33 | /// elfmalloc. 34 | pub struct ElfMallocGlobal; 35 | 36 | unsafe impl<'a> Alloc for &'a ElfMallocGlobal { 37 | unsafe fn alloc(&mut self, l: Layout) -> Result<*mut u8, AllocErr> { 38 | // All objects are only guaranteed to be word-aligned except for powers of two. Powers of 39 | // two up to 1MiB are aligned to their size. Past that size, only page-alignment is 40 | // guaranteed. 41 | if l.size().is_power_of_two() || l.align() <= mem::size_of::() { 42 | Ok(global::alloc(l.size())) 43 | } else { 44 | Ok(global::alloc(l.size().next_power_of_two())) 45 | } 46 | } 47 | 48 | unsafe fn dealloc(&mut self, p: *mut u8, _l: Layout) { 49 | global::free(p); 50 | } 51 | 52 | unsafe fn realloc(&mut self, p: *mut u8, _l1: Layout, l2: Layout) -> Result<*mut u8, AllocErr> { 53 | Ok(global::aligned_realloc(p, l2.size(), l2.align())) 54 | } 55 | } 56 | 57 | #[cfg(feature = "c-api")] 58 | unsafe impl Malloc for ElfMallocGlobal { 59 | unsafe fn c_malloc(&self, size: size_t) -> *mut c_void { 60 | let p = global::alloc(size as usize) as *mut c_void; 61 | alloc_debug_assert_eq!((p as usize) % MIN_ALIGN, 62 | 0, 63 | "object does not have the required alignment of {}: {:?}", 64 | MIN_ALIGN, 65 | p); 66 | p 67 | } 68 | unsafe fn c_free(&self, p: *mut c_void) { 69 | alloc_debug_assert_eq!((p as usize) % MIN_ALIGN, 70 | 0, 71 | "object does not have the required alignment of {}: {:?}", 72 | MIN_ALIGN, 73 | p); 74 | if unlikely(p.is_null()) { 75 | return; 76 | } 77 | global::free(p as *mut u8) 78 | } 79 | unsafe fn c_realloc(&self, p: *mut c_void, new_size: size_t) -> *mut c_void { 80 | alloc_debug_assert_eq!((p as usize) % MIN_ALIGN, 81 | 0, 82 | "object does not have the required alignment of {}: {:?}", 83 | MIN_ALIGN, 84 | p); 85 | global::realloc(p as *mut u8, new_size as usize) as *mut c_void 86 | } 87 | } 88 | 89 | #[cfg(feature = "c-api")] 90 | unsafe impl LayoutFinder for ElfMallocGlobal { 91 | unsafe fn get_layout(&self, p: *mut u8) -> Layout { 92 | // Note that per the current implementation of malloc-bind, we could just return a dummy 93 | // value, e.g. Layout::from_size_align(8, 8).unwrap() 94 | // But, seeing as that (and `Alloc`'s default impl's internals) may change, we are going to 95 | // err on the side of caution for the time being 96 | let (size, align) = global::get_layout(p); 97 | Layout::from_size_align(size, align).unwrap() 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | Contributing 9 | ============ 10 | 11 | Hi there! Interested in contributing? We'd love to have you! 12 | 13 | ## Getting started 14 | 15 | If you're new to Rust or to this project and just want an easy issue to get started with, check out the issues with the [experience-easy](https://github.com/ezrosent/allocators-rs/issues?q=is%3Aissue+is%3Aopen+label%3Aexperience-easy) label. If you have any questions about anything, feel free to contact [joshlf](https://github.com/joshlf) or [ezrosent](https://github.com/ezrosent). 16 | 17 | ### Finding something to fix/improve 18 | 19 | All issues are labeled with the difficulty ([experience-easy](https://github.com/ezrosent/allocators-rs/issues?q=is%3Aissue+is%3Aopen+label%3Aexperience-easy), [experience-medium](https://github.com/ezrosent/allocators-rs/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aopen%20label%3Aexperience-medium%20), and [experience-hard](https://github.com/ezrosent/allocators-rs/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aopen%20label%3Aexperience-hard%20)) and the component of the project that they apply to (e.g., [crate-elfmalloc](https://github.com/ezrosent/allocators-rs/issues?q=is%3Aissue+is%3Aopen+label%3Acrate-elfmalloc), [travis-ci](https://github.com/ezrosent/allocators-rs/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aopen%20label%3Atravis-ci), etc). Use label search or browse the issues to find something you're interested in. 20 | 21 | Once you've found an issue to work on, please comment on it so that people know you're working on it and don't duplicate your work. 22 | 23 | ### Mentoring 24 | 25 | Since we're a small project, we haven't assigned particular mentors for particular issues. However, we're happy to mentor anybody working on any issue, so if you'd like mentorship on working through an issue, don't hesitate to reach out! 26 | 27 | ## Submitting a change 28 | 29 | We use GitHub PRs to accept and review changes. In order to submit a change: 30 | - Fork this repository 31 | - Make the change in your fork 32 | - Open a PR with your changes 33 | 34 | PRs should abide by the following guidelines, but if you aren't sure about how to do that, feel free to submit a work-in-progress PR and we'll help you out. 35 | 36 | ### Tests 37 | The following tests are automatically run by Travis CI when a PR is submitted. Since Travis can be slow, it's probably best to make sure they pass by running them locally before pushing changes. 38 | - `cargo clippy` must not show any warnings 39 | - `cargo test` must pass 40 | - If there are any features for the crate, then `cargo clippy` and `cargo test` should also be run with each of the features enabled 41 | 42 | Please add tests for any significant changed or added features. If you're not sure where tests should go or how they should be written, just ask and we'd be happy to help! 43 | 44 | ### API Design 45 | Providing clean, ergonomic APIs are important. If you're adding or modifying APIs, please take care to design them with the user in mind. See the [Rust API Guidelines](https://rust-lang-nursery.github.io/api-guidelines/) for more detailed recommendations. 46 | 47 | ### Documentation 48 | All public-facing APIs should have documentation which abides by [RFC 505](https://github.com/rust-lang/rfcs/blob/master/text/0505-api-comment-conventions.md). Try running `cargo doc --open` to compile and view documentation in the browser to see if it looks as you expect. 49 | 50 | ### Commit messages 51 | Commit messages should be of the form `: `, where the description is written in the third person singular present imperative form, and doesn't have a period at the end. What a mouthful! Basically, this means a description like "Do the thing." For example, `slab-alloc: Add documentation comments`. 52 | 53 | ### `CHANGELOG.md` file 54 | Each crate has its own `CHANGELOG.md` file, and unreleased changes are tracked in the "Unreleased" section at the top of the file. Any changes that you make should be added to this section so that they can be incorporated into the changelog entry for the next release. Our changelogs follow the [keep a changelog](http://keepachangelog.com/) format. 55 | 56 | ### Copyright comment 57 | Every new file should bear the following copyright comment at the top, preceding any other text: 58 | 59 | ```rust 60 | // Copyright 2017 the authors. See the 'Copyright and license' section of the 61 | // README.md file at the top-level directory of this repository. 62 | // 63 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 64 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 65 | // copied, modified, or distributed except according to those terms. 66 | ``` 67 | 68 | Modified files should have the copyright dates updated. For example, if a file's comment reads `Copyright 2017`, but it is modified in 2018, then the comment should be updated to read `Copyright 2017-2018`. 69 | 70 | ### Rebasing 71 | All changes should be rebased onto the master branch so that we maintain a linear history. If you're not sure how to do that, just ask! For a good introduction to rebasing and why it's helpful, see [here](https://git-scm.com/book/en/v2/Git-Branching-Rebasing). 72 | 73 | ## Conduct 74 | 75 | We follow the [Rust Code of Conduct](https://www.rust-lang.org/en-US/conduct.html). 76 | -------------------------------------------------------------------------------- /slab-alloc/src/backing.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | // don't conflict with the alloc module 9 | extern crate alloc as crate_alloc; 10 | extern crate object_alloc; 11 | 12 | use self::object_alloc::UntypedObjectAlloc; 13 | 14 | /// A pair of `UntypedObjectAlloc`s to provide memory for a slab allocator. 15 | /// 16 | /// A `BackingAlloc` encapsulates a pair of allocator types that are used to provide the memory 17 | /// needed to construct the slabs used by a slab allocator. Under the hood, slab allocators use 18 | /// two different types of slabs - "aligned" slabs and "large" slabs - with different requirements 19 | /// for backing memory. A `BackingAlloc` provides an allocator for each of the two slab types. 20 | pub trait BackingAlloc { 21 | /// An allocator for aligned slabs. 22 | /// 23 | /// An `Aligned` allocator allocates the memory for aligned slabs. All memory allocated by this 24 | /// allocator must have an alignment which is equal to its size. An `Aligned` allocator must be 25 | /// capable of allocating page-sized blocks of memory. Smaller or larger blocks of memory are 26 | /// not required to be supported. Allocated memory should be uninitialized. 27 | type Aligned: UntypedObjectAlloc; 28 | 29 | /// An allocator for large slabs. 30 | /// 31 | /// A `Large` allocator allocates the memory for large slabs. All memory allocated by this 32 | /// allocator must be page-aligned; a `Large` allocator will never be constructed with an 33 | /// object size smaller than the page size. All sizes larger than or equal to a page size must 34 | /// be supported. Allocated memory should be uninitialized. 35 | type Large: UntypedObjectAlloc; 36 | } 37 | 38 | /// An `UntypedObjectAlloc` that uses arbitrary allocators. 39 | pub mod alloc { 40 | use alloc::alloc::{Alloc, AllocErr, Layout}; 41 | use core::ptr::NonNull; 42 | use object_alloc::UntypedObjectAlloc; 43 | 44 | /// An `UntypedObjectAlloc` that uses an arbitrary allocator. 45 | #[derive(Clone)] 46 | pub struct AllocObjectAlloc { 47 | alloc: A, 48 | layout: Layout, 49 | } 50 | 51 | impl AllocObjectAlloc { 52 | pub fn new(alloc: A, layout: Layout) -> AllocObjectAlloc { 53 | AllocObjectAlloc { alloc, layout } 54 | } 55 | } 56 | 57 | unsafe impl UntypedObjectAlloc for AllocObjectAlloc { 58 | fn layout(&self) -> Layout { 59 | self.layout.clone() 60 | } 61 | 62 | unsafe fn alloc(&mut self) -> Option> { 63 | match self.alloc.alloc(self.layout.clone()) { 64 | Ok(ptr) => Some(ptr), 65 | Err(AllocErr) => None, 66 | } 67 | } 68 | 69 | unsafe fn dealloc(&mut self, ptr: NonNull) { 70 | self.alloc.dealloc(ptr, self.layout.clone()); 71 | } 72 | } 73 | } 74 | 75 | /// A `BackingAlloc` that uses the heap. 76 | #[cfg(feature = "std")] 77 | pub mod heap { 78 | use alloc::alloc::{Global, Layout}; 79 | 80 | use super::alloc::AllocObjectAlloc; 81 | use super::BackingAlloc; 82 | use PAGE_SIZE; 83 | 84 | pub struct HeapBackingAlloc; 85 | 86 | impl BackingAlloc for HeapBackingAlloc { 87 | type Aligned = AllocObjectAlloc; 88 | type Large = AllocObjectAlloc; 89 | } 90 | 91 | pub fn get_aligned(layout: Layout) -> Option> { 92 | if layout.size() > *PAGE_SIZE { 93 | None 94 | } else { 95 | Some(AllocObjectAlloc::new(Global, layout)) 96 | } 97 | } 98 | 99 | pub fn get_large(layout: Layout) -> AllocObjectAlloc { 100 | AllocObjectAlloc::new(Global, layout) 101 | } 102 | } 103 | 104 | /// A `BackingAlloc` that uses mmap. 105 | #[cfg(feature = "os")] 106 | pub mod mmap { 107 | extern crate alloc; 108 | extern crate mmap_alloc; 109 | extern crate sysconf; 110 | use self::alloc::alloc::Layout; 111 | #[cfg(target_os = "linux")] 112 | use self::mmap_alloc::MapAlloc; 113 | #[cfg(not(target_os = "linux"))] 114 | use self::mmap_alloc::MapAlloc; 115 | use super::alloc::AllocObjectAlloc; 116 | use super::BackingAlloc; 117 | use PAGE_SIZE; 118 | 119 | pub struct MmapBackingAlloc; 120 | 121 | impl BackingAlloc for MmapBackingAlloc { 122 | type Aligned = AllocObjectAlloc; 123 | type Large = AllocObjectAlloc; 124 | } 125 | 126 | pub fn get_aligned(layout: Layout) -> Option> { 127 | debug_assert_eq!(layout.size(), layout.align()); 128 | if layout.size() != *PAGE_SIZE { 129 | // TODO: Use an elfmalloc-style approach of allocating a large 130 | // region and then freeing the pages that are not part of the 131 | // size-aligned subset. 132 | None 133 | } else { 134 | Some(AllocObjectAlloc::new(MapAlloc::default(), layout)) 135 | } 136 | } 137 | 138 | pub fn get_large(layout: Layout) -> AllocObjectAlloc { 139 | debug_assert!(layout.align() <= *PAGE_SIZE); 140 | AllocObjectAlloc::new(MapAlloc::default(), layout) 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /bsalloc/src/bsalloc.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of 2 | // the README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | use core::ptr::NonNull; 9 | use core::sync::atomic::{AtomicPtr, Ordering}; 10 | use core::{mem, ptr}; 11 | 12 | use mmap_alloc::MapAlloc; 13 | #[cfg(windows)] 14 | use mmap_alloc::MapAllocBuilder; 15 | 16 | use {Alloc, Layout}; 17 | 18 | #[derive(Copy, Clone)] 19 | pub struct BsAlloc; 20 | 21 | lazy_static! { 22 | pub static ref ALLOC: GlobalAllocator = GlobalAllocator::new(); 23 | } 24 | 25 | pub struct GlobalAllocator { 26 | small_objs: large::Cache, 27 | large_objs: small::Cache, 28 | ma: MapAlloc, 29 | } 30 | 31 | impl GlobalAllocator { 32 | fn new() -> Self { 33 | GlobalAllocator { 34 | small_objs: large::Cache::new(1 << 10), 35 | large_objs: small::Cache::new(18 << 10), 36 | // On Windows, memory must be explicitly committed - it will not 37 | // simply be committed on first use like on Linux and Mac. 38 | #[cfg(windows)] 39 | ma: MapAllocBuilder::default().commit(true).build(), 40 | #[cfg(not(windows))] 41 | ma: MapAlloc::default(), 42 | } 43 | } 44 | 45 | pub unsafe fn alloc(&self, size: usize) -> *mut u8 { 46 | let mut ma = &self.ma; 47 | if size < self.small_objs.object_size { 48 | self.small_objs.alloc(ma) 49 | } else if size < self.large_objs.object_size { 50 | self.large_objs.alloc(ma) 51 | } else { 52 | ma.alloc(Layout::from_size_align(size, 1).unwrap()) 53 | .map(|ptr| ptr.as_ptr()) 54 | .unwrap_or(ptr::null_mut()) 55 | } 56 | } 57 | 58 | pub unsafe fn free(&self, item: *mut u8, size: usize) { 59 | let mut ma = &self.ma; 60 | if size < self.small_objs.object_size { 61 | self.small_objs.free(item, ma); 62 | return; 63 | } 64 | if size < self.large_objs.object_size { 65 | self.large_objs.free(item, ma); 66 | return; 67 | } 68 | ma.dealloc( 69 | NonNull::new_unchecked(item), 70 | Layout::from_size_align(size, 1).unwrap(), 71 | ) 72 | } 73 | } 74 | 75 | fn rng() -> usize { 76 | const RAND_A: usize = 16_807; 77 | let seed_ptr: usize = 0; 78 | let seed = (&seed_ptr) as *const _ as usize; 79 | seed * RAND_A 80 | } 81 | 82 | macro_rules! sized_cache { 83 | ($name:ident, $cache_size:expr) => { 84 | mod $name { 85 | use super::*; 86 | const CACHE_SIZE: usize = $cache_size; 87 | const PATIENCE: usize = CACHE_SIZE >> 2; 88 | pub struct Cache { 89 | pub object_size: usize, 90 | layout: Layout, 91 | cache: [AtomicPtr; CACHE_SIZE], 92 | } 93 | 94 | impl Cache { 95 | pub fn new(obj_size: usize) -> Self { 96 | let res = Cache { 97 | object_size: obj_size, 98 | layout: Layout::from_size_align(obj_size, 1).unwrap(), 99 | cache: unsafe { 100 | mem::transmute::<[usize; CACHE_SIZE], [AtomicPtr; CACHE_SIZE]>( 101 | [0; CACHE_SIZE], 102 | ) 103 | }, 104 | }; 105 | res 106 | } 107 | 108 | pub unsafe fn alloc(&self, mut ma: &MapAlloc) -> *mut u8 { 109 | let start = rng(); 110 | let mut cur_slot = start % CACHE_SIZE; 111 | let stride = cur_slot * 2 + 1; 112 | for _ in 0..PATIENCE { 113 | let p = self 114 | .cache 115 | .get_unchecked(cur_slot) 116 | .swap(ptr::null_mut(), Ordering::Relaxed); 117 | if p.is_null() { 118 | cur_slot += stride; 119 | cur_slot %= CACHE_SIZE; 120 | continue; 121 | } 122 | return p; 123 | } 124 | ma.alloc(self.layout.clone()) 125 | .map(|ptr| ptr.as_ptr()) 126 | .unwrap_or(ptr::null_mut()) 127 | } 128 | 129 | pub unsafe fn free(&self, item: *mut u8, mut ma: &MapAlloc) { 130 | let mut cur_ptr = item; 131 | let start = rng(); 132 | let mut cur_slot = start % CACHE_SIZE; 133 | let stride = cur_slot * 2 + 1; 134 | for _ in 0..PATIENCE { 135 | cur_ptr = self 136 | .cache 137 | .get_unchecked(cur_slot) 138 | .swap(cur_ptr, Ordering::Relaxed); 139 | if cur_ptr.is_null() { 140 | return; 141 | } 142 | cur_slot += stride; 143 | cur_slot %= CACHE_SIZE; 144 | } 145 | ma.dealloc(NonNull::new_unchecked(cur_ptr), self.layout.clone()); 146 | } 147 | } 148 | 149 | } 150 | }; 151 | } 152 | sized_cache!(large, 8); 153 | sized_cache!(small, 4); 154 | 155 | #[cfg(test)] 156 | mod tests { 157 | use core::ptr::write_volatile; 158 | 159 | use alloc::alloc::GlobalAlloc; 160 | 161 | use super::*; 162 | 163 | #[test] 164 | fn test_alloc_basic() { 165 | for size in 1..((1 << 18) + 1) { 166 | let layout = Layout::from_size_align(size * 8, 8).unwrap(); 167 | unsafe { 168 | let item = (&BsAlloc).alloc(layout.clone()); 169 | write_volatile(item, 10); 170 | (&BsAlloc).dealloc(item, layout); 171 | } 172 | } 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /slab-alloc/src/init.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | use core::marker::PhantomData; 9 | use core::ptr::NonNull; 10 | 11 | /// A system to perform initialization of objects. 12 | /// 13 | /// Since slab allocators can either initialize their objects or not, we need a way of abstracting 14 | /// over the logic of keeping track of whether objects are initialized that can be completely 15 | /// optimized away if no initialization (and thus no tracking) is actually being done. `InitSystem` 16 | /// abstracts over the logic of keeping track of whether an object has been initialized and of 17 | /// initializing uninitialized objects. 18 | pub trait InitSystem { 19 | /// The initialization status of an object. 20 | /// 21 | /// For implementations that perform initialization, this is a type which keeps track of two 22 | /// states - initialized or uninitialized. For implementations that do not perform 23 | /// initialization, this is a ZST - it has only one state, and can be completely optimized out. 24 | type Status: Copy; 25 | 26 | fn status_initialized() -> Self::Status; 27 | fn status_uninitialized() -> Self::Status; 28 | 29 | /// The minimum alignment required to pack a status into a pointer. 30 | /// 31 | /// All pointers used with this object must guarantee this alignment, or else the behavior will 32 | /// be undefined. 33 | fn min_align() -> usize; 34 | 35 | /// Pack a `Status` and a pointer. 36 | /// 37 | /// `pack` packs a pointer and a status into a `usize`. `ptr` must be aligned to `min_align()`, 38 | /// or else the behavior of `pack` is undefined. 39 | fn pack(ptr: NonNull, status: Self::Status) -> usize; 40 | 41 | /// Unpack a `Status` from a packed `usize`. 42 | /// 43 | /// `unpack_status` unpacks a `Status` from `packed`. It should only be called on values that 44 | /// were previously returned from `pack`. 45 | fn unpack_status(packed: usize) -> Self::Status; 46 | 47 | /// Unpack a pointer from a packed `usize`. 48 | /// 49 | /// `unpack_ptr` unpacks a pointer from `packed`. It should only be called on values that were 50 | /// previously returned from `pack`. 51 | fn unpack_ptr(packed: usize) -> NonNull; 52 | 53 | /// Initialize an object. 54 | /// 55 | /// For implementations that perform initialization, `init` initializes `obj` if it is 56 | /// currently uninitialized. For implementations that do not perform initialization, `init` is 57 | /// a no-op. 58 | fn init(&self, obj: NonNull, init_status: Self::Status); 59 | 60 | /// Drop an object. 61 | /// 62 | /// For implementations that perform initialization, `drop` drops `obj` if it is currently 63 | /// initialized. For implementations that do not perform initialization, `drop` is a no-op. 64 | fn drop(obj: NonNull, init_status: Self::Status); 65 | } 66 | 67 | pub struct NopInitSystem; 68 | 69 | impl InitSystem for NopInitSystem { 70 | type Status = (); 71 | 72 | fn status_initialized() -> () { 73 | () 74 | } 75 | fn status_uninitialized() -> () { 76 | () 77 | } 78 | fn min_align() -> usize { 79 | 1 80 | } 81 | fn pack(ptr: NonNull, _status: ()) -> usize { 82 | ptr.as_ptr() as usize 83 | } 84 | fn unpack_status(_packed: usize) -> () { 85 | () 86 | } 87 | fn unpack_ptr(packed: usize) -> NonNull { 88 | unsafe { NonNull::new_unchecked(packed as *mut u8) } 89 | } 90 | fn init(&self, _obj: NonNull, _init_status: ()) {} 91 | fn drop(_obj: NonNull, _init_status: ()) {} 92 | } 93 | 94 | pub struct InitInitSystem> { 95 | init: I, 96 | _marker: PhantomData, 97 | } 98 | 99 | impl> InitInitSystem { 100 | pub fn new(init: I) -> InitInitSystem { 101 | InitInitSystem { 102 | init: init, 103 | _marker: PhantomData, 104 | } 105 | } 106 | } 107 | 108 | impl> InitSystem for InitInitSystem { 109 | type Status = bool; 110 | 111 | fn status_initialized() -> bool { 112 | true 113 | } 114 | fn status_uninitialized() -> bool { 115 | false 116 | } 117 | fn min_align() -> usize { 118 | 2 119 | } 120 | 121 | fn pack(ptr: NonNull, status: bool) -> usize { 122 | (ptr.as_ptr() as usize) | if status { 1 } else { 0 } 123 | } 124 | 125 | fn unpack_status(packed: usize) -> bool { 126 | packed & 1 == 1 127 | } 128 | 129 | fn unpack_ptr(packed: usize) -> NonNull { 130 | unsafe { NonNull::new_unchecked((packed & (::max_value() - 1)) as *mut u8) } 131 | } 132 | 133 | fn init(&self, obj: NonNull, init: bool) { 134 | if !init { 135 | unsafe { 136 | self.init.init(obj.cast()); 137 | } 138 | } 139 | } 140 | 141 | fn drop(obj: NonNull, init: bool) { 142 | if init { 143 | unsafe { 144 | use core::ptr::drop_in_place; 145 | drop_in_place::(obj.cast().as_ptr()); 146 | } 147 | } 148 | } 149 | } 150 | 151 | pub unsafe trait Initializer { 152 | unsafe fn init(&self, ptr: NonNull); 153 | } 154 | 155 | #[derive(Default)] 156 | pub struct DefaultInitializer { 157 | _marker: PhantomData, 158 | } 159 | 160 | impl DefaultInitializer { 161 | pub fn new() -> DefaultInitializer { 162 | DefaultInitializer { 163 | _marker: PhantomData, 164 | } 165 | } 166 | } 167 | 168 | unsafe impl Initializer for DefaultInitializer { 169 | unsafe fn init(&self, ptr: NonNull) { 170 | ::core::ptr::write(ptr.as_ptr(), T::default()); 171 | } 172 | } 173 | 174 | pub struct FnInitializer T>(F); 175 | 176 | impl T> FnInitializer { 177 | pub fn new(f: F) -> FnInitializer { 178 | FnInitializer(f) 179 | } 180 | } 181 | 182 | unsafe impl T> Initializer for FnInitializer { 183 | unsafe fn init(&self, ptr: NonNull) { 184 | ::core::ptr::write(ptr.as_ptr(), (self.0)()); 185 | } 186 | } 187 | 188 | pub struct UnsafeFnInitializer)>(F, PhantomData); 189 | 190 | impl)> UnsafeFnInitializer { 191 | pub fn new(f: F) -> UnsafeFnInitializer { 192 | UnsafeFnInitializer(f, PhantomData) 193 | } 194 | } 195 | 196 | unsafe impl)> Initializer for UnsafeFnInitializer { 197 | unsafe fn init(&self, ptr: NonNull) { 198 | (self.0)(ptr); 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /slab-alloc/src/large.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | // So clippy doesn't complain that IllumOS isn't in tick marks 9 | #![cfg_attr(feature = "cargo-clippy", allow(doc_markdown))] 10 | //! A slab that uses an allocator-global hash table to map objects to slabs. 11 | //! 12 | //! This module provides a slab implementation similar to Bonwick's original large slabs. Unlike 13 | //! aligned slabs, large slabs cannot rely on facts about memory layout to map object pointers to 14 | //! the slabs from which they were allocated. Instead, an allocator-global hash table is 15 | //! maintained. This is unfortunately somewhat slower than the aligned slab algorithm, but is 16 | //! necessary for objects which are too large to fit in an aligned slab. 17 | //! 18 | //! This implementation differs significantly from the original Bonwick algorithm in its allocation 19 | //! of metadata. In the Bonwick algorithm, each slab consists of a metadata structure which is 20 | //! allocated separately from the buffer used to back objects. Additionally, each object has its 21 | //! own separately-allocated metadata object (a `kmem_bufctl` in the IllumOS implementation). This 22 | //! implementation instead takes a simpler approach, storing slab metadata in an inline header and 23 | //! keeping track of available objects via a stack of pointers in the header - every large slab 24 | //! consists of a single allocated region of memory. For more details on this stack-based approach 25 | //! to keeping track of available objects, see the `stack` module. 26 | 27 | use alloc::alloc; 28 | use core::ptr::NonNull; 29 | use init::InitSystem; 30 | use object_alloc::UntypedObjectAlloc; 31 | use stack; 32 | use stack::{Layout, SlabHeader}; 33 | use util::ptrmap::*; 34 | use {OBJECTS_PER_SLAB, PAGE_ALIGN_MASK, PAGE_SIZE}; 35 | 36 | pub struct ConfigData { 37 | pub map: Map>, 38 | map_by_page_addr: bool, 39 | } 40 | 41 | impl stack::ConfigData for ConfigData { 42 | fn post_alloc(&mut self, layout: &Layout, slab_size: usize, slab: NonNull) { 43 | if self.map_by_page_addr { 44 | for i in 0..(slab_size / *PAGE_SIZE) { 45 | let page_ptr = ((slab.as_ptr() as usize) + (i * *PAGE_SIZE)) as *mut u8; 46 | debug_assert_eq!(page_ptr as usize % *PAGE_SIZE, 0); 47 | self.map.insert(page_ptr, slab); 48 | } 49 | } else { 50 | for i in 0..layout.num_obj { 51 | let ptr = layout.nth_obj(slab, unsafe { (*slab.as_ptr()).get_color() }, i); 52 | self.map.insert(ptr.as_ptr(), slab); 53 | } 54 | } 55 | } 56 | 57 | fn pre_dealloc(&mut self, layout: &Layout, slab_size: usize, slab: NonNull) { 58 | if self.map_by_page_addr { 59 | for i in 0..(slab_size / *PAGE_SIZE) { 60 | let page_ptr = ((slab.as_ptr() as usize) + (i * *PAGE_SIZE)) as *mut u8; 61 | debug_assert_eq!(page_ptr as usize % *PAGE_SIZE, 0); 62 | self.map.delete(page_ptr); 63 | } 64 | } else { 65 | for i in 0..layout.num_obj { 66 | let ptr = layout.nth_obj(slab, unsafe { (*slab.as_ptr()).get_color() }, i); 67 | self.map.delete(ptr.as_ptr()); 68 | } 69 | } 70 | } 71 | 72 | fn ptr_to_slab(&self, _slab_size: usize, ptr: NonNull) -> NonNull { 73 | if self.map_by_page_addr { 74 | self.map 75 | .get(((ptr.as_ptr() as usize) & *PAGE_ALIGN_MASK) as *mut u8) 76 | } else { 77 | self.map.get(ptr.as_ptr()) 78 | } 79 | } 80 | } 81 | 82 | pub type System = stack::System; 83 | 84 | pub const DEFAULT_MAP_SIZE: usize = 256; 85 | 86 | impl System { 87 | pub fn new(layout: alloc::Layout, alloc: A) -> Option> { 88 | if let Some((slab_layout, _)) = Layout::for_slab_size(layout.clone(), alloc.layout().size()) 89 | { 90 | let map_by_page_addr = layout.size() < *PAGE_SIZE; 91 | let map_key_align = if map_by_page_addr { 92 | *PAGE_SIZE 93 | } else { 94 | layout.size().next_power_of_two() 95 | }; 96 | Some(Self::from_config_data( 97 | ConfigData { 98 | map: Map::new(DEFAULT_MAP_SIZE, map_key_align), 99 | map_by_page_addr: map_by_page_addr, 100 | }, 101 | slab_layout, 102 | alloc, 103 | )) 104 | } else { 105 | None 106 | } 107 | } 108 | } 109 | 110 | pub fn backing_size_for(layout: &alloc::Layout) -> usize { 111 | struct PageIterator(usize); 112 | 113 | impl Iterator for PageIterator { 114 | type Item = usize; 115 | 116 | fn next(&mut self) -> Option { 117 | let (next, _) = self.0.overflowing_add(*PAGE_SIZE); 118 | if next > self.0 { 119 | self.0 = next; 120 | Some(next) 121 | } else { 122 | None 123 | } 124 | } 125 | } 126 | 127 | let unused = |slab_size: usize| { 128 | Layout::for_slab_size(layout.clone(), slab_size) 129 | .map(|(layout, unused)| (layout.num_obj, unused)) 130 | }; 131 | 132 | // pick a reasonable lower bound on slab size to avoid wasting unnecessary work on slab sizes 133 | // that are too small to accomodate even a single object; for large object sizes, this can save 134 | // a considerable amount of work (e.g., for 128M objects, counting up from 1 page would take 135 | // ~32K iterations) 136 | let init_size = if layout.size() % *PAGE_SIZE == 0 { 137 | layout.size() 138 | } else { 139 | // round down to a multiple of the page size 140 | (layout.size() / *PAGE_SIZE) * *PAGE_SIZE 141 | }; 142 | 143 | ::util::size::choose_size(PageIterator(init_size), unused, OBJECTS_PER_SLAB) 144 | } 145 | 146 | #[cfg(not(feature = "use-stdlib-hashmap"))] 147 | #[cfg(test)] 148 | mod tests { 149 | extern crate alloc; 150 | 151 | use self::alloc::alloc::Layout; 152 | use init::DefaultInitializer; 153 | use {DefaultInitSystem, SizedSlabAlloc}; 154 | 155 | fn test_hash_table_bucket_distribution() { 156 | for i in 0..4 { 157 | use backing::heap; 158 | use sysconf::page::pagesize; 159 | 160 | let layout = Layout::new::(); 161 | let backing_layout = Layout::from_size_align(pagesize(), pagesize()).unwrap(); 162 | let mut alloc = SizedSlabAlloc::new( 163 | DefaultInitSystem::::new(DefaultInitializer::new()), 164 | layout.clone(), 165 | super::System::new(layout, heap::get_large(backing_layout)).unwrap(), 166 | ); 167 | let mut ptrs = Vec::new(); 168 | 169 | let size = super::DEFAULT_MAP_SIZE << i; 170 | for _ in 0..size { 171 | ptrs.push(alloc.alloc().unwrap()); 172 | } 173 | let buckets = alloc.slab_system.data.map.map.dump_by_bucket(); 174 | for p in ptrs { 175 | alloc.dealloc(p); 176 | } 177 | 178 | eprintln!("Size: {}", size); 179 | for (i, b) in buckets.iter().enumerate() { 180 | eprintln!("{:>3}: {:?}", i, b); 181 | } 182 | eprintln!(""); 183 | } 184 | } 185 | 186 | macro_rules! make_test_hash_table_bucket_distribution { 187 | ($name:ident, $type:ty) => { 188 | #[test] 189 | #[ignore] 190 | fn $name() { 191 | test_hash_table_bucket_distribution::<$type>(); 192 | } 193 | }; 194 | } 195 | 196 | call_for_all_types_prefix!( 197 | make_test_hash_table_bucket_distribution, 198 | test_hash_table_bucket_distribution 199 | ); 200 | } 201 | -------------------------------------------------------------------------------- /elfmalloc/src/utils.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | //! Some basic utilities used throughout the allocator code. 9 | use std::cmp; 10 | use std::ops::{Deref, DerefMut}; 11 | use std::cell::UnsafeCell; 12 | 13 | pub mod mmap { 14 | use mmap_alloc::{MapAlloc, MapAllocBuilder}; 15 | use alloc::alloc::{Alloc, Layout}; 16 | 17 | lazy_static!{ 18 | static ref MMAP: MapAlloc = MapAllocBuilder::default() 19 | .commit(cfg!(windows)) 20 | .build(); 21 | } 22 | 23 | pub fn page_size() -> usize { 24 | ::sysconf::page::pagesize() 25 | } 26 | 27 | pub fn map(size: usize) -> *mut u8 { 28 | fallible_map(size).expect("mmap should not fail") 29 | } 30 | 31 | pub fn fallible_map(size: usize) -> Option<*mut u8> { 32 | unsafe { (&*MMAP).alloc(layout_for_size(size)).ok() } 33 | } 34 | 35 | pub unsafe fn unmap(p: *mut u8, size: usize) { 36 | (&*MMAP).dealloc(p, layout_for_size(size)); 37 | } 38 | 39 | pub unsafe fn commit(p: *mut u8, size: usize) { 40 | (&*MMAP).commit(p, layout_for_size(size)) 41 | } 42 | 43 | pub unsafe fn uncommit(p: *mut u8, size: usize) { 44 | (&*MMAP).uncommit(p, layout_for_size(size)); 45 | } 46 | 47 | fn layout_for_size(size: usize) -> Layout { 48 | Layout::from_size_align(size, page_size()).unwrap() 49 | } 50 | } 51 | 52 | // we use the unlikely intrinsic if it is available. 53 | 54 | #[cfg(feature = "nightly")] 55 | pub use std::intrinsics::{likely, unlikely}; 56 | 57 | #[cfg(not(feature = "nightly"))] 58 | #[cfg_attr(feature = "cargo-clippy", allow(inline_always))] 59 | #[inline(always)] 60 | pub unsafe fn unlikely(b: bool) -> bool { 61 | b 62 | } 63 | 64 | #[cfg(not(feature = "nightly"))] 65 | #[cfg_attr(feature = "cargo-clippy", allow(inline_always))] 66 | #[inline(always)] 67 | pub unsafe fn likely(b: bool) -> bool { 68 | b 69 | } 70 | 71 | /// A `LazyInitializable` type can be constructed from `Params`. 72 | /// 73 | /// Types that implement this trate can be wrapped in the `Lazy` construct. 74 | pub trait LazyInitializable { 75 | type Params; 76 | fn init(p: &Self::Params) -> Self; 77 | } 78 | 79 | /// A `Lazy` instance of a type `T` keeps `T::Params` strict but only initializes the value with 80 | /// `T::init` when it is first accessed. 81 | /// 82 | /// `Lazy` implements `Clone` if `T::Params` does. Note that this only clones the constructor 83 | /// parameters and not the object itself. These semantics are appropriate for their use in 84 | /// allocator size classes, but may be unintuitive elsewhere. It implements `deref` and `deref_mut` 85 | /// to facilitate access to the underlying object. 86 | pub struct Lazy { 87 | params: T::Params, 88 | val: UnsafeCell>, 89 | } 90 | 91 | impl Clone for Lazy 92 | where 93 | T::Params: Clone, 94 | { 95 | fn clone(&self) -> Self { 96 | Lazy { 97 | params: self.params.clone(), 98 | val: UnsafeCell::new(None), 99 | } 100 | } 101 | } 102 | 103 | impl Lazy { 104 | /// Create a new `Lazy` with constructor parameters given by `params`. 105 | pub fn new(params: T::Params) -> Self { 106 | Lazy { 107 | params: params, 108 | val: UnsafeCell::new(None), 109 | } 110 | } 111 | } 112 | 113 | impl Deref for Lazy { 114 | type Target = T; 115 | 116 | #[cfg_attr(feature = "cargo-clippy", allow(inline_always))] 117 | #[inline(always)] 118 | fn deref(&self) -> &T { 119 | let state = unsafe { &mut *self.val.get() }; 120 | if unsafe { unlikely(state.is_none()) } { 121 | *state = Some(T::init(&self.params)); 122 | } 123 | state.as_ref().unwrap() 124 | } 125 | } 126 | 127 | impl DerefMut for Lazy { 128 | #[cfg_attr(feature = "cargo-clippy", allow(inline_always))] 129 | #[inline(always)] 130 | fn deref_mut(&mut self) -> &mut T { 131 | let state = unsafe { &mut *self.val.get() }; 132 | if unsafe { unlikely(state.is_none()) } { 133 | *state = Some(T::init(&self.params)); 134 | } 135 | state.as_mut().unwrap() 136 | } 137 | } 138 | 139 | 140 | /// A low-level dynamic collection of `T` values. 141 | /// 142 | /// `TypedArray` uses mmap for memory allocation. This means that memory consumption from a 143 | /// `TypedArray` is lazy: the pages are only backed by physical memory after they are used. A 144 | /// `TypedArray` does not free its memory in a destructor (these semantics are required for the 145 | /// global allocator in the `general` module). To reclaim the memory used by the array, a `destroy` 146 | /// method is supplied. 147 | /// 148 | /// To use a `TypedArray` with a more traditional RAII-style destructor, use `OwnedArray`. 149 | pub struct TypedArray { 150 | // TODO: replace with non-null once that stabilizes. 151 | data: *mut T, 152 | len: usize, 153 | mapped: usize, 154 | } 155 | 156 | impl TypedArray { 157 | pub fn new(size: usize) -> TypedArray { 158 | use std::mem::size_of; 159 | let page_size = mmap::page_size(); 160 | let bytes = size_of::() * size; 161 | let rem = bytes % page_size; 162 | let n_pages = bytes / page_size + cmp::min(1, rem); 163 | let region_size = n_pages * page_size; 164 | let mem = mmap::map(region_size); 165 | TypedArray { 166 | data: mem as *mut T, 167 | len: size, 168 | mapped: region_size, 169 | } 170 | } 171 | 172 | pub fn iter(&self) -> TypedArrayIter { 173 | TypedArrayIter { 174 | inner: self, 175 | cur: 0, 176 | } 177 | } 178 | 179 | /// Get an index into the array. Unsafe because this operation is unchecked: it may provide a 180 | /// pointer out of bounds. 181 | pub unsafe fn get(&self, n: usize) -> *mut T { 182 | self.data.offset(n as isize) 183 | } 184 | 185 | pub fn len(&self) -> usize { 186 | self.len 187 | } 188 | 189 | pub unsafe fn destroy(&self) { 190 | mmap::unmap(self.data as *mut u8, self.mapped); 191 | } 192 | } 193 | 194 | /// A variant of `TypedArray` that unmaps its memory during `drop`. 195 | pub struct OwnedArray(TypedArray); 196 | 197 | impl OwnedArray { 198 | pub fn new(size: usize) -> OwnedArray { 199 | OwnedArray(TypedArray::new(size)) 200 | } 201 | } 202 | 203 | impl Deref for OwnedArray { 204 | type Target = TypedArray; 205 | fn deref(&self) -> &TypedArray { 206 | &self.0 207 | } 208 | } 209 | 210 | impl Drop for OwnedArray { 211 | fn drop(&mut self) { 212 | unsafe { self.destroy() } 213 | } 214 | } 215 | 216 | pub struct TypedArrayIter<'a, T: 'a> { 217 | inner: &'a TypedArray, 218 | cur: usize, 219 | } 220 | 221 | impl<'a, T: 'a> Iterator for TypedArrayIter<'a, T> { 222 | type Item = *mut T; 223 | fn next(&mut self) -> Option<*mut T> { 224 | if self.cur == self.inner.len { 225 | None 226 | } else { 227 | unsafe { 228 | let res = self.inner.data.offset(self.cur as isize); 229 | self.cur += 1; 230 | Some(res) 231 | } 232 | } 233 | } 234 | } 235 | 236 | #[cfg(test)] 237 | mod tests { 238 | #[derive(Debug)] 239 | struct DefaultInit(T); 240 | impl LazyInitializable for DefaultInit { 241 | type Params = (); 242 | fn init(_p: &()) -> Self { 243 | DefaultInit(T::default()) 244 | } 245 | } 246 | 247 | use super::*; 248 | #[test] 249 | fn basic_functionality() { 250 | let mut l = Lazy::>::new(()); 251 | let l_u = l.0; 252 | alloc_assert_eq!(l_u, 0); 253 | *l = DefaultInit(1); 254 | let l_u = l.0; 255 | alloc_assert_eq!(l_u, 1); 256 | } 257 | 258 | } 259 | -------------------------------------------------------------------------------- /info/bagpipes.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # Bagpipes: Scalable Concurrent Bags 9 | 10 | Bagpipes are a simple data-structure that relies on common structure present in 11 | non-blocking queues to provide a more scalable queue-like data-structure with 12 | weaker ordering guarantees. We use them in the `elfmalloc` allocator to act as 13 | repositories for slabs of memory. The scalability of this data-structure allows 14 | more memory to be stored in them, as opposed to in thread-local caches. Our 15 | approach here bares some resemblance to [`scalloc`'s](https://arxiv.org/pdf/1503.09006.pdf) 16 | *span pool* data-structure, itself inspired by the [Distributed Queues](https://www.semanticscholar.org/paper/Distributed-queues-in-shared-memory-multicore-perf-Haas-Lippautz/3c1e0e9c5b774f8d1b7522e7b7ea90634b1e252a) 17 | of Haas et al. 18 | 19 | The primary difference between our approach and theirs is: 20 | 21 | * We leverage more scalable fetch-add based queues as the basis of the 22 | data-structure. 23 | 24 | * We take a different approach to load-balancing that is lower-overhead at the 25 | cost of FIFO-like consistency. 26 | 27 | ## Concurrent queues with transient failures 28 | The basis of a Bagpipe is some non-blocking data-structure with `push` and `pop` 29 | methods. While either stacks or queues work for our purposes, most of the time 30 | we instantiate Bagpipes with queues as they currently perform better in 31 | practice. The only other structure that we rely on is some notion of *transient 32 | failure* that can be attributed to contention among different threads 33 | interacting with the data-structure. One common example is a CAS failure in a 34 | non-blocking queue, or a failed call to `try_lock` in a blocking setting. This 35 | structure is encoded in the `SharedWeakBag` trait. 36 | 37 | ```rust 38 | pub enum PopStatus { 39 | Empty, 40 | TransientFailure, 41 | } 42 | pub type PopResult = Result; 43 | 44 | pub trait SharedWeakBag { 45 | type Item; 46 | fn new() -> Self; 47 | fn try_push(&self, it: Self::Item) -> Result<(), Self::Item>; 48 | fn try_pop(&self) -> PopResult; 49 | fn push(&self, it: Self::Item) { 50 | let mut cur_item = it; 51 | while let Err(old_item) = self.try_push(cur_item) { 52 | cur_item = old_item 53 | } 54 | } 55 | fn pop(&self) -> Option { 56 | loop { 57 | return match self.try_pop() { 58 | Ok(it) => Some(it), 59 | Err(PopStatus::Empty) => None, 60 | Err(PopStatus::TransientFailure) => continue, 61 | }; 62 | } 63 | } 64 | } 65 | ``` 66 | 67 | ### Recent queues of interest 68 | 69 | Over the past few years, a number of queue designs have been proposed that 70 | improve upon the standard non-blocking queue design of [Michael and 71 | Scott](https://www.research.ibm.com/people/m/michael/podc-1996.pdf). Instead of 72 | designs where enqueuers and dequeuers contend on a single head or tail node in a 73 | CAS loop, these algorithms instead use the atomic `fetch-add` instruction to 74 | acquire a cell in the array to enqueue or dequeue a particular node. To our 75 | knowledge, the first queue with this basic design was [Morrison and Afek 76 | (2013)](http://www.cs.tau.ac.il/~mad/publications/ppopp2013-x86queues.pdf). 77 | While the queue in this paper outperforms the ones we currently use, it relies 78 | on a double-word CAS instruction. This makes the queue both non-portable, and 79 | hard to implement in Rust, as that primitive is not exposed in the `atomic` 80 | package at the moment. 81 | 82 | Still, there are two queues that perform almost as well that do not rely on this 83 | instruction: 84 | 85 | * An implementation of the obstruction-free algorithm alluded to in Morrison and 86 | Afek's original paper, based on the "fast path" in [Yang-Muller and Crummy 87 | (2016)](http://chaoran.me/assets/pdf/wfq-ppopp16.pdf). 88 | 89 | * An implementation of Concurrency Freaks' [Fetch-add Array Queue](http://concurrencyfreaks.blogspot.com/2016/11/faaarrayqueue-mpmc-lock-free-queue-part.html) 90 | 91 | Both of these queues can exhibit transient failures if a receiver reaches a cell 92 | before a matching sender. 93 | 94 | ## Throwing away consistency 95 | 96 | Following work like [`scalloc` (Aigner et al)](https://arxiv.org/abs/1503.09006), 97 | we store reusable slabs of objects in global data-structures in order to 98 | minimize unused memory in thread-local caches. We aim to decrease the size of 99 | caches even further than `scalloc` does (indeed, it is possible to configure 100 | `elfmalloc` with no cache whatsoever beyond a single slab). The queues listed 101 | above are quite scalable, but they do eventually reach bottlenecks due to the 102 | limitations of the fetch-add primitive in use. To increase scalability, we 103 | simply throw away the consistency guarantees (in the above case, 104 | linearizability) of the queue in question. 105 | 106 | It is normally a dubious move to throw away linearizability. Indeed, there is a 107 | reasonable case to be made that a concurrent queue that is not linearizable is 108 | not really a correct (FIFO) queue at all. However, we do not not require FIFO 109 | semantics from our queues, we simply require that they do not lose their 110 | contents. 111 | 112 | ## Revocation 113 | 114 | One extra operation that we require in `elfmalloc` is the ability to remove a 115 | node that is somewhere in the queue. To do this, certain `Revocable` types hold 116 | handles that can store a pointer into the queue cell in which they are pushed. 117 | Once in place, a very fast `revoke` operation can be performed to CAS this cell 118 | to a sentinel value: 119 | 120 | ```rust 121 | pub trait Revocable { fn handle(&self) -> &AtomicUsize; } 122 | pub trait RevocableWeakBag: SharedWeakBag 123 | where Self::Item: Revocable 124 | { 125 | /// Attempt to remove `it` from the bag, returning `true` if successful. 126 | /// 127 | /// This operation is unsafe because the underlying implementation may assume that `it` is (or 128 | /// was) successfully pushed into the bag at some point. 129 | unsafe fn revoke(it: &Self::Item) -> bool; 130 | } 131 | ``` 132 | 133 | ### Bagpipes 134 | 135 | A `Bagpipe` is simply a data-structure that holds an array of `SharedWeakBag`s. 136 | Threads interacting with a `Bagpipe` traverse random permutations of this array, 137 | calling `try_pop` and `try_push` on each individual "pipe" they encounter. The 138 | semantics of the *try* methods from the `SharedWeakBag` trait are such that 139 | threads only stop pushing to or popping from queues that are under load. This 140 | allows for some light-weight load-balancing of the queues without any additional 141 | coordination required. 142 | 143 | This structure also allows threads to bail out early if they are preempted too 144 | often. In this case, the thread can use some fallback mechanism in the allocator. 145 | 146 | ## Performance 147 | 148 | The queues that a `Bagpipe` wraps are both structured as linked lists of arrays 149 | of cells. These arrays are referred to as *segments*. The precise performance of 150 | different `Bagpipes` is highly sensitive to the size of these segments. On our 151 | current test machine with 32 threads, enqueue-dequeue pairs for a `Bagpipe` 152 | backed by a fetch-add array queue had throughput of 100Mops/second. By contrast, 153 | a Trieber stack had 7Mops/second and a single array queue had a throughput of 154 | 25Mops/s. `Bagpipe`s do not perform quite as well in a producer-consumer 155 | setting, but they still outperform any other queue by over 2x. 156 | 157 | Better performance of these data-structures is possible at the cost of increased 158 | memory usage. While they are currently configured for the more memory-cautious 159 | environment of an allocator, we plan to make them configurable to optimize for 160 | throughput, either via feature flags or type parameters. Anecdotally, I believe 161 | I got the Yang Crummey-backed `Bagpipe` to get 200Mops/s, but that required 162 | 64KiB segments, increasing steady-state memory consumption by an order of 163 | magnitude. 164 | 165 | The benchmark code is in the `bagpipe/src/bin/bench_bag.rs` directory. 166 | -------------------------------------------------------------------------------- /elfmalloc/src/sources.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | //! Low-level data-structures for getting more memory from the system. 9 | use std::sync::Arc; 10 | use std::sync::atomic::{AtomicUsize, AtomicPtr, Ordering}; 11 | use std::mem; 12 | use super::utils::{likely, mmap}; 13 | 14 | use alloc::alloc::{Alloc, Layout}; 15 | use mmap_alloc::MapAlloc; 16 | 17 | /// A generator of chunks of memory providing an `sbrk`-like interface. 18 | pub trait MemorySource 19 | where 20 | Self: Clone, 21 | { 22 | fn new(page_size: usize) -> Self; 23 | /// The smallest unit of memory that can be `carve`d. 24 | fn page_size(&self) -> usize; 25 | /// Return `npages` fresh pages from the `Creek`. Each of these pages is aligned to 26 | /// `page_size`. 27 | /// 28 | /// Currently, there is code (see the `Coalescer` in the `slag` module) that relies on fresh 29 | /// pages returned from `carve` to be filled with zeros. 30 | fn carve(&self, npages: usize) -> Option<*mut u8>; 31 | } 32 | 33 | /// A `MemorySource` that can tell which pointers lie in a region returned by `carve`. 34 | pub trait MemoryBlock: MemorySource { 35 | /// Is `it` a pointer to somewhere in the block of memory. 36 | fn contains(&self, it: *mut u8) -> bool; 37 | } 38 | 39 | 40 | /// A `MemorySource` that just calls mmap. It still maintains that all pages returned by `carve` 41 | /// are aligned to their size. 42 | #[derive(Copy, Clone)] 43 | pub struct MmapSource { 44 | page_size: usize, 45 | } 46 | 47 | unsafe impl Send for MmapSource {} 48 | 49 | impl MemorySource for MmapSource { 50 | fn new(page_size: usize) -> MmapSource { 51 | MmapSource { page_size: page_size.next_power_of_two() } 52 | } 53 | fn page_size(&self) -> usize { 54 | self.page_size 55 | } 56 | 57 | fn carve(&self, npages: usize) -> Option<*mut u8> { 58 | trace!("carve({:?})", npages); 59 | // Even if self.page_size is larger than the system page size, that's OK 60 | // because we're using mmap-alloc's large-align feature, which allows 61 | // allocations to be aligned to any alignment, even larger than the page size. 62 | let layout = Layout::from_size_align(npages * self.page_size, self.page_size).unwrap(); 63 | unsafe { MapAlloc::default().alloc(layout).ok() } 64 | } 65 | } 66 | 67 | /// Base address and size of a memory map. 68 | /// 69 | /// This could also just be a `*mut [u8]`, but having two fields is more explicit. We need a new 70 | /// type because the `Drop` implementation calls `unmap`. 71 | #[derive(Debug)] 72 | struct MapAddr(*mut u8, usize); 73 | 74 | impl Drop for MapAddr { 75 | fn drop(&mut self) { 76 | unsafe { 77 | mmap::unmap(self.0, self.1); 78 | } 79 | } 80 | } 81 | 82 | /// A large, contiguous, memory-mapped region of memory. 83 | /// 84 | /// A `Creek` can be seen as a very basic memory allocator that can hand back multiples of its 85 | /// `page_size`. While it does not implement `free`, the programmer can still call `uncommit` or 86 | /// `unmap` on pages that are returned from a `Creek`. In this module, the `Creek` is used to 87 | /// manage the (thread-safe) introduction of fresh (clean) pages into the rest of the allocator to 88 | /// be used. 89 | /// 90 | /// `Creek`s are initialized with a maximum size and never grow beyond that size. They are made 91 | /// with a particular idiom in mind in which a larger-than-physical-memory mapping is requested for 92 | /// the `Creek`, only a fraction of which is ever used by the running program (hence, ever backed 93 | /// by physical page frames). 94 | #[derive(Debug)] 95 | pub struct Creek { 96 | page_size: usize, 97 | /// We use a `clone`-based interface in order to facilitate passing across threads and 98 | /// leveraging `Arc` to call `unmap`. 99 | map_info: Arc, 100 | base: *mut u8, 101 | bump: AtomicPtr, 102 | } 103 | 104 | unsafe impl Send for MapAddr {} 105 | unsafe impl Sync for MapAddr {} 106 | unsafe impl Send for Creek {} 107 | unsafe impl Sync for Creek {} 108 | 109 | macro_rules! check_bump { 110 | ($slf:expr) => { 111 | #[cfg(debug_assertions)] 112 | { 113 | let bump = $slf.bump.load(Ordering::Relaxed); 114 | alloc_debug_assert!(!bump.is_null()); 115 | } 116 | }; 117 | } 118 | 119 | impl MemorySource for Creek { 120 | #[inline] 121 | fn page_size(&self) -> usize { 122 | check_bump!(self); 123 | self.page_size 124 | } 125 | 126 | fn carve(&self, npages: usize) -> Option<*mut u8> { 127 | check_bump!(self); 128 | unsafe { 129 | let new_bump = self.bump 130 | .load(Ordering::Relaxed) 131 | .as_ref() 132 | .unwrap() 133 | .fetch_add(npages, Ordering::Relaxed); 134 | if likely((new_bump + npages) * self.page_size < self.map_info.1) { 135 | Some(self.base.offset((new_bump * self.page_size) as isize)) 136 | } else { 137 | None 138 | } 139 | } 140 | } 141 | 142 | 143 | /// Create a new `Creek` with pages of size `page_size` total heap size of `heap_size`, 144 | /// optionally backed by huge pages. 145 | /// 146 | /// Page size and heap size should be powers of two. The allocator may want to reserve some 147 | /// pages for itself (or for alignment reasons), as a result it is a good idea to have 148 | /// heap_size be much larger than page_size. 149 | fn new(page_size: usize) -> Self { 150 | use self::mmap::fallible_map; 151 | let get_heap = || { 152 | let mut heap_size: usize = 2 << 40; 153 | while heap_size > (1 << 30) { 154 | if let Some(heap) = fallible_map(heap_size) { 155 | return (heap, heap_size); 156 | } 157 | heap_size /= 2; 158 | } 159 | alloc_panic!("unable to map heap") 160 | }; 161 | // lots of stuff breaks if this isn't true 162 | alloc_assert!(page_size.is_power_of_two()); 163 | alloc_assert!(page_size > mem::size_of::()); 164 | // first, let's grab some memory; 165 | let (orig_base, heap_size) = get_heap(); 166 | info!("created heap of size {}", heap_size); 167 | let orig_addr = orig_base as usize; 168 | let (slush_addr, real_addr) = { 169 | // allocate some `slush` space at the beginning of the creek. This gives us space to 170 | // store the `bump` pointer. In the future, we may store more things in this slush 171 | // space as well. 172 | // 173 | // In addition, we must ensure that pages are aligned to their size. 174 | let base = if orig_addr == 0 { 175 | // this is a real possibility if we are calling `mmap` directly. 176 | // However, `MmapAlloc` currently handles `mmap` returning null, so this is 177 | // technically a redundant check. 178 | orig_addr + page_size 179 | } else if orig_addr % page_size != 0 { 180 | let rem = orig_addr % page_size; 181 | orig_addr + (page_size - rem) 182 | } else { 183 | orig_addr 184 | }; 185 | (base as *mut u8, (base + page_size) as *mut u8) 186 | }; 187 | Creek { 188 | page_size: page_size, 189 | map_info: Arc::new(MapAddr(orig_base, heap_size)), 190 | base: real_addr, 191 | bump: AtomicPtr::new(slush_addr as *mut AtomicUsize), 192 | } 193 | } 194 | } 195 | 196 | impl MemoryBlock for Creek { 197 | fn contains(&self, it: *mut u8) -> bool { 198 | check_bump!(self); 199 | let it_num = it as usize; 200 | let base_num = self.base as usize; 201 | it_num >= base_num && it_num < base_num + self.map_info.1 202 | } 203 | } 204 | 205 | impl Clone for Creek { 206 | fn clone(&self) -> Self { 207 | let bump = self.bump.load(Ordering::Relaxed); 208 | alloc_debug_assert!(!bump.is_null()); 209 | Creek { 210 | page_size: self.page_size, 211 | map_info: self.map_info.clone(), 212 | base: self.base, 213 | bump: AtomicPtr::new(bump), 214 | } 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /elfmalloc/src/bin/bench_vec.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | #![feature(alloc)] 9 | #![feature(log_syntax)] 10 | #![feature(allocator_api)] 11 | #![feature(test)] 12 | extern crate alloc; 13 | extern crate elfmalloc; 14 | extern crate num_cpus; 15 | extern crate smallvec; 16 | extern crate test; 17 | use test::stats::Stats; 18 | use elfmalloc::rust_alloc::SharedAlloc; 19 | use elfmalloc::vec_alloc::AVec; 20 | use alloc::heap::Heap; 21 | use smallvec::VecLike; 22 | 23 | use std::thread; 24 | use std::time; 25 | use std::sync::{Arc, Barrier}; 26 | 27 | /// A basic resumable timer type used for benchmark time measurements. 28 | struct Timer { 29 | start: time::Instant, 30 | elapsed: time::Duration, 31 | } 32 | 33 | impl Default for Timer { 34 | fn default() -> Timer { 35 | Timer { 36 | start: time::Instant::now(), 37 | elapsed: time::Duration::default(), 38 | } 39 | } 40 | } 41 | 42 | impl Timer { 43 | fn new() -> Self { 44 | Timer::default() 45 | } 46 | fn reset(&mut self) { 47 | *self = Timer::new(); 48 | } 49 | fn stop(&mut self) { 50 | let cur_elapsed = self.start.elapsed(); 51 | self.elapsed += cur_elapsed; 52 | } 53 | fn resume(&mut self) { 54 | self.start = time::Instant::now(); 55 | } 56 | fn elapsed_ns(&self) -> u64 { 57 | let dur = self.start.elapsed() + self.elapsed; 58 | dur.as_secs() * 1_000_000_000 + u64::from(dur.subsec_nanos()) 59 | } 60 | } 61 | 62 | 63 | /// Run a workload given set parameters in parallel, timing the execution of each thread. 64 | /// 65 | /// This function returns an `f64` because `[f64]` has some pre-written statistics computationss 66 | /// from the `test` crate that we can reuse. 67 | fn run_parallel_bench( 68 | params: A, 69 | threads: usize, 70 | work: fn(A, &mut Timer) -> Vec, 71 | ) -> Vec { 72 | let b = Arc::new(Barrier::new(threads + 1)); 73 | let threads: Vec<_> = (0..threads) 74 | .map(|_| { 75 | let p = params.clone(); 76 | let b = b.clone(); 77 | thread::spawn(move || { 78 | b.wait(); 79 | let mut t = Timer::new(); 80 | work(p, &mut t) 81 | }) 82 | }) 83 | .collect(); 84 | b.wait(); 85 | let mut res = Vec::new(); 86 | for t in threads.into_iter().map(|j| { 87 | j.join().expect("threads should exit successfully") 88 | }) 89 | { 90 | res.extend(t) 91 | } 92 | res 93 | } 94 | 95 | fn format_dur(mut nsecs: f64) -> String { 96 | if nsecs >= 1_000_000_000f64 { 97 | nsecs /= 1_000_000_000f64; 98 | format!("{:.03} s", nsecs) 99 | } else if nsecs >= 1_000_000f64 { 100 | nsecs /= 1_000_000f64; 101 | format!("{:.03} ms", nsecs) 102 | } else if nsecs >= 1_000f64 { 103 | nsecs /= 1_000f64; 104 | format!("{:.03} μs", nsecs) 105 | } else { 106 | format!("{:.03} ns", nsecs) 107 | } 108 | } 109 | 110 | 111 | /// Create a benchmark function based around `run_parallel_bench`, including printing benchmark 112 | /// output. This is in a macro because we have to define a full `fn` in order to send the function 113 | /// across thread boundaries and clone it. 114 | macro_rules! create_bench { 115 | 116 | ($name:ident, $desc:expr, $param:tt ::: $pty:ty = $pval:expr, $timer:ident, $nthr:expr, $iters:expr, $work:expr) => { 117 | fn $name() { 118 | fn inner($param: $pty, $timer: &mut Timer) -> Vec { 119 | // warm up round; 120 | let p = $work; 121 | let _ = test::black_box(p); 122 | let mut __samples = Vec::with_capacity($iters); 123 | for _ in 0..$iters { 124 | $timer.reset(); 125 | let p = $work; 126 | let _ = test::black_box(p); 127 | __samples.push($timer.elapsed_ns() as f64); 128 | } 129 | __samples 130 | } 131 | let params = $pval; 132 | let nthr = $nthr; 133 | let res = run_parallel_bench(params, nthr, inner); 134 | let stats = &res[..]; 135 | println!("benchmark-n{:02} {:40} {:12} per iteration (+/- {:.02}%)", nthr, $desc, 136 | format_dur(stats.mean()), 137 | stats.median_abs_dev_pct()); 138 | } 139 | }; 140 | } 141 | 142 | /// Group a number of benchmarks created using `create_bench` for three `Vec`-like types. 143 | macro_rules! bench_group { 144 | ($name:ident, $param:tt ::: $pty:ty = $pval:expr, $iters:expr, $fn:tt) => { 145 | fn $name() { 146 | create_bench!(v1, 147 | format!("{}_vec", stringify!($name)), 148 | $param ::: $pty = $pval, 149 | _t, 150 | 1, 151 | $iters, 152 | $fn::>($param, _t)); 153 | create_bench!(v1n, 154 | format!("{}_vec", stringify!($name)), 155 | $param ::: $pty = $pval, 156 | _t, 157 | num_cpus::get(), 158 | $iters, 159 | $fn::>($param, _t)); 160 | create_bench!(v2, 161 | format!("{}_avec_heap", stringify!($name)), 162 | $param ::: $pty = $pval, 163 | _t, 164 | 1, 165 | $iters, 166 | $fn::>($param, _t)); 167 | create_bench!(v2n, 168 | format!("{}_avec_heap", stringify!($name)), 169 | $param ::: $pty = $pval, 170 | _t, 171 | num_cpus::get(), 172 | $iters, 173 | $fn::>($param, _t)); 174 | create_bench!(v3, 175 | format!("{}_avec_elf", stringify!($name)), 176 | $param ::: $pty = $pval, 177 | _t, 178 | 1, 179 | $iters, 180 | $fn::>($param, _t)); 181 | create_bench!(v3n, 182 | format!("{}_avec_elf", stringify!($name)), 183 | $param ::: $pty = $pval, 184 | _t, 185 | num_cpus::get(), 186 | $iters, 187 | $fn::>($param, _t)); 188 | 189 | v3(); 190 | v3n(); 191 | v1(); 192 | v1n(); 193 | v2(); 194 | v2n(); 195 | } 196 | }; 197 | } 198 | 199 | #[inline(never)] 200 | fn push_noinline>(v: &mut V, t: T) { 201 | v.push(t) 202 | } 203 | 204 | /// A benchmark testing smaller-sized allocations. 205 | fn do_push + Default>(ops: usize, _timer: &mut Timer) -> V { 206 | let mut v = V::default(); 207 | for _ in 0..8 { 208 | for i in 0..ops { 209 | push_noinline(&mut v, i); 210 | } 211 | v = V::default(); 212 | } 213 | v 214 | } 215 | 216 | /// A benchmark stressing medium-sized allocations. 217 | fn do_push_medium + Default>(ops: usize, timer: &mut Timer) -> V { 218 | timer.stop(); 219 | let mut v = V::default(); 220 | v.extend(1..(1 << 14)); 221 | timer.resume(); 222 | for i in 0..ops { 223 | push_noinline(&mut v, i); 224 | } 225 | v 226 | } 227 | 228 | /// A benchmark stressing large-sized allocations. 229 | fn do_push_large + Default>(ops: usize, _timer: &mut Timer) -> V { 230 | let mut v = V::default(); 231 | for i in 0..ops { 232 | // noinline may cause overflow issues 233 | v.push([i; 1024]); 234 | } 235 | v 236 | } 237 | 238 | bench_group!(bench_push, ops ::: usize = 500, 250_000, do_push); 239 | bench_group!(bench_push_medium, ops ::: usize = 500_000, 100, do_push_medium); 240 | bench_group!(bench_push_large, ops ::: usize = 1_000, 50, do_push_large); 241 | 242 | fn main() { 243 | bench_push(); 244 | bench_push_medium(); 245 | bench_push_large(); 246 | } 247 | -------------------------------------------------------------------------------- /bagpipe/src/bag.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | //! Specification of best-effort bags and implementation for `crossbeam` 9 | //! data-structures. 10 | use super::crossbeam::sync::{TreiberStack, SegQueue, MsQueue}; 11 | use super::crossbeam::mem::epoch; 12 | use std::sync::atomic::AtomicUsize; 13 | use std::sync::Arc; 14 | 15 | /// Ways that operations on a `SharedWeakBag` or `WeakBag` can fail. 16 | /// 17 | /// We permit `push` and `pop` operations to exhibit transient failures. 18 | pub enum PopStatus { 19 | Empty, 20 | TransientFailure, 21 | } 22 | 23 | pub type PopResult = Result; 24 | 25 | 26 | /// A best-effort Bag data-structure. 27 | /// 28 | /// As embodied in the `PopResult` definition, `try_pop` is permitted to 29 | /// fail even if the bag in question is not empty. 30 | pub trait SharedWeakBag { 31 | type Item; 32 | /// Returns a new instance of the data-structure. 33 | fn new() -> Self; 34 | 35 | /// Attempts to push `it` onto the data-structure. 36 | /// 37 | /// If successful, `try_push` will return `true`. 38 | fn try_push(&self, it: Self::Item) -> Result<(), Self::Item>; 39 | 40 | /// Attempts to pop a value from the data-structure. 41 | /// 42 | /// There is no guaranteed ordering of popped values. This method 43 | /// may fail arbitrarily even if there are accessible values in the 44 | /// data-structure. 45 | fn try_pop(&self) -> PopResult; 46 | 47 | /// A push operation that will not fail. 48 | /// 49 | /// The default implementation of `push` simply calls `try_push` 50 | /// in a loop. until it succeeds. Depending on the underlying 51 | /// data-structure this may loop infinitely under some 52 | /// circumstances. 53 | /// 54 | /// `push` also creates a `Guard` for the duration of the function 55 | /// to avoid excessive checking in the hot loop. 56 | fn push(&self, it: Self::Item) { 57 | let _g = epoch::pin(); 58 | let mut cur_item = it; 59 | while let Err(old_item) = self.try_push(cur_item) { 60 | cur_item = old_item 61 | } 62 | } 63 | 64 | /// A pop operation that will not fail. 65 | /// 66 | /// Same caveats apply to those of `push`. 67 | fn pop(&self) -> Option { 68 | let _g = epoch::pin(); 69 | loop { 70 | return match self.try_pop() { 71 | Ok(it) => Some(it), 72 | Err(PopStatus::Empty) => None, 73 | Err(PopStatus::TransientFailure) => continue, 74 | }; 75 | } 76 | } 77 | 78 | fn debug(&self) {} 79 | } 80 | 81 | /// An `Arc`-style variant of `SharedWeakBag`. 82 | /// 83 | /// This gives implementations the freedom of modifying mutable 84 | /// local metadata. Any `SharedWeakBag` is also a `WeakBag` if 85 | /// behind an `Arc`. Methods on `WeakBag` have the same semantics as 86 | /// `SharedWeakbag` except that the `try...` methods are permitted to 87 | /// modify any thread-local state. 88 | pub trait WeakBag: Clone { 89 | // TODO(ezrosent): should we keep Clone here? 90 | type Item; 91 | // fn new() -> Self; 92 | fn try_push_mut(&mut self, Self::Item) -> Result<(), Self::Item>; 93 | fn try_pop_mut(&mut self) -> PopResult; 94 | fn push_mut(&mut self, it: Self::Item) { 95 | let _g = epoch::pin(); 96 | let mut cur_item = it; 97 | while let Err(old_item) = self.try_push_mut(cur_item) { 98 | cur_item = old_item 99 | } 100 | } 101 | fn pop_mut(&mut self) -> Option { 102 | let _g = epoch::pin(); 103 | loop { 104 | return match self.try_pop_mut() { 105 | Ok(it) => Some(it), 106 | Err(PopStatus::Empty) => None, 107 | Err(PopStatus::TransientFailure) => continue, 108 | }; 109 | } 110 | } 111 | 112 | /// Add all items in `I` to the `WeakBag`. 113 | /// 114 | /// This allows data-structures to optimize bulk-add operations if 115 | /// possible. 116 | fn bulk_add>(&mut self, i: I) { 117 | for it in i { 118 | self.push_mut(it) 119 | } 120 | } 121 | } 122 | 123 | pub struct ArcLike(Arc); 124 | 125 | impl Clone for ArcLike { 126 | fn clone(&self) -> Self { 127 | ArcLike(self.0.clone()) 128 | } 129 | } 130 | 131 | impl Default for ArcLike { 132 | fn default() -> Self { ArcLike(Arc::new(B::new())) } 133 | } 134 | 135 | impl WeakBag for ArcLike { 136 | type Item = B::Item; 137 | fn try_push_mut(&mut self, it: Self::Item) -> Result<(), Self::Item> { 138 | self.0.try_push(it) 139 | } 140 | fn try_pop_mut(&mut self) -> PopResult { 141 | self.0.try_pop() 142 | } 143 | } 144 | 145 | 146 | /// Types that can revoke their membership in a `RevocableWeakBag`. 147 | /// 148 | /// This is a fairly low-level interface; most of the time it should not be needed. There are also 149 | /// some performance pitfalls in the way it is implemented in, e.g., the `FAAQueueLowLevel` 150 | /// data-structure. In that case, it works by pointing the value of `handle` to the cell in which a 151 | /// value is stored. Revocation is therefore simply a compare-and-swap operation on this value, 152 | /// attempting to change it to the "poison" sentinel value. If this happens infrequently it is 153 | /// likely fine. However, excessive calls to `revoke` will lead to `pop` operations slowing down 154 | /// because they must skip over poisoned cells. 155 | pub trait Revocable { 156 | /// A reference to an `AtomicUsize` value that can be used by a `revoke` implementation. 157 | /// 158 | /// The intended use for this is to set aside a word of memory in `Self` to hold a reference to 159 | /// its location in a `SharedWeakBag`. That way, the underlying data-structure can revoke 160 | /// membership with a single CAS. 161 | /// 162 | /// This interface is low-level; it may change depending on use and demand for queues with this 163 | /// feature. 164 | fn handle(&self) -> &AtomicUsize; 165 | } 166 | 167 | 168 | // This is a code smell. The reason why it is here is to allow for custom 169 | // revocable types to be revocable even when they are stored as raw pointers in one of the "low 170 | // level" bags. 171 | 172 | impl Revocable for *mut T { 173 | fn handle(&self) -> &AtomicUsize { 174 | unsafe { 175 | self.as_ref() 176 | .expect("revocable impl dereferences raw pointers") 177 | .handle() 178 | } 179 | } 180 | } 181 | 182 | /// A `SharedWeakBag` that can attempt to revoke `push` operations. 183 | pub trait RevocableWeakBag: SharedWeakBag 184 | where 185 | Self::Item: Revocable, 186 | { 187 | /// Attempt to remove `it` from the bag, returning `true` if successful. 188 | /// 189 | /// This operation is unsafe because the underlying implementation may assume that `it` is (or 190 | /// was) successfully pushed into the bag at some point. 191 | unsafe fn revoke(it: &Self::Item) -> bool; 192 | } 193 | 194 | // implement WeakBag for the stack and queues in crossbeam. Note that these don't have the full 195 | // "try" semantics that we want, as they never fail. As a result, they should not be used in a 196 | // `BagPipe`: everything will work, but 197 | 198 | impl SharedWeakBag for TreiberStack { 199 | type Item = T; 200 | fn new() -> Self { 201 | Self::new() 202 | } 203 | fn try_push(&self, t: T) -> Result<(), T> { 204 | self.push(t); 205 | Ok(()) 206 | } 207 | fn try_pop(&self) -> PopResult { 208 | match self.pop() { 209 | Some(res) => Ok(res), 210 | None => Err(PopStatus::Empty), 211 | } 212 | } 213 | } 214 | 215 | impl SharedWeakBag for SegQueue { 216 | type Item = T; 217 | fn new() -> Self { 218 | Self::new() 219 | } 220 | fn try_push(&self, t: T) -> Result<(), T> { 221 | self.push(t); 222 | Ok(()) 223 | } 224 | fn try_pop(&self) -> PopResult { 225 | match self.try_pop() { 226 | Some(res) => Ok(res), 227 | None => Err(PopStatus::Empty), 228 | } 229 | } 230 | } 231 | 232 | impl SharedWeakBag for MsQueue { 233 | type Item = T; 234 | fn new() -> Self { 235 | Self::new() 236 | } 237 | fn try_push(&self, t: T) -> Result<(), T> { 238 | self.push(t); 239 | Ok(()) 240 | } 241 | fn try_pop(&self) -> PopResult { 242 | match self.try_pop() { 243 | Some(res) => Ok(res), 244 | None => Err(PopStatus::Empty), 245 | } 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /object-alloc/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | #![no_std] 9 | #![feature(alloc, allocator_api)] 10 | #![feature(core_intrinsics)] 11 | 12 | extern crate alloc; 13 | use alloc::alloc::Layout; 14 | use core::intrinsics::abort; 15 | use core::ptr::NonNull; 16 | 17 | /// Allocators which allocate objects of a particular type. 18 | /// 19 | /// `ObjectAlloc`s provide an interface which is slightly different than the interface provided by 20 | /// a standard allocator. By definition, they are only capable of allocating objects of a 21 | /// particular type. Additionally, memory returned from a call to `alloc` is guaranteed to already 22 | /// be a valid, initialized instance of `T`. `ObjectAlloc`s may differ in how much flexibility they 23 | /// provide in specifying how allocated objects are initialized, and `ObjectAlloc`s obtained using 24 | /// `unsafe` constructors are allowed to break these initialization rules, allocating uninitialized 25 | /// or invalid objects. 26 | /// 27 | /// These differences allow `ObjectAlloc`s to provide significant performance improvements over 28 | /// general-purpose allocators. First, only having to allocate objects of a particular size and 29 | /// alignment allows them to make optimizations that are not available to general-purpose 30 | /// allocators. Second, since `alloc` is required to return already-constructed objects, clients 31 | /// don't have to initialize allocated objects. This, coupled with an object-caching scheme for 32 | /// `dealloc`'d objects, allows many calls to `allloc` to avoid initialization altogether. 33 | /// 34 | /// # Dropping 35 | /// 36 | /// When an `ObjectAlloc` that allocates initialized objects is dropped, all cached `T` objects 37 | /// that have not yet been dropped are dropped. The order in which they are dropped is undefined. 38 | /// 39 | /// # Use in unsafe code 40 | /// 41 | /// Unsafe code may rely on the fact that objects allocated by an `ObjectAlloc` are initialized. 42 | /// Because of this, it must _not_ be possible for safe code to construct an `ObjectAlloc` that 43 | /// doesn't properly initialize objects, or else safe code could construct such an `ObjectAlloc`, 44 | /// pass it to a safe API that uses unsafe code under the hood (and that relies on the 45 | /// initialization behavior of `ObjectAlloc`s), and thus cause memory unsafety. 46 | /// 47 | /// It is acceptable for implementors to provide constructors that produce `ObjectAlloc`s that do 48 | /// not abide by the initialization requirements, but these constructors must be `unsafe` so that 49 | /// they cannot be called from safe code. 50 | pub unsafe trait ObjectAlloc { 51 | /// Allocates an object of type `T`. 52 | /// 53 | /// If this `ObjectAlloc` was obtained using a safe constructor (as opposed to an `unsafe` 54 | /// one), then the memory pointed to by the returned raw pointer is guaranteed to be a valid, 55 | /// initialized instance of `T`. In particular, the returned object will be in one of the 56 | /// following two states: 57 | /// 58 | /// * The result of a call to whatever initialization function was used to configure this 59 | /// `ObjectAlloc` 60 | /// * The same state as a `T` which was previously returned via a call to `dealloc` 61 | /// 62 | /// On the other hand, if this `ObjectAlloc` was obtained using an `unsafe` constructor, then 63 | /// `alloc` may return uninitialized or invalid instances of `T` - the exact behavior should 64 | /// be documented in the constructor. 65 | /// 66 | /// The memory returned by `alloc` is guaranteed to be aligned according to the requirements of 67 | /// `T` (that is, according to `core::mem::align_of::()`). 68 | unsafe fn alloc(&mut self) -> Option>; 69 | 70 | /// Deallocates an object previously returned by `alloc`. 71 | /// 72 | /// If `x` was not obtained through a call to `alloc`, or if `x` has already been `dealloc`'d, 73 | /// the behavior of `dealloc` is undefined. 74 | /// 75 | /// It is valid for `x` to be cached and used to serve future calls to `alloc`. The only 76 | /// guarantee that is made is that if this `ObjectAlloc` allocates initialized objects (unsafe 77 | /// constructors are allowed to produce `ObjectAlloc`s that do not allocate initialized 78 | /// objects), then `x` will be dropped at some point during the `ObjectAlloc`'s lifetime. This 79 | /// may happen during this call to `dealloc`, when the `ObjectAlloc` itself is dropped, or some 80 | /// time in between. 81 | unsafe fn dealloc(&mut self, x: NonNull); 82 | 83 | /// Allocator-specific method for signalling an out-of-memory condition. 84 | /// 85 | /// `oom` aborts the thread or process, optionally performing cleanup or logging diagnostic 86 | /// information before panicking or aborting. 87 | /// 88 | /// `oom` is meant to be used by clients which are unable to cope with an unsatisfied 89 | /// allocation request, and wish to abandon computation rather than attempt to recover locally. 90 | /// The allocator likely has more insight into why the request failed, and thus can likely 91 | /// print more informative diagnostic information than the client could. 92 | /// 93 | /// Implementations of the `oom` method are discouraged from infinitely regressing in nested 94 | /// calls to `oom`. In practice this means implementors should eschew allocating, especially 95 | /// from `self` (directly or indirectly). 96 | /// 97 | /// Implementions of `alloc` are discouraged from panicking (or aborting) in the event of 98 | /// memory exhaustion; instead they should return an error and let the client decide whether to 99 | /// invoke this `oom` method in response. 100 | fn oom(&mut self) -> ! { 101 | unsafe { abort() } 102 | } 103 | } 104 | 105 | /// An allocator for objects whose type or size is not known at compile time. 106 | /// 107 | /// `UntypedObjectAlloc` is like `ObjectAlloc`, except that the size that it allocates may be 108 | /// configured at runtime. Also unlike `ObjectAlloc`, `UntypedObjectAlloc`s make no guarantees 109 | /// about initialization of objects. An individual implementation of `UntypedObjectAlloc` may 110 | /// decide to make such guarantees, but it is not required in order to be a correct implementation 111 | /// of this trait, and the correctness of unsafe code must not rely on this behavior. 112 | pub unsafe trait UntypedObjectAlloc { 113 | /// Obtains the `Layout` of allocated objects. 114 | /// 115 | /// `layout` returns a `Layout` object describing objects allocated by this 116 | /// `UntypedObjectAlloc`. All objects obtained via `alloc` are guaranteed to satisfy this 117 | /// `Layout`. 118 | fn layout(&self) -> Layout; 119 | 120 | /// Allocates a new object. 121 | /// 122 | /// The memory returned by `alloc` is guaranteed to abide by the `Layout` returned from 123 | /// `layout`. 124 | unsafe fn alloc(&mut self) -> Option>; 125 | 126 | /// Deallocates an object previously returned by `alloc`. 127 | /// 128 | /// If `x` was not obtained through a call to `alloc`, or if `x` has already been `dealloc`'d, 129 | /// the behavior of `dealloc` is undefined. 130 | unsafe fn dealloc(&mut self, x: NonNull); 131 | 132 | /// Allocator-specific method for signalling an out-of-memory condition. 133 | /// 134 | /// `oom` aborts the thread or process, optionally performing cleanup or logging diagnostic 135 | /// information before panicking or aborting. 136 | /// 137 | /// `oom` is meant to be used by clients which are unable to cope with an unsatisfied 138 | /// allocation request, and wish to abandon computation rather than attempt to recover locally. 139 | /// The allocator likely has more insight into why the request failed, and thus can likely 140 | /// print more informative diagnostic information than the client could. 141 | /// 142 | /// Implementations of the `oom` method are discouraged from infinitely regressing in nested 143 | /// calls to `oom`. In practice this means implementors should eschew allocating, especially 144 | /// from `self` (directly or indirectly). 145 | /// 146 | /// Implementions of `alloc` are discouraged from panicking (or aborting) in the event of 147 | /// memory exhaustion; instead they should return an error and let the client decide whether to 148 | /// invoke this `oom` method in response. 149 | fn oom(&mut self) -> ! { 150 | unsafe { abort() } 151 | } 152 | } 153 | 154 | unsafe impl UntypedObjectAlloc for ObjectAlloc { 155 | fn layout(&self) -> Layout { 156 | // NOTE: This is safe because the layout method doesn't guarantee that it provides the most 157 | // specific layout, but rather simply that all objects returned from alloc are guaranteed 158 | // to satisfy by this layout. This particular ObjectAlloc could have been configured with a 159 | // more strict alignment than T's alignment, but that's OK. 160 | Layout::new::() 161 | } 162 | 163 | unsafe fn alloc(&mut self) -> Option> { 164 | ObjectAlloc::alloc(self).map(|x| x.cast()) 165 | } 166 | 167 | unsafe fn dealloc(&mut self, x: NonNull) { 168 | ObjectAlloc::dealloc(self, x.cast()); 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /info/elfmalloc-performance.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | # elfmalloc Performance Evaluation 9 | 10 | This document describes the performance of `elfmalloc` in existing benchmarks 11 | that have been used to evaluate `malloc` performance in C/C++. Using the 12 | `LD_PRELOAD` mechanism on Linux, we can get something of an apples-to-apples 13 | measure of how our work stacks up. 14 | 15 | This document is a work in progress. We are currently working on providing 16 | results for more `malloc` implementations in more workloads. We are also 17 | exploring various optimizations for the allocator. Various tuning parameters 18 | improve `elfmalloc` performance substantially in some of these benchmarks, 19 | sometimes at the cost of performance elsewhere. We are also still investigating 20 | the best balance of these parameters. We are also still investigating the 21 | different benchmark settings in use to ensure that all settings are fair wrt 22 | different allocators' thread-local cache sizes. 23 | 24 | ## Two variants of `elfmalloc` 25 | 26 | In addition to a standard version (called simply `elfmalloc`), we provide the 27 | variant `elfmalloc-l`. The `-l` refers to the use of the `LocalCache` front-end 28 | for the allocator, as opposed to the `MagazineCache` front-end used in the 29 | default configuration. While `elfmalloc` is almost always superior in terms of 30 | both throughput and memory usage, `elfmalloc-l` is useful for performance 31 | evaluation as it provides a "no-cache" baseline for allocator performance. 32 | 33 | ## Benchmarks 34 | 35 | We use a subset of the benchmarks used in the `scalloc` 36 | [paper](https://arxiv.org/pdf/1503.09006.pdf). These benchmarks 37 | come from several teams of developers that have worked on allocator 38 | performance over the years. We describe them briefly here, but we 39 | encourage anyone curious to examine Section 7 of that paper. 40 | 41 | * *Threadtest*: This benchmark allocates and deallocates a number of 42 | small objects in rounds each round with several allocations, causing 43 | thread-local caches to be overrun. While this is performed with 44 | multiple threads, we call this workload "thread-local" as all pointers 45 | are freed in the same thread in which they were 46 | allocated. Threadtest was developed to assess allocator performance 47 | during [Hoard's](http://www.cs.utexas.edu/users/mckinley/papers/asplos-2000.pdf) 48 | development. 49 | 50 | * *Shbench*: Similar to *Threadtest* but with varying object sizes and 51 | varying object lifetimes. Our understanding is that this benchmark is due to 52 | [Larson et al](https://pdfs.semanticscholar.org/e41a/d0406628edf82712d16cf4c6d7e486f26f9f.pdf). 53 | 54 | * *rpmalloc-benchmark*: This benchmark performs a (somewhat unrealistic) 55 | workload of randomly-distributed allocation sizes. This benchmark is 56 | included because it can be configured to provide a succinct 57 | producer-consumer workload: one where objects are allocated and freed in 58 | different threads. This is the configuration we use in the measurements 59 | below. 60 | 61 | 74 | 75 | The data here were gathered using a version of the [`scalloc` 76 | artifact](https://github.com/cksystemsgroup/scalloc-artifact). The only 77 | modifications to the artifact were tweaks of object sizes (for the "large 78 | object" variants) and iteration counts to reduce variance and test robustness 79 | for larger workloads. The rpmalloc benchmark is not included in the artifact, so 80 | we adapted it to output similar results. 81 | 82 | These benchmarks were conducted on a 16-core 32-thread workstation with 83 | 2 Xeon E5-2620v4 CPUs on the WSL. We benchmark these workloads at 1, 84 | 2, 4, 8, 16, 24, and 32 threads. For the (1-8) thread configurations, 85 | all use physical cores. For 16 threads, all threads are scheduled on a 86 | single socket using all hardware threads. The 24-thread configuration 87 | uses some subset of the available hardware threads, this time crossing 88 | a NUMA domain. Finally, the 32-thread benchmark uses all available 89 | hardware threads across both sockets. 90 | 91 | ### Other Allocators Measured 92 | 93 | We include two representative allocators to benchmark against. We plan 94 | to add more in the future (e.g. `scalloc`, which we are having trouble 95 | getting to run on WSL due to some `mmap`-related issues). 96 | 97 | * `jemalloc`: A mature, efficient `malloc` used as the default in the 98 | Rust ecosystem and used in production at Facebook. We pulled a dev 99 | version of this allocator in early August 2017. Many of its numbers 100 | appear improved over the earlier version benchmarked in the `scalloc` 101 | paper. [link](https://github.com/jemalloc/jemalloc) 102 | 103 | * `llalloc`: An efficient, proprietary `malloc` from Lockless Inc. 104 | [link](https://locklessinc.com/) 105 | 106 | * `ptmalloc2`: The default allocator on Linux. Throughput numbers are 107 | expressed as a multiple of the numbers for this allocator. These 108 | numbers represent performance for ptmalloc version 2.19; the default 109 | present from the version of glibc in use. 110 | 111 | * `rpmalloc`: [link](https://github.com/rampantpixels/rpmalloc) for more 112 | information. 113 | 114 | ## Measurements 115 | 116 | We provide measurements of both memory consumption and throughput for the 3 117 | workloads described above. A common theme here is that `elfmalloc` provides 118 | consistently high throughput, sometimes at the cost of increased memory usage. 119 | By the same token, if you see an allocator failing to scale up it is worth 120 | looking at heap growth for the same workload. Some allocators like `rpmalloc` 121 | return memory to the OS more aggressively than `elfmalloc`, effectively trading 122 | off improved memory efficiency for reduced throughput. We are still working on 123 | tuning `elfmalloc` to be less profligate with memory under certain 124 | circumstances. 125 | 126 | In order to make the graphs at all readable, we express throughput in terms of a 127 | multiple over the performance of `ptmalloc2`, which is consistently the slowest 128 | allocator. For `threadtest` and `shbench`, we provide numbers for both small (64 129 | bytes or smaller) and medium-sized (a few KB) objects. For throughput, more is 130 | better; for memory consumption less is better. 131 | 132 | ### Threadtest 133 | 134 | Threadtest shows `elfmalloc` having the highest throughput (except for 32 135 | threads, where it is a bit behind `jemalloc`), at the cost of a noticeable 136 | increase in memory usage. 137 | 138 | *Threadtest Throughput (Small Objects)* 139 | 140 | ![Threadtest Throughput](elfmalloc-data/threadtest-small-tp.png?raw=true) 141 | 142 | *Threadtest Memory Consumption (Small Objects)* 143 | 144 | ![Threadtest Memory](elfmalloc-data/threadtest-small-mem.png?raw=true) 145 | 146 | For larger objects, `elfmalloc` comes out on top in terms of throughput for 32 147 | threads. Unlike the case of smaller objects, `elfmalloc` has memory usage 148 | roughly in line with that of `jemalloc`. One interesting point here is the 149 | memory usage of `elfmalloc-l`. Here we see the counter-intuitive result that 150 | increased thread-local cache sizes can lead to *lower* memory consumption. We 151 | suspect that this is because `threadtest` includes long stretches of allocation 152 | followed by relatively limited deallocation, limiting the re-use of allocated 153 | pages. 154 | 155 | *Threadtest Throughput (Medium Objects)* 156 | 157 | ![Threadtest Throughput](elfmalloc-data/threadtest-large-tp.png?raw=true) 158 | 159 | *Threadtest Memory Consumption (Medium Objects)* 160 | 161 | ![Threadtest Memory](elfmalloc-data/threadtest-large-mem.png?raw=true) 162 | 163 | ### Shbench 164 | 165 | For `shbench`, `elfmalloc` out-performs all competition in terms of both memory 166 | and throughput for higher core-counts, though it lags behind `llalloc` early 167 | on.. `elfmalloc-l` performs similarly well, though slightly worse on average for 168 | smaller objects, and slightly better for larger objects. `elfmalloc` also 169 | provides very good memory performance across the board, sometimes better than 170 | `jemalloc`. 171 | 172 | Also of note is the memory consumption of `ptmalloc2`, `llalloc` and `rpmalloc`: 173 | something about the varying lifetimes of objects seems to trip these allocators 174 | up here, as they use over an order of magnitude more memory than `jemalloc` and 175 | `elfmalloc`. 176 | 177 | *Shbench Throughput (Small Objects)* 178 | 179 | ![Shbench Throughput](elfmalloc-data/shbench-small-tp.png?raw=true) 180 | 181 | *Shbench Memory Consumption (Small Objects)* 182 | 183 | ![Shbench Memomry](elfmalloc-data/shbench-small-mem.png?raw=true) 184 | 185 | We see a similar improvement for shebench on larger objects. 186 | 187 | *Shbench Throughput (Medium Objects)* 188 | 189 | ![Shbench Large Object Throughput](elfmalloc-data/shbench-large-tp.png?raw=true) 190 | 191 | *Shbench Memory Consumption (Small Objects)* 192 | 193 | ![Shbench Large Object Memory](elfmalloc-data/shbench-large-mem.png?raw=true) 194 | 195 | ### Producer-Consumer 196 | 197 | Like threadtest, `elfmalloc` has the highest throughput in this workload at the 198 | cost of increased memory consumption. 199 | 200 | *Producer-Consumer Throughput* 201 | 202 | ![Producer-Consumer Throughput](elfmalloc-data/prod-cons-tp.png?raw=true) 203 | 204 | *Producer-Consumer Memory Consumption* 205 | 206 | ![Producer-Consumer Memory](elfmalloc-data/prod-cons-mem.png?raw=true) 207 | -------------------------------------------------------------------------------- /object-alloc-test/src/types.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of 2 | // the README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | //! Types of various sizes for testing `ObjectAlloc`s. 9 | //! 10 | //! This module defines a number of types ranging in size from 1 through 2^13 = 8192 bytes. The 11 | //! sizes used include all powers of two, all midpoints between successive powers of two (that is, 12 | //! `x + x/2` where `x` is a power of two), and at least one prime in between any two even sizes. 13 | //! Each type is named `ByteN` where `N` is the type's size in bytes. 14 | 15 | /// Invoke a macro on multiple sets of arguments. 16 | /// 17 | /// `call_macro!` invokes the macro `m` on 0 or more sets of arguments. Each set of arguments must 18 | /// be surrounded in parentheses. 19 | /// 20 | /// # Examples 21 | /// 22 | /// ```rust 23 | /// # #[macro_use] extern crate object_alloc_test; 24 | /// # fn main() { 25 | /// call_macro!(println, ("Once upon a midnight dreary, while I pondered, weak and weary,"), 26 | /// ("Over many a quaint and curious volume of forgotten lore—"), 27 | /// ("While I nodded, nearly napping, suddenly there came a tapping,"), 28 | /// ("As of some one gently rapping, rapping at my chamber door."), 29 | /// ("\"'Tis some visiter,\" I muttered, \"tapping at my chamber door—"), 30 | /// (" Only this and nothing more.")); 31 | /// # } 32 | /// ``` 33 | #[doc(hidden)] 34 | #[macro_export] 35 | macro_rules! call_macro { 36 | ($m:ident, $($arg:tt),*) => { 37 | $( 38 | $m! $arg ; 39 | )* 40 | } 41 | } 42 | 43 | macro_rules! impl_byte_n { 44 | ($type:ident, $n:tt) => { 45 | #[derive(Copy)] 46 | pub struct $type(pub [u8; $n]); 47 | impl Default for $type { 48 | fn default() -> $type { 49 | $type([0; $n]) 50 | } 51 | } 52 | 53 | #[cfg_attr(feature = "cargo-clippy", allow(expl_impl_clone_on_copy))] 54 | impl Clone for $type { 55 | fn clone(&self) -> $type { 56 | *self 57 | } 58 | } 59 | }; 60 | } 61 | 62 | call_macro!( 63 | impl_byte_n, 64 | (Byte1, 1), 65 | (Byte2, 2), 66 | (Byte3, 3), 67 | (Byte4, 4), 68 | (Byte5, 5), 69 | (Byte6, 6), 70 | (Byte7, 7), 71 | (Byte8, 8), 72 | (Byte11, 11), 73 | (Byte12, 12), 74 | (Byte13, 13), 75 | (Byte16, 16), 76 | (Byte19, 19), 77 | (Byte24, 24), 78 | (Byte29, 29), 79 | (Byte32, 32), 80 | (Byte41, 41), 81 | (Byte48, 48), 82 | (Byte59, 59), 83 | (Byte64, 64), 84 | (Byte73, 73), 85 | (Byte96, 96), 86 | (Byte113, 113), 87 | (Byte128, 128), 88 | (Byte157, 157), 89 | (Byte192, 192), 90 | (Byte229, 229), 91 | (Byte256, 256), 92 | (Byte317, 317), 93 | (Byte384, 384), 94 | (Byte457, 457), 95 | (Byte512, 512), 96 | (Byte768, 768), 97 | (Byte617, 617), 98 | (Byte1024, 1024), 99 | (Byte1277, 1277), 100 | (Byte1536, 1536), 101 | (Byte1777, 1777), 102 | (Byte2048, 2048), 103 | (Byte2557, 2557), 104 | (Byte3072, 3072), 105 | (Byte3539, 3539), 106 | (Byte4096, 4096), 107 | (Byte5119, 5119), 108 | (Byte6144, 6144), 109 | (Byte7151, 7151), 110 | (Byte8192, 8192) 111 | ); 112 | 113 | // Re-export this so that we can access it from the macro, and thus not require 114 | // our clients to manually specify a dependency on the paste crate themselves. 115 | #[doc(hidden)] 116 | pub use paste::item as paste_item; 117 | 118 | // TODO: Document the pattern of matching particular types to avoid defining anything for them. 119 | 120 | /// Call a macro once for each type defined in the `types` module. 121 | /// 122 | /// `call_for_all_types_prefix` is useful for defining something (usually test functions) once for 123 | /// each type defined in the `test` module. The first argument, `fn`, is the name of a macro to 124 | /// invoke. This macro will be invoked once for each type with its first argument being a 125 | /// constructed identifier and the second argument being the type. For example, 126 | /// `call_for_all_types_prefix!(foo, bar)` expands to: 127 | /// 128 | /// ```rust,ignore 129 | /// foo!(bar_0001_byte, $crate::types::Byte1); 130 | /// foo!(bar_0002_byte, $crate::types::Byte2); 131 | /// foo!(bar_0003_byte, $crate::types::Byte3); 132 | /// // ...etc 133 | /// ``` 134 | /// 135 | /// An arbitrary number of optional arguments may also be supplied; these will be 136 | /// passed as additional trailing arguments in the invocation of the macro. For example, 137 | /// `call_for_all_types_prefix!(foo, bar, baz, blah)` expands to: 138 | /// 139 | /// ```rust,ignore 140 | /// foo!(bar_0001_byte, $crate::types::Byte1, baz, blah); 141 | /// foo!(bar_0002_byte, $crate::types::Byte2, baz, blah); 142 | /// foo!(bar_0003_byte, $crate::types::Byte3, baz, blah); 143 | /// // ...etc 144 | /// ``` 145 | /// 146 | /// # Examples 147 | /// 148 | /// ```rust 149 | /// # #[macro_use] extern crate object_alloc_test; 150 | /// # fn main() {} 151 | /// macro_rules! make_default_test { 152 | /// ($name:ident, $type:ty) => ( 153 | /// fn $name() { 154 | /// <$type>::default(); 155 | /// } 156 | /// ) 157 | /// } 158 | /// 159 | /// call_for_all_types_prefix!(make_default_test, default_test); 160 | /// ``` 161 | #[macro_export] 162 | macro_rules! call_for_all_types_prefix { 163 | // NOTE: The '$(, $arg:tt)*' syntax defines a set of optional arguments (note that there's only 164 | // a comma following '$prefix:ident' if there are optional arguments) 165 | ($fn:ident, $prefix:ident $(, $arg:tt)*) => ( 166 | // TODO(joshlf): Replace object_alloc_test with $crate once the 167 | // following issue is fixed: 168 | // https://github.com/rust-lang/rust/issues/55640 169 | object_alloc_test::types::paste_item! { 170 | call_macro!($fn, 171 | ([<$prefix _0001_byte>], object_alloc_test::types::Byte1 $(,$arg)*), 172 | ([<$prefix _0002_byte>], object_alloc_test::types::Byte2 $(,$arg)*), 173 | ([<$prefix _0003_byte>], object_alloc_test::types::Byte3 $(,$arg)*), 174 | ([<$prefix _0004_byte>], object_alloc_test::types::Byte4 $(,$arg)*), 175 | ([<$prefix _0005_byte>], object_alloc_test::types::Byte5 $(,$arg)*), 176 | ([<$prefix _0006_byte>], object_alloc_test::types::Byte6 $(,$arg)*), 177 | ([<$prefix _0007_byte>], object_alloc_test::types::Byte7 $(,$arg)*), 178 | ([<$prefix _0008_byte>], object_alloc_test::types::Byte8 $(,$arg)*), 179 | ([<$prefix _0011_byte>], object_alloc_test::types::Byte11 $(,$arg)*), 180 | ([<$prefix _0012_byte>], object_alloc_test::types::Byte12 $(,$arg)*), 181 | ([<$prefix _0013_byte>], object_alloc_test::types::Byte13 $(,$arg)*), 182 | ([<$prefix _0016_byte>], object_alloc_test::types::Byte16 $(,$arg)*), 183 | ([<$prefix _0019_byte>], object_alloc_test::types::Byte19 $(,$arg)*), 184 | ([<$prefix _0024_byte>], object_alloc_test::types::Byte24 $(,$arg)*), 185 | ([<$prefix _0029_byte>], object_alloc_test::types::Byte29 $(,$arg)*), 186 | ([<$prefix _0032_byte>], object_alloc_test::types::Byte32 $(,$arg)*), 187 | ([<$prefix _0041_byte>], object_alloc_test::types::Byte41 $(,$arg)*), 188 | ([<$prefix _0048_byte>], object_alloc_test::types::Byte48 $(,$arg)*), 189 | ([<$prefix _0059_byte>], object_alloc_test::types::Byte59 $(,$arg)*), 190 | ([<$prefix _0064_byte>], object_alloc_test::types::Byte64 $(,$arg)*), 191 | ([<$prefix _0073_byte>], object_alloc_test::types::Byte73 $(,$arg)*), 192 | ([<$prefix _0096_byte>], object_alloc_test::types::Byte96 $(,$arg)*), 193 | ([<$prefix _0113_byte>], object_alloc_test::types::Byte113 $(,$arg)*), 194 | ([<$prefix _0128_byte>], object_alloc_test::types::Byte128 $(,$arg)*), 195 | ([<$prefix _0157_byte>], object_alloc_test::types::Byte157 $(,$arg)*), 196 | ([<$prefix _0192_byte>], object_alloc_test::types::Byte192 $(,$arg)*), 197 | ([<$prefix _0229_byte>], object_alloc_test::types::Byte229 $(,$arg)*), 198 | ([<$prefix _0256_byte>], object_alloc_test::types::Byte256 $(,$arg)*), 199 | ([<$prefix _0317_byte>], object_alloc_test::types::Byte317 $(,$arg)*), 200 | ([<$prefix _0384_byte>], object_alloc_test::types::Byte384 $(,$arg)*), 201 | ([<$prefix _0457_byte>], object_alloc_test::types::Byte457 $(,$arg)*), 202 | ([<$prefix _0512_byte>], object_alloc_test::types::Byte512 $(,$arg)*), 203 | ([<$prefix _0768_byte>], object_alloc_test::types::Byte768 $(,$arg)*), 204 | ([<$prefix _0617_byte>], object_alloc_test::types::Byte617 $(,$arg)*), 205 | ([<$prefix _1024_byte>], object_alloc_test::types::Byte1024 $(,$arg)*), 206 | ([<$prefix _1277_byte>], object_alloc_test::types::Byte1277 $(,$arg)*), 207 | ([<$prefix _1536_byte>], object_alloc_test::types::Byte1536 $(,$arg)*), 208 | ([<$prefix _1777_byte>], object_alloc_test::types::Byte1777 $(,$arg)*), 209 | ([<$prefix _2048_byte>], object_alloc_test::types::Byte2048 $(,$arg)*), 210 | ([<$prefix _2557_byte>], object_alloc_test::types::Byte2557 $(,$arg)*), 211 | ([<$prefix _3072_byte>], object_alloc_test::types::Byte3072 $(,$arg)*), 212 | ([<$prefix _3539_byte>], object_alloc_test::types::Byte3539 $(,$arg)*), 213 | ([<$prefix _4096_byte>], object_alloc_test::types::Byte4096 $(,$arg)*), 214 | ([<$prefix _5119_byte>], object_alloc_test::types::Byte5119 $(,$arg)*), 215 | ([<$prefix _6144_byte>], object_alloc_test::types::Byte6144 $(,$arg)*), 216 | ([<$prefix _7151_byte>], object_alloc_test::types::Byte7151 $(,$arg)*), 217 | ([<$prefix _8192_byte>], object_alloc_test::types::Byte8192 $(,$arg)*)); 218 | } 219 | ); 220 | } 221 | -------------------------------------------------------------------------------- /slab-alloc/src/tests.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 the authors. See the 'Copyright and license' section of the 2 | // README.md file at the top-level directory of this repository. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or 5 | // the MIT license (the LICENSE-MIT file) at your option. This file may not be 6 | // copied, modified, or distributed except according to those terms. 7 | 8 | use alloc::alloc::{Alloc, Global, Layout}; 9 | use object_alloc::ObjectAlloc; 10 | use object_alloc_test::leaky_alloc::LeakyAlloc; 11 | use test::{black_box, Bencher}; 12 | 13 | use backing::alloc::AllocObjectAlloc; 14 | use backing::BackingAlloc; 15 | use SlabAlloc; 16 | use SlabAllocBuilder; 17 | 18 | use core::ptr::NonNull; 19 | 20 | fn infer_allocator_type(alloc: &mut ObjectAlloc) { 21 | if false { 22 | let _: Option> = unsafe { alloc.alloc() }; 23 | } 24 | } 25 | 26 | struct LeakyBackingAlloc; 27 | 28 | impl BackingAlloc for LeakyBackingAlloc { 29 | type Aligned = AllocObjectAlloc; 30 | type Large = AllocObjectAlloc; 31 | } 32 | 33 | fn leaky_get_aligned(layout: Layout) -> Option> { 34 | if layout.align() == ::sysconf::page::pagesize() { 35 | Some(AllocObjectAlloc::new(LeakyAlloc::default(), layout)) 36 | } else { 37 | None 38 | } 39 | } 40 | 41 | fn leaky_get_large(layout: Layout) -> AllocObjectAlloc { 42 | AllocObjectAlloc::new(LeakyAlloc::default(), layout) 43 | } 44 | 45 | fn test_memory_corruption() { 46 | use object_alloc_test::corruption::{CorruptionTesterDefault, TestBuilder}; 47 | use object_alloc_test::foreach_align; 48 | use std::env; 49 | let default = 100_000; 50 | let iters = match env::var("SLAB_TEST_ITERS") { 51 | Ok(val) => val.parse().unwrap_or(default), 52 | Err(_) => default, 53 | }; 54 | let f = |align| { 55 | let new = move || { 56 | SlabAllocBuilder::default() 57 | .align(align) 58 | .build_backing(leaky_get_aligned, leaky_get_large) 59 | as SlabAlloc<_, _, LeakyBackingAlloc> 60 | }; 61 | infer_allocator_type::>(&mut new()); 62 | TestBuilder::new(new).test_iters(iters).test(); 63 | }; 64 | foreach_align::, _>(f, ::sysconf::page::pagesize()); 65 | } 66 | 67 | fn test_quickcheck_memory_corruption() { 68 | use object_alloc_test::corruption::{CorruptionTesterDefault, TestBuilder}; 69 | use object_alloc_test::foreach_align; 70 | use std::env; 71 | let default = 100_000; 72 | let tests = match env::var("SLAB_QUICKCHECK_TESTS") { 73 | Ok(val) => val.parse().unwrap_or(default), 74 | Err(_) => default, 75 | }; 76 | let f = |align| { 77 | let new = move || { 78 | SlabAllocBuilder::default() 79 | .align(align) 80 | .build_backing(leaky_get_aligned, leaky_get_large) 81 | as SlabAlloc<_, _, LeakyBackingAlloc> 82 | }; 83 | infer_allocator_type::>(&mut new()); 84 | TestBuilder::new(new).quickcheck_tests(tests).quickcheck(); 85 | }; 86 | foreach_align::, _>(f, ::sysconf::page::pagesize()); 87 | } 88 | 89 | macro_rules! make_test_memory_corruption { 90 | // The corruption checker can't handle types smaller than 9 bytes. It's easier to turn these 91 | // into noops and still call define_for_all_types_prefix! than to manually define all of the 92 | // tests that we /do/ want to define. 93 | ($name:ident,crate::types::Byte1) => {}; 94 | ($name:ident,crate::types::Byte2) => {}; 95 | ($name:ident,crate::types::Byte3) => {}; 96 | ($name:ident,crate::types::Byte4) => {}; 97 | ($name:ident,crate::types::Byte5) => {}; 98 | ($name:ident,crate::types::Byte6) => {}; 99 | ($name:ident,crate::types::Byte7) => {}; 100 | ($name:ident,crate::types::Byte8) => {}; 101 | ($name:ident, $type:ty) => { 102 | #[test] 103 | fn $name() { 104 | test_memory_corruption::<$type>(); 105 | } 106 | }; 107 | } 108 | 109 | macro_rules! make_test_quickcheck_memory_corruption { 110 | // The corruption checker can't handle types smaller than 9 bytes. It's easier to turn these 111 | // into noops and still call define_for_all_types_prefix! than to manually define all of the 112 | // tests that we /do/ want to define. 113 | ($name:ident,crate::types::Byte1) => {}; 114 | ($name:ident,crate::types::Byte2) => {}; 115 | ($name:ident,crate::types::Byte3) => {}; 116 | ($name:ident,crate::types::Byte4) => {}; 117 | ($name:ident,crate::types::Byte5) => {}; 118 | ($name:ident,crate::types::Byte6) => {}; 119 | ($name:ident,crate::types::Byte7) => {}; 120 | ($name:ident,crate::types::Byte8) => {}; 121 | ($name:ident, $type:ty) => { 122 | #[test] 123 | #[ignore] 124 | fn $name() { 125 | test_quickcheck_memory_corruption::<$type>(); 126 | } 127 | }; 128 | } 129 | 130 | call_for_all_types_prefix!(make_test_memory_corruption, test_memory_corruption); 131 | call_for_all_types_prefix!( 132 | make_test_quickcheck_memory_corruption, 133 | quickcheck_memory_corruption 134 | ); 135 | 136 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 137 | fn bench_alloc_no_free(b: &mut Bencher) { 138 | let mut alloc = SlabAllocBuilder::default().build(); 139 | infer_allocator_type::(&mut alloc); 140 | b.iter(|| unsafe { black_box(alloc.alloc().unwrap()) }); 141 | // since we didn't free anything, dropping alloc would result in a refcnt check failing 142 | use std::mem; 143 | mem::forget(alloc); 144 | } 145 | 146 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 147 | fn bench_alloc_no_free_no_init(b: &mut Bencher) { 148 | let mut alloc = unsafe { SlabAllocBuilder::no_initialize().build() }; 149 | infer_allocator_type::(&mut alloc); 150 | b.iter(|| unsafe { black_box(alloc.alloc().unwrap()) }); 151 | // since we didn't free anything, dropping alloc would result in a refcnt check failing 152 | use std::mem; 153 | mem::forget(alloc); 154 | } 155 | 156 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 157 | fn bench_alloc_no_free_heap(b: &mut Bencher) { 158 | let layout = Layout::new::(); 159 | b.iter(|| unsafe { black_box(Global.alloc(layout.clone()).unwrap()) }); 160 | } 161 | 162 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 163 | fn bench_alloc_free_pairs(b: &mut Bencher) { 164 | let mut alloc = SlabAllocBuilder::default().build(); 165 | infer_allocator_type::(&mut alloc); 166 | // keep one allocated at all times so the slab will never be freed; 167 | // we're trying to bench the best-case performance, not the slab gc policy 168 | let t = unsafe { alloc.alloc().unwrap() }; 169 | b.iter(|| unsafe { 170 | let t = alloc.alloc().unwrap(); 171 | black_box(t); 172 | alloc.dealloc(t); 173 | }); 174 | unsafe { 175 | alloc.dealloc(t); 176 | } 177 | } 178 | 179 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 180 | fn bench_alloc_free_pairs_no_init(b: &mut Bencher) { 181 | let mut alloc = unsafe { SlabAllocBuilder::no_initialize().build() }; 182 | infer_allocator_type::(&mut alloc); 183 | // keep one allocated at all times so the slab will never be freed; 184 | // we're trying to bench the best-case performance, not the slab gc policy 185 | let t = unsafe { alloc.alloc().unwrap() }; 186 | b.iter(|| unsafe { 187 | let t = alloc.alloc().unwrap(); 188 | black_box(t); 189 | alloc.dealloc(t); 190 | }); 191 | unsafe { 192 | alloc.dealloc(t); 193 | } 194 | } 195 | 196 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 197 | fn bench_alloc_free_pairs_heap(b: &mut Bencher) { 198 | let layout = Layout::new::(); 199 | b.iter(|| unsafe { 200 | let t = Global.alloc(layout.clone()).unwrap(); 201 | black_box(t); 202 | Global.dealloc(t, layout.clone()); 203 | }); 204 | } 205 | 206 | macro_rules! make_bench_alloc_no_free { 207 | ($name:ident, $typ:ty) => { 208 | #[bench] 209 | #[cfg(feature = "build-ignored-tests")] 210 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 211 | #[ignore] 212 | fn $name(b: &mut Bencher) { 213 | bench_alloc_no_free::<$typ>(b); 214 | } 215 | }; 216 | } 217 | macro_rules! make_bench_alloc_no_free_no_init { 218 | ($name:ident, $typ:ty) => { 219 | #[bench] 220 | #[cfg(feature = "build-ignored-tests")] 221 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 222 | #[ignore] 223 | fn $name(b: &mut Bencher) { 224 | bench_alloc_no_free_no_init::<$typ>(b); 225 | } 226 | }; 227 | } 228 | macro_rules! make_bench_alloc_no_free_heap { 229 | ($name:ident, $typ:ty) => { 230 | #[bench] 231 | #[cfg(feature = "build-ignored-tests")] 232 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 233 | #[ignore] 234 | fn $name(b: &mut Bencher) { 235 | bench_alloc_no_free_heap::<$typ>(b); 236 | } 237 | }; 238 | } 239 | macro_rules! make_bench_alloc_free_pairs { 240 | ($name:ident, $typ:ty) => { 241 | #[bench] 242 | #[cfg(feature = "build-ignored-tests")] 243 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 244 | #[ignore] 245 | fn $name(b: &mut Bencher) { 246 | bench_alloc_free_pairs::<$typ>(b); 247 | } 248 | }; 249 | } 250 | macro_rules! make_bench_alloc_free_pairs_no_init { 251 | ($name:ident, $typ:ty) => { 252 | #[bench] 253 | #[cfg(feature = "build-ignored-tests")] 254 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 255 | #[ignore] 256 | fn $name(b: &mut Bencher) { 257 | bench_alloc_free_pairs_no_init::<$typ>(b); 258 | } 259 | }; 260 | } 261 | macro_rules! make_bench_alloc_free_pairs_heap { 262 | ($name:ident, $typ:ty) => { 263 | #[bench] 264 | #[cfg(feature = "build-ignored-tests")] 265 | #[cfg_attr(not(feature = "build-ignored-tests"), allow(unused))] 266 | #[ignore] 267 | fn $name(b: &mut Bencher) { 268 | bench_alloc_free_pairs_heap::<$typ>(b); 269 | } 270 | }; 271 | } 272 | 273 | call_for_all_types_prefix!(make_bench_alloc_no_free, bench_alloc_no_free); 274 | call_for_all_types_prefix!( 275 | make_bench_alloc_no_free_no_init, 276 | bench_alloc_no_free_no_init 277 | ); 278 | call_for_all_types_prefix!(make_bench_alloc_no_free_heap, bench_alloc_no_free_heap); 279 | call_for_all_types_prefix!(make_bench_alloc_free_pairs, bench_alloc_free_pairs); 280 | call_for_all_types_prefix!( 281 | make_bench_alloc_free_pairs_no_init, 282 | bench_alloc_free_pairs_no_init 283 | ); 284 | call_for_all_types_prefix!( 285 | make_bench_alloc_free_pairs_heap, 286 | bench_alloc_free_pairs_heap 287 | ); 288 | --------------------------------------------------------------------------------