├── src ├── snmalloc │ ├── snmalloc_front.h │ ├── snmalloc_core.h │ ├── stl │ │ ├── cxx │ │ │ ├── README.md │ │ │ ├── array.h │ │ │ ├── utility.h │ │ │ ├── atomic.h │ │ │ └── type_traits.h │ │ ├── array.h │ │ ├── atomic.h │ │ ├── utility.h │ │ ├── type_traits.h │ │ ├── gnu │ │ │ ├── README.md │ │ │ ├── utility.h │ │ │ └── array.h │ │ ├── README.md │ │ └── new.h │ ├── global │ │ ├── global.h │ │ ├── scopedalloc.h │ │ └── bounds_checks.h │ ├── override │ │ ├── memcpy.cc │ │ ├── malloc-extensions.cc │ │ ├── override.h │ │ ├── malloc-extensions.h │ │ ├── rust.cc │ │ └── malloc.cc │ ├── ds │ │ ├── ds.h │ │ ├── entropy.h │ │ └── mpmcstack.h │ ├── ds_aal │ │ ├── ds_aal.h │ │ └── singleton.h │ ├── mem │ │ ├── mem.h │ │ ├── check_init.h │ │ ├── secondary │ │ │ ├── default.h │ │ │ └── gwp_asan.h │ │ ├── pooled.h │ │ ├── ticker.h │ │ └── backend_wrappers.h │ ├── backend_helpers │ │ ├── globalrange.h │ │ ├── empty_range.h │ │ ├── backend_helpers.h │ │ ├── noprange.h │ │ ├── staticrange.h │ │ ├── pagemapregisterrange.h │ │ ├── indirectrange.h │ │ ├── palrange.h │ │ ├── lockrange.h │ │ ├── subrange.h │ │ ├── logrange.h │ │ ├── commitrange.h │ │ ├── statsrange.h │ │ ├── authmap.h │ │ ├── defaultpagemapentry.h │ │ └── staticconditionalrange.h │ ├── backend │ │ ├── base_constants.h │ │ └── standard_range.h │ ├── ds_core │ │ ├── ds_core.h │ │ ├── tid.h │ │ └── concept.h │ ├── pal │ │ ├── pal_timer_default.h │ │ ├── pal_plain.h │ │ ├── pal_haiku.h │ │ ├── pal_dragonfly.h │ │ ├── pal_bsd.h │ │ ├── pal_open_enclave.h │ │ ├── pal_bsd_aligned.h │ │ ├── pal_solaris.h │ │ ├── pal_netbsd.h │ │ ├── pal_openbsd.h │ │ ├── pal_consts.h │ │ ├── pal_noalloc.h │ │ └── pal_freebsd_kernel.h │ ├── aal │ │ ├── aal_powerpc.h │ │ ├── aal_consts.h │ │ ├── aal_sparc.h │ │ ├── aal_x86_sgx.h │ │ ├── aal_arm.h │ │ ├── aal_riscv.h │ │ ├── aal_x86.h │ │ └── aal_concept.h │ ├── snmalloc.h │ └── README.md └── test │ ├── func │ ├── two_alloc_types │ │ ├── alloc2.cc │ │ ├── alloc1.cc │ │ └── main.cc │ ├── bits │ │ └── bits.cc │ ├── external_pagemap │ │ └── external_pagemap.cc │ ├── multi_atexit │ │ └── multi_atexit.cc │ ├── release-rounding │ │ └── rounding.cc │ ├── multi_threadatexit │ │ └── multi_threadatexit.cc │ ├── client_meta │ │ └── client_meta.cc │ ├── fixed_region │ │ └── fixed_region.cc │ ├── fixed_region_alloc │ │ └── fixed_region_alloc.cc │ ├── protect_fork │ │ └── protect_fork.cc │ ├── thread_alloc_external │ │ └── thread_alloc_external.cc │ ├── memory_usage │ │ └── memory_usage.cc │ ├── multi_setspecific │ │ └── multi_setspecific.cc │ ├── statistics │ │ └── stats.cc │ ├── first_operation │ │ └── first_operation.cc │ └── teardown │ │ └── teardown.cc │ ├── helpers.h │ ├── measuretime.h │ ├── usage.h │ ├── opt.h │ ├── xoroshiro.h │ ├── perf │ ├── startup │ │ └── startup.cc │ ├── singlethread │ │ └── singlethread.cc │ ├── external_pointer │ │ └── externalpointer.cc │ └── lotsofthreads │ │ └── lotsofthread.cc │ └── setup.h ├── snmalloc.pdf ├── docs └── security │ ├── data │ ├── ChunkMap.png │ ├── perfgraph.png │ ├── memcpy_perf.png │ ├── doublefreeprotection.gif │ └── perfgraph-memcpy-only.png │ └── README.md ├── MODULE.bazel ├── .bazelrc ├── .gitignore ├── fuzzing ├── CMakeLists.txt └── BUILD.bazel ├── ci ├── Toolchain.cmake └── README.md ├── LICENSE ├── benchmark └── Dockerfile ├── .clang-tidy ├── .github └── workflows │ └── benchmark.yml ├── BUILD.bazel ├── security.md ├── fuzztest.bazelrc ├── .clang-format └── README.md /src/snmalloc/snmalloc_front.h: -------------------------------------------------------------------------------- 1 | #include "global/global.h" 2 | -------------------------------------------------------------------------------- /snmalloc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/snmalloc/HEAD/snmalloc.pdf -------------------------------------------------------------------------------- /src/snmalloc/snmalloc_core.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "backend_helpers/backend_helpers.h" 4 | -------------------------------------------------------------------------------- /docs/security/data/ChunkMap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/snmalloc/HEAD/docs/security/data/ChunkMap.png -------------------------------------------------------------------------------- /docs/security/data/perfgraph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/snmalloc/HEAD/docs/security/data/perfgraph.png -------------------------------------------------------------------------------- /docs/security/data/memcpy_perf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/snmalloc/HEAD/docs/security/data/memcpy_perf.png -------------------------------------------------------------------------------- /src/snmalloc/stl/cxx/README.md: -------------------------------------------------------------------------------- 1 | # CXX Standard Library 2 | 3 | This directory provides an interface to use default STL. 4 | -------------------------------------------------------------------------------- /docs/security/data/doublefreeprotection.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/snmalloc/HEAD/docs/security/data/doublefreeprotection.gif -------------------------------------------------------------------------------- /docs/security/data/perfgraph-memcpy-only.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/snmalloc/HEAD/docs/security/data/perfgraph-memcpy-only.png -------------------------------------------------------------------------------- /src/snmalloc/global/global.h: -------------------------------------------------------------------------------- 1 | #include "bounds_checks.h" 2 | #include "globalalloc.h" 3 | #include "libc.h" 4 | #include "memcpy.h" 5 | #include "scopedalloc.h" 6 | #include "threadalloc.h" 7 | -------------------------------------------------------------------------------- /src/snmalloc/stl/array.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifdef SNMALLOC_USE_SELF_VENDORED_STL 4 | # include "snmalloc/stl/gnu/array.h" 5 | #else 6 | # include "snmalloc/stl/cxx/array.h" 7 | #endif 8 | -------------------------------------------------------------------------------- /src/snmalloc/stl/atomic.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifdef SNMALLOC_USE_SELF_VENDORED_STL 4 | # include "snmalloc/stl/gnu/atomic.h" 5 | #else 6 | # include "snmalloc/stl/cxx/atomic.h" 7 | #endif 8 | -------------------------------------------------------------------------------- /src/snmalloc/stl/utility.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifdef SNMALLOC_USE_SELF_VENDORED_STL 4 | # include "snmalloc/stl/gnu/utility.h" 5 | #else 6 | # include "snmalloc/stl/cxx/utility.h" 7 | #endif 8 | -------------------------------------------------------------------------------- /src/snmalloc/stl/type_traits.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifdef SNMALLOC_USE_SELF_VENDORED_STL 4 | # include "snmalloc/stl/gnu/type_traits.h" 5 | #else 6 | # include "snmalloc/stl/cxx/type_traits.h" 7 | #endif 8 | -------------------------------------------------------------------------------- /src/snmalloc/stl/gnu/README.md: -------------------------------------------------------------------------------- 1 | # Self-Vendored STL Using GNU Language Extensions 2 | 3 | This directory contains implementations of STL functionalities using GNU extensions. Such extensions are also available with clang. 4 | -------------------------------------------------------------------------------- /MODULE.bazel: -------------------------------------------------------------------------------- 1 | module(name = "snmalloc") 2 | 3 | bazel_dep(name = "rules_cc", version = "0.1.1") 4 | bazel_dep(name = "rules_foreign_cc", version = "0.14.0") 5 | bazel_dep(name = "fuzztest", version = "20250214.0") 6 | bazel_dep(name = "googletest", version = "1.16.0") 7 | -------------------------------------------------------------------------------- /src/snmalloc/stl/README.md: -------------------------------------------------------------------------------- 1 | # Standard Library Implementation 2 | 3 | To support build environment without C++ STL, snmalloc can optionally use self-vendored STL functionalities. 4 | To use self-vendored implementations, one need to set `SNMALLOC_USE_SELF_VENDORED_STL` to `ON` when configuring cmake. 5 | -------------------------------------------------------------------------------- /src/test/func/two_alloc_types/alloc2.cc: -------------------------------------------------------------------------------- 1 | #ifndef SNMALLOC_TRACING 2 | # define SNMALLOC_TRACING 3 | #endif 4 | 5 | #define SNMALLOC_NAME_MANGLE(a) host_##a 6 | // Redefine the namespace, so we can have two versions. 7 | #define snmalloc snmalloc_host 8 | #include 9 | -------------------------------------------------------------------------------- /.bazelrc: -------------------------------------------------------------------------------- 1 | common --enable_platform_specific_config 2 | 3 | build --cxxopt=-std=c++2a 4 | 5 | # Show everything when running tests. 6 | test --test_output=streamed 7 | 8 | build:macos --macos_minimum_os=10.15 9 | build:macos --no@fuzztest//fuzztest:use_riegeli 10 | 11 | try-import %workspace%/fuzztest.bazelrc 12 | -------------------------------------------------------------------------------- /src/snmalloc/override/memcpy.cc: -------------------------------------------------------------------------------- 1 | #include "override.h" 2 | 3 | extern "C" 4 | { 5 | /** 6 | * Snmalloc checked memcpy. 7 | */ 8 | SNMALLOC_EXPORT void* 9 | SNMALLOC_NAME_MANGLE(memcpy)(void* dst, const void* src, size_t len) 10 | { 11 | return snmalloc::memcpy(dst, src, len); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/snmalloc/ds/ds.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Data structures used by snmalloc. 3 | * 4 | */ 5 | #pragma once 6 | #include "../ds_aal/ds_aal.h" 7 | #include "../pal/pal.h" 8 | #include "aba.h" 9 | #include "allocconfig.h" 10 | #include "combininglock.h" 11 | #include "entropy.h" 12 | #include "mpmcstack.h" 13 | #include "pagemap.h" 14 | -------------------------------------------------------------------------------- /src/snmalloc/stl/cxx/array.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace snmalloc 6 | { 7 | namespace stl 8 | { 9 | template 10 | using Array = std::array; 11 | 12 | using std::begin; 13 | using std::end; 14 | } // namespace stl 15 | } // namespace snmalloc 16 | -------------------------------------------------------------------------------- /src/snmalloc/ds_aal/ds_aal.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Data structures used by snmalloc that only depend on the AAL (Architecture 3 | * Abstraction Layer) and not on the platform. These structures can be used 4 | * to implement the Pal. 5 | */ 6 | #pragma once 7 | #include "../aal/aal.h" 8 | #include "flaglock.h" 9 | #include "prevent_fork.h" 10 | #include "singleton.h" -------------------------------------------------------------------------------- /src/snmalloc/stl/cxx/utility.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace snmalloc 4 | { 5 | namespace stl 6 | { 7 | using std::declval; 8 | using std::exchange; 9 | using std::forward; 10 | using std::move; 11 | template 12 | using Pair = std::pair; 13 | } // namespace stl 14 | } // namespace snmalloc 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # conventional build dirs 2 | /release*/ 3 | /debug*/ 4 | /build*/ 5 | /cmake-build-*/ 6 | /out*/ 7 | /dist/ 8 | 9 | /tidy.fail 10 | 11 | # cmake intermediate files 12 | /CMakeFiles/ 13 | 14 | # vscode dirs 15 | .vscode/ 16 | .vs/ 17 | 18 | # jetbrains IDE dirs 19 | .idea/ 20 | 21 | # special endings 22 | *~ 23 | *.sw? 24 | 25 | # cache dirs 26 | .cache 27 | -------------------------------------------------------------------------------- /src/snmalloc/mem/mem.h: -------------------------------------------------------------------------------- 1 | #include "backend_concept.h" 2 | #include "backend_wrappers.h" 3 | #include "check_init.h" 4 | #include "corealloc.h" 5 | #include "entropy.h" 6 | #include "freelist.h" 7 | #include "metadata.h" 8 | #include "pool.h" 9 | #include "pooled.h" 10 | #include "remoteallocator.h" 11 | #include "remotecache.h" 12 | #include "sizeclasstable.h" 13 | #include "ticker.h" 14 | -------------------------------------------------------------------------------- /src/snmalloc/override/malloc-extensions.cc: -------------------------------------------------------------------------------- 1 | #include "malloc-extensions.h" 2 | 3 | #include "../snmalloc.h" 4 | 5 | using namespace snmalloc; 6 | 7 | void get_malloc_info_v1(malloc_info_v1* stats) 8 | { 9 | auto curr = Alloc::Config::Backend::get_current_usage(); 10 | auto peak = Alloc::Config::Backend::get_peak_usage(); 11 | stats->current_memory_usage = curr; 12 | stats->peak_memory_usage = peak; 13 | } 14 | -------------------------------------------------------------------------------- /src/snmalloc/override/override.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "snmalloc/snmalloc.h" 4 | 5 | #ifndef SNMALLOC_EXPORT 6 | # define SNMALLOC_EXPORT 7 | #endif 8 | #ifdef SNMALLOC_STATIC_LIBRARY_PREFIX 9 | # define __SN_CONCAT(a, b) a##b 10 | # define __SN_EVALUATE(a, b) __SN_CONCAT(a, b) 11 | # define SNMALLOC_NAME_MANGLE(a) \ 12 | __SN_EVALUATE(SNMALLOC_STATIC_LIBRARY_PREFIX, a) 13 | #elif !defined(SNMALLOC_NAME_MANGLE) 14 | # define SNMALLOC_NAME_MANGLE(a) a 15 | #endif 16 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/globalrange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../ds/ds.h" 4 | #include "empty_range.h" 5 | #include "lockrange.h" 6 | #include "staticrange.h" 7 | 8 | namespace snmalloc 9 | { 10 | /** 11 | * Makes the supplied ParentRange into a global variable, 12 | * and protects access with a lock. 13 | */ 14 | struct GlobalRange 15 | { 16 | template> 17 | class Type : public Pipe 18 | {}; 19 | }; 20 | } // namespace snmalloc 21 | -------------------------------------------------------------------------------- /src/snmalloc/backend/base_constants.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | #pragma once 4 | 5 | #include "../backend/backend.h" 6 | 7 | namespace snmalloc 8 | { 9 | /** 10 | * Base range configuration contains common parts of other ranges. 11 | */ 12 | struct BaseLocalStateConstants 13 | { 14 | protected: 15 | // Size of requests that the global cache should use 16 | static constexpr size_t GlobalCacheSizeBits = 24; 17 | 18 | // Size of requests that the local cache should use 19 | static constexpr size_t LocalCacheSizeBits = 21; 20 | }; 21 | } // namespace snmalloc -------------------------------------------------------------------------------- /fuzzing/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(FetchContent) 2 | 3 | FetchContent_Declare( 4 | fuzztest 5 | GIT_REPOSITORY https://github.com/google/fuzztest.git 6 | GIT_TAG 2024-10-28 7 | ) 8 | 9 | FetchContent_MakeAvailable(fuzztest) 10 | 11 | enable_testing() 12 | fuzztest_setup_fuzzing_flags() 13 | 14 | add_executable( 15 | snmalloc-fuzzer 16 | snmalloc-fuzzer.cpp 17 | ) 18 | 19 | target_link_libraries(snmalloc-fuzzer PRIVATE snmalloc) 20 | target_compile_options(snmalloc-fuzzer PRIVATE -fsanitize=address -DADDRESS_SANITIZER) 21 | 22 | link_fuzztest(snmalloc-fuzzer) 23 | -------------------------------------------------------------------------------- /src/snmalloc/ds_core/ds_core.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * The core definitions for snmalloc. These provide some basic helpers that do 4 | * not depend on anything except for a working C++ implementation. 5 | * 6 | * Files in this directory may not include anything from any other directory in 7 | * snmalloc. 8 | */ 9 | 10 | #include "bits.h" 11 | #include "cheri.h" 12 | #include "concept.h" 13 | #include "defines.h" 14 | #include "helpers.h" 15 | #include "mitigations.h" 16 | #include "ptrwrap.h" 17 | #include "redblacktree.h" 18 | #include "seqset.h" 19 | #include "tid.h" -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/empty_range.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "../ds_core/ds_core.h" 3 | 4 | namespace snmalloc 5 | { 6 | template 7 | class EmptyRange 8 | { 9 | public: 10 | static constexpr bool Aligned = true; 11 | 12 | static constexpr bool ConcurrencySafe = true; 13 | 14 | using ChunkBounds = B; 15 | 16 | constexpr EmptyRange() = default; 17 | 18 | CapPtr alloc_range(size_t) 19 | { 20 | return nullptr; 21 | } 22 | }; 23 | } // namespace snmalloc 24 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/backend_helpers.h: -------------------------------------------------------------------------------- 1 | #include "../mem/mem.h" 2 | #include "authmap.h" 3 | #include "buddy.h" 4 | #include "commitrange.h" 5 | #include "commonconfig.h" 6 | #include "defaultpagemapentry.h" 7 | #include "empty_range.h" 8 | #include "globalrange.h" 9 | #include "indirectrange.h" 10 | #include "largebuddyrange.h" 11 | #include "logrange.h" 12 | #include "noprange.h" 13 | #include "pagemap.h" 14 | #include "pagemapregisterrange.h" 15 | #include "palrange.h" 16 | #include "range_helpers.h" 17 | #include "smallbuddyrange.h" 18 | #include "staticconditionalrange.h" 19 | #include "statsrange.h" 20 | #include "subrange.h" 21 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_timer_default.h: -------------------------------------------------------------------------------- 1 | 2 | #pragma once 3 | 4 | #include "pal_consts.h" 5 | #include "pal_ds.h" 6 | 7 | namespace snmalloc 8 | { 9 | template 10 | class PalTimerDefaultImpl 11 | { 12 | inline static PalTimer timers{}; 13 | 14 | public: 15 | static uint64_t time_in_ms() 16 | { 17 | auto time = PalTime::internal_time_in_ms(); 18 | 19 | // Process timers 20 | timers.check(time); 21 | 22 | return time; 23 | } 24 | 25 | static void register_timer(PalTimerObject* timer) 26 | { 27 | timers.register_timer(timer); 28 | } 29 | }; 30 | } // namespace snmalloc 31 | -------------------------------------------------------------------------------- /src/snmalloc/ds/entropy.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifndef SNMALLOC_PLATFORM_HAS_GETENTROPY 4 | # include 5 | #endif 6 | 7 | namespace snmalloc 8 | { 9 | template 10 | stl::enable_if_t, uint64_t> get_entropy64() 11 | { 12 | return PAL::get_entropy64(); 13 | } 14 | 15 | template 16 | stl::enable_if_t, uint64_t> get_entropy64() 17 | { 18 | #ifdef SNMALLOC_PLATFORM_HAS_GETENTROPY 19 | return DefaultPal::get_entropy64(); 20 | #else 21 | std::random_device rd; 22 | uint64_t a = rd(); 23 | return (a << 32) ^ rd(); 24 | #endif 25 | } 26 | } // namespace snmalloc 27 | -------------------------------------------------------------------------------- /src/snmalloc/ds_core/tid.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "snmalloc/stl/atomic.h" 4 | 5 | namespace snmalloc 6 | { 7 | /** 8 | * @brief Get the an id for the current thread. 9 | * 10 | * @return the thread id, this should never be the default of 11 | * ThreadIdentity. Callers can assume it is a non-default value. 12 | * 13 | * Note, this is only for debug. We should not assume that this is unique. 14 | */ 15 | inline size_t debug_get_tid() noexcept 16 | { 17 | static thread_local size_t tid{0}; 18 | static stl::Atomic tid_source{0}; 19 | 20 | if (tid == 0) 21 | { 22 | tid = ++tid_source; 23 | } 24 | return tid; 25 | } 26 | } // namespace snmalloc -------------------------------------------------------------------------------- /src/test/func/two_alloc_types/alloc1.cc: -------------------------------------------------------------------------------- 1 | #ifndef SNMALLOC_TRACING 2 | # define SNMALLOC_TRACING 3 | #endif 4 | 5 | // Redefine the namespace, so we can have two versions. 6 | #define snmalloc snmalloc_enclave 7 | 8 | #include 9 | #include 10 | 11 | // Specify type of allocator 12 | #define SNMALLOC_PROVIDE_OWN_CONFIG 13 | 14 | namespace snmalloc 15 | { 16 | using Config = FixedRangeConfig>; 17 | } 18 | 19 | #define SNMALLOC_NAME_MANGLE(a) enclave_##a 20 | #include 21 | 22 | extern "C" void oe_allocator_init(void* base, void* end) 23 | { 24 | snmalloc::Config::init(nullptr, base, address_cast(end) - address_cast(base)); 25 | } 26 | -------------------------------------------------------------------------------- /src/snmalloc/stl/cxx/atomic.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace snmalloc 6 | { 7 | namespace stl 8 | { 9 | template 10 | using Atomic = std::atomic; 11 | 12 | constexpr auto memory_order_relaxed = std::memory_order_relaxed; 13 | constexpr auto memory_order_consume = std::memory_order_consume; 14 | constexpr auto memory_order_acquire = std::memory_order_acquire; 15 | constexpr auto memory_order_release = std::memory_order_release; 16 | constexpr auto memory_order_acq_rel = std::memory_order_acq_rel; 17 | constexpr auto memory_order_seq_cst = std::memory_order_seq_cst; 18 | 19 | using AtomicBool = std::atomic; 20 | using MemoryOrder = std::memory_order; 21 | } // namespace stl 22 | } // namespace snmalloc 23 | -------------------------------------------------------------------------------- /ci/Toolchain.cmake: -------------------------------------------------------------------------------- 1 | set(CMAKE_SYSTEM_NAME Linux) 2 | set(CMAKE_SYSTEM_PROCESSOR $ENV{ARCH}) 3 | 4 | set(triple $ENV{TRIPLE}) 5 | 6 | set(CMAKE_C_COMPILER clang-$ENV{SNMALLOC_CI_CLANG_VERSION}) 7 | set(CMAKE_C_COMPILER_TARGET ${triple}) 8 | set(CMAKE_CXX_COMPILER clang++-$ENV{SNMALLOC_CI_CLANG_VERSION}) 9 | set(CMAKE_CXX_COMPILER_TARGET ${triple}) 10 | 11 | set(CROSS_LINKER_FLAGS "-fuse-ld=${SNMALLOC_LINKER_FLAVOUR} -Wl,--dynamic-linker=/usr/${triple}/lib/$ENV{RTLD_NAME},-rpath,/usr/${triple}/lib") 12 | if((DEFINED SNMALLOC_LINKER) AND NOT ("${SNMALLOC_LINKER}" MATCHES "^$")) 13 | string(APPEND CROSS_LINKER_FLAGS " --ld-path=${SNMALLOC_LINKER}") 14 | endif() 15 | set(CMAKE_EXE_LINKER_FLAGS ${CROSS_LINKER_FLAGS}) 16 | set(CMAKE_SHARED_LINKER_FLAGS ${CROSS_LINKER_FLAGS}) 17 | set(CMAKE_MODULE_LINKER_FLAGS ${CROSS_LINKER_FLAGS}) 18 | -------------------------------------------------------------------------------- /fuzzing/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@rules_cc//cc:defs.bzl", "cc_test") 2 | 3 | # will raise `ERROR: AddressSanitizer: SEGV on unknown address` if run without -c opt 4 | # --config=asan adds AddressSanitizer libs 5 | # bazel test -c opt --config=asan //fuzzing:snmalloc_fuzzer 6 | cc_test( 7 | name = "snmalloc_fuzzer", 8 | srcs = ["snmalloc-fuzzer.cpp"], 9 | copts = [ 10 | "-fsanitize=address", 11 | ] + select({ 12 | "@bazel_tools//tools/cpp:clang-cl": ["-fexperimental-library"], # needed for std::execution::unseq, 13 | "//conditions:default": ["-mcx16"], 14 | }), 15 | defines = [ 16 | "SNMALLOC_USE_WAIT_ON_ADDRESS=0", 17 | "ADDRESS_SANITIZER", 18 | ], 19 | linkstatic = True, 20 | malloc = "//:snmalloc", 21 | deps = [ 22 | "@fuzztest//fuzztest", 23 | "@fuzztest//fuzztest:fuzztest_gtest_main", 24 | ], 25 | ) 26 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_plain.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../ds_core/ds_core.h" 4 | #include "pal_timer_default.h" 5 | 6 | namespace snmalloc 7 | { 8 | // Can be extended 9 | // Will require a reserve method in subclasses. 10 | template 11 | class PALPlainMixin : public State, public PalTimerDefaultImpl 12 | { 13 | public: 14 | // Notify platform that we will not be using these pages 15 | static void notify_not_using(void*, size_t) noexcept {} 16 | 17 | // Notify platform that we will not be using these pages 18 | template 19 | static bool notify_using(void* p, size_t size) noexcept 20 | { 21 | if constexpr (zero_mem == YesZero) 22 | { 23 | State::zero(p, size); 24 | } 25 | else 26 | { 27 | UNUSED(p, size); 28 | } 29 | return true; 30 | } 31 | }; 32 | } // namespace snmalloc 33 | -------------------------------------------------------------------------------- /src/snmalloc/stl/new.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "snmalloc/ds_core/defines.h" 4 | 5 | namespace snmalloc 6 | { 7 | 8 | // This is used in both vendored and non-vendored mode. 9 | struct PlacementToken 10 | {}; 11 | 12 | inline constexpr PlacementToken placement_token{}; 13 | } // namespace snmalloc 14 | 15 | SNMALLOC_FAST_PATH_INLINE void* 16 | operator new(size_t, void* ptr, snmalloc::PlacementToken) noexcept 17 | { 18 | return ptr; 19 | } 20 | 21 | SNMALLOC_FAST_PATH_INLINE void* 22 | operator new[](size_t, void* ptr, snmalloc::PlacementToken) noexcept 23 | { 24 | return ptr; 25 | } 26 | 27 | // The following is not really needed, but windows expects that new/delete 28 | // definitions are paired. 29 | SNMALLOC_FAST_PATH_INLINE void 30 | operator delete(void*, void*, snmalloc::PlacementToken) noexcept 31 | {} 32 | 33 | SNMALLOC_FAST_PATH_INLINE void 34 | operator delete[](void*, void*, snmalloc::PlacementToken) noexcept 35 | {} 36 | -------------------------------------------------------------------------------- /src/snmalloc/aal/aal_powerpc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #if defined(__powerpc64__) 4 | # define SNMALLOC_VA_BITS_64 5 | #else 6 | # define SNMALLOC_VA_BITS_32 7 | #endif 8 | 9 | namespace snmalloc 10 | { 11 | /** 12 | * ARM-specific architecture abstraction layer. 13 | */ 14 | class AAL_PowerPC 15 | { 16 | public: 17 | /** 18 | * Bitmap of AalFeature flags 19 | */ 20 | static constexpr uint64_t aal_features = IntegerPointers; 21 | 22 | static constexpr enum AalName aal_name = PowerPC; 23 | 24 | static constexpr size_t smallest_page_size = 0x1000; 25 | 26 | /** 27 | * On pipelined processors, notify the core that we are in a spin loop and 28 | * that speculative execution past this point may not be a performance gain. 29 | */ 30 | static inline void pause() 31 | { 32 | __asm__ volatile("or 27,27,27"); // "yield" 33 | } 34 | }; 35 | 36 | using AAL_Arch = AAL_PowerPC; 37 | } // namespace snmalloc 38 | -------------------------------------------------------------------------------- /src/snmalloc/override/malloc-extensions.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /** 4 | * Malloc extensions 5 | * 6 | * This file contains additional non-standard API surface for snmalloc. 7 | * The API is subject to changes, but will be clearly noted in release 8 | * notes. 9 | */ 10 | 11 | /** 12 | * Structure for returning memory used by snmalloc. 13 | * 14 | * The statistics are very coarse grained as they only track 15 | * usage at the superslab/chunk level. Meta-data and object 16 | * data is not tracked independantly. 17 | */ 18 | struct malloc_info_v1 19 | { 20 | /** 21 | * Current memory usage of the allocator. Extremely coarse 22 | * grained for efficient calculation. 23 | */ 24 | size_t current_memory_usage; 25 | 26 | /** 27 | * High-water mark of current_memory_usage. 28 | */ 29 | size_t peak_memory_usage; 30 | }; 31 | 32 | /** 33 | * Populates a malloc_info_v1 structure for the latest values 34 | * from snmalloc. 35 | */ 36 | void get_malloc_info_v1(malloc_info_v1* stats); 37 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/noprange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "range_helpers.h" 3 | 4 | namespace snmalloc 5 | { 6 | struct NopRange 7 | { 8 | template 9 | class Type : public ContainsParent 10 | { 11 | using ContainsParent::parent; 12 | 13 | public: 14 | static constexpr bool Aligned = ParentRange::Aligned; 15 | 16 | static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; 17 | 18 | using ChunkBounds = typename ParentRange::ChunkBounds; 19 | static_assert( 20 | ChunkBounds::address_space_control == 21 | capptr::dimension::AddressSpaceControl::Full); 22 | 23 | constexpr Type() = default; 24 | 25 | CapPtr alloc_range(size_t size) 26 | { 27 | return parent.alloc_range(size); 28 | } 29 | 30 | void dealloc_range(CapPtr base, size_t size) 31 | { 32 | parent.dealloc_range(base, size); 33 | } 34 | }; 35 | }; 36 | } // namespace snmalloc 37 | -------------------------------------------------------------------------------- /src/snmalloc/aal/aal_consts.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | namespace snmalloc 5 | { 6 | /** 7 | * Flags in a bitfield of attributes of this architecture, much like 8 | * PalFeatures. 9 | */ 10 | enum AalFeatures : uint64_t 11 | { 12 | /** 13 | * This architecture does not discriminate between integers and pointers, 14 | * and so may use bit operations on pointer values. 15 | */ 16 | IntegerPointers = (1 << 0), 17 | /** 18 | * This architecture cannot access cpu cycles counters. 19 | */ 20 | NoCpuCycleCounters = (1 << 1), 21 | /** 22 | * This architecture enforces strict pointer provenance; we bound the 23 | * pointers given out on malloc() and friends and must, therefore retain 24 | * internal high-privilege pointers for recycling memory on free(). 25 | */ 26 | StrictProvenance = (1 << 2), 27 | }; 28 | 29 | enum AalName : int 30 | { 31 | ARM, 32 | PowerPC, 33 | X86, 34 | X86_SGX, 35 | Sparc, 36 | RISCV 37 | }; 38 | } // namespace snmalloc 39 | -------------------------------------------------------------------------------- /src/test/func/bits/bits.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Unit tests for operations in bits.h 3 | */ 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | void test_ctz() 10 | { 11 | for (size_t i = 0; i < sizeof(size_t) * 8; i++) 12 | if (snmalloc::bits::ctz(snmalloc::bits::one_at_bit(i)) != i) 13 | { 14 | std::cout << "Failed with ctz(one_at_bit(i)) != i for i=" << i 15 | << std::endl; 16 | abort(); 17 | } 18 | } 19 | 20 | void test_clz() 21 | { 22 | const size_t PTRSIZE_LOG = sizeof(size_t) * 8; 23 | 24 | for (size_t i = 0; i < sizeof(size_t) * 8; i++) 25 | if ( 26 | snmalloc::bits::clz(snmalloc::bits::one_at_bit(i)) != 27 | (PTRSIZE_LOG - i - 1)) 28 | { 29 | std::cout 30 | << "Failed with clz(one_at_bit(i)) != (PTRSIZE_LOG - i - 1) for i=" << i 31 | << std::endl; 32 | abort(); 33 | } 34 | } 35 | 36 | int main(int argc, char** argv) 37 | { 38 | snmalloc::UNUSED(argc, argv); 39 | 40 | setup(); 41 | 42 | test_clz(); 43 | test_ctz(); 44 | } 45 | -------------------------------------------------------------------------------- /src/test/helpers.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifdef _MSC_VER 3 | # define __PRETTY_FUNCTION__ __FUNCSIG__ 4 | #endif 5 | 6 | namespace snmalloc 7 | { 8 | /** 9 | * The name of the function under test. This is set in the START_TEST macro 10 | * and used for error reporting in EXPECT. 11 | */ 12 | const char* current_test = ""; 13 | 14 | /** 15 | * Log that the test started. 16 | */ 17 | #define START_TEST(msg, ...) \ 18 | do \ 19 | { \ 20 | current_test = __PRETTY_FUNCTION__; \ 21 | MessageBuilder<1024> mb{"Starting test: " msg "\n", ##__VA_ARGS__}; \ 22 | DefaultPal::message(mb.get_message()); \ 23 | } while (0) 24 | 25 | /** 26 | * An assertion that fires even in debug builds. Uses the value set by 27 | * START_TEST. 28 | */ 29 | #define EXPECT(x, msg, ...) \ 30 | SNMALLOC_CHECK_MSG(x, " in test {} " msg "\n", current_test, ##__VA_ARGS__) 31 | 32 | #define INFO(msg, ...) \ 33 | do \ 34 | { \ 35 | MessageBuilder<1024> mb{msg "\n", ##__VA_ARGS__}; \ 36 | DefaultPal::message(mb.get_message()); \ 37 | } while (0) 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/test/measuretime.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | class MeasureTime 9 | { 10 | std::stringstream ss; 11 | std::chrono::time_point start = 12 | std::chrono::high_resolution_clock::now(); 13 | 14 | bool quiet = false; 15 | 16 | public: 17 | ~MeasureTime() 18 | { 19 | auto finish = std::chrono::high_resolution_clock::now(); 20 | auto diff = finish - start; 21 | if (!quiet) 22 | { 23 | std::cout << ss.str() << ": " << std::setw(12) << diff.count() << " ns" 24 | << std::endl; 25 | } 26 | } 27 | 28 | MeasureTime(bool quiet = false) : quiet(quiet) {} 29 | 30 | template 31 | MeasureTime& operator<<(const T& s) 32 | { 33 | ss << s; 34 | start = std::chrono::high_resolution_clock::now(); 35 | return *this; 36 | } 37 | 38 | std::chrono::nanoseconds get_time() 39 | { 40 | auto finish = std::chrono::high_resolution_clock::now(); 41 | auto diff = finish - start; 42 | return diff; 43 | } 44 | }; -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/staticrange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../ds/ds.h" 4 | #include "empty_range.h" 5 | 6 | namespace snmalloc 7 | { 8 | /** 9 | * Makes the supplied ParentRange into a global variable. 10 | */ 11 | struct StaticRange 12 | { 13 | template> 14 | class Type : public StaticParent 15 | { 16 | using StaticParent::parent; 17 | 18 | public: 19 | static constexpr bool Aligned = ParentRange::Aligned; 20 | 21 | static_assert( 22 | ParentRange::ConcurrencySafe, 23 | "StaticRange requires a concurrency safe parent."); 24 | 25 | static constexpr bool ConcurrencySafe = true; 26 | 27 | using ChunkBounds = typename ParentRange::ChunkBounds; 28 | 29 | constexpr Type() = default; 30 | 31 | CapPtr alloc_range(size_t size) 32 | { 33 | return parent.alloc_range(size); 34 | } 35 | 36 | void dealloc_range(CapPtr base, size_t size) 37 | { 38 | parent.dealloc_range(base, size); 39 | } 40 | }; 41 | }; 42 | } // namespace snmalloc 43 | -------------------------------------------------------------------------------- /ci/README.md: -------------------------------------------------------------------------------- 1 | # Keeping CI images up to date 2 | 3 | ## Re-building an image after updating the docker file 4 | 5 | Run `docker build -t snmallocciteam/build_${IMG}:latest -f ci/${IMG} .` from the root of the repo, 6 | where `$IMG` is the image you want to rebuild, for example `linux_x64` 7 | 8 | If you are building a multiarch image, ie. an image targeting an architecture other than 9 | the one you are running, you will need to install the qemu handler before you can build: 10 | `sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset` 11 | 12 | ## Testing locally 13 | 14 | Having built a docker image, you can build your current tree therein with 15 | 16 | docker run --rm -u $(id -u) -v ${PWD}:/opt:ro snmallocciteam/build_${IMG}:latest \ 17 | sh -c "cd /tmp; CC=clang-10 CXX=clang++-10 BUILD_TYPE=Debug SNMALLOC_SRC=/opt /opt/ci/scripts/build.sh && (cd build; ninja test)" 18 | 19 | ## Pushing the updated docker image to Docker Hub 20 | 21 | Run `docker push snmallocciteam/$IMG:latest` 22 | 23 | ## Permissions 24 | 25 | You must be part of the [snmalloc ci team](https://hub.docker.com/orgs/snmallocciteam) to 26 | push updated images. Contact @mjp41, or @achamayou to be given access. 27 | -------------------------------------------------------------------------------- /src/snmalloc/snmalloc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Core implementation of snmalloc independent of the configuration mode 4 | #include "snmalloc_core.h" 5 | 6 | // Provides the global configuration for the snmalloc implementation. 7 | #include "backend/globalconfig.h" 8 | 9 | #ifdef SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION 10 | # include "snmalloc/mem/secondary/gwp_asan.h" 11 | #endif 12 | 13 | namespace snmalloc 14 | { 15 | // If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own 16 | // definition of `snmalloc::Alloc` before including any files that include 17 | // `snmalloc.h` or consume the global allocation APIs. 18 | #ifndef SNMALLOC_PROVIDE_OWN_CONFIG 19 | # ifdef SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION 20 | using Config = snmalloc::StandardConfigClientMeta< 21 | NoClientMetaDataProvider, 22 | GwpAsanSecondaryAllocator>; 23 | # else 24 | using Config = snmalloc::StandardConfigClientMeta; 25 | # endif 26 | #endif 27 | /** 28 | * Create allocator type for this configuration. 29 | */ 30 | using Alloc = snmalloc::Allocator; 31 | } // namespace snmalloc 32 | 33 | // User facing API surface, needs to know what `Alloc` is. 34 | #include "snmalloc_front.h" 35 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_haiku.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #if defined(__HAIKU__) 4 | 5 | # include "pal_posix.h" 6 | 7 | # include 8 | 9 | namespace snmalloc 10 | { 11 | /** 12 | * Platform abstraction layer for Haiku. This provides features for this 13 | * system. 14 | */ 15 | class PALHaiku : public PALPOSIX 16 | { 17 | public: 18 | /** 19 | * Bitmap of PalFeatures flags indicating the optional features that this 20 | * PAL supports. 21 | * 22 | */ 23 | static constexpr uint64_t pal_features = PALPOSIX::pal_features | Entropy; 24 | 25 | /** 26 | * Haiku requires an explicit no-reserve flag in `mmap` to guarantee lazy 27 | * commit. 28 | */ 29 | static constexpr int default_mmap_flags = MAP_NORESERVE; 30 | 31 | /** 32 | * Notify platform that we will not be needing these pages. 33 | * Haiku does not provide madvise call per say only the posix equivalent. 34 | */ 35 | static void notify_not_using(void* p, size_t size) noexcept 36 | { 37 | SNMALLOC_ASSERT(is_aligned_block(p, size)); 38 | posix_madvise(p, size, POSIX_MADV_DONTNEED); 39 | } 40 | }; 41 | } // namespace snmalloc 42 | #endif 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /src/snmalloc/mem/check_init.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace snmalloc 4 | { 5 | /** 6 | * This class provides a default implementation for checking for 7 | * initialisation. It does nothing. This is used to allow the thread local 8 | * state to inject code for correctly initialising the thread local state with 9 | * an allocator, but this check is performed off the fast path. 10 | */ 11 | class CheckInitNoOp 12 | { 13 | public: 14 | /** 15 | * @brief 16 | * 17 | * @tparam Success - Lambda type that is called if initialised 18 | * @tparam Restart - Lambda type that is called if the allocator was not 19 | * initialised, and has now been initialised. 20 | * @tparam Args - Arguments to pass to the Restart lambda. 21 | * @param s - the actual success lambda. 22 | * @param r - the restart lambda - ignored in this case. 23 | * @param args - arguments to pass to the restart lambda - ignored in this 24 | * case. 25 | * @return auto - the result of the success or restart lambda 26 | */ 27 | template 28 | static auto check_init(Success s, Restart, Args...) 29 | { 30 | return s(); 31 | } 32 | }; 33 | } // namespace snmalloc -------------------------------------------------------------------------------- /src/snmalloc/stl/cxx/type_traits.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace snmalloc 6 | { 7 | namespace stl 8 | { 9 | using std::add_const_t; 10 | using std::add_lvalue_reference_t; 11 | using std::add_pointer_t; 12 | using std::add_rvalue_reference_t; 13 | using std::bool_constant; 14 | using std::conditional; 15 | using std::conditional_t; 16 | using std::enable_if; 17 | using std::enable_if_t; 18 | using std::false_type; 19 | using std::has_unique_object_representations_v; 20 | using std::integral_constant; 21 | using std::is_base_of_v; 22 | using std::is_copy_assignable_v; 23 | using std::is_copy_constructible_v; 24 | using std::is_integral; 25 | using std::is_integral_v; 26 | using std::is_move_assignable_v; 27 | using std::is_move_constructible_v; 28 | using std::is_same; 29 | using std::is_same_v; 30 | using std::is_trivially_copyable_v; 31 | using std::remove_all_extents_t; 32 | using std::remove_const_t; 33 | using std::remove_cv; 34 | using std::remove_cv_t; 35 | using std::remove_reference; 36 | using std::remove_reference_t; 37 | using std::true_type; 38 | using std::void_t; 39 | } // namespace stl 40 | } // namespace snmalloc 41 | -------------------------------------------------------------------------------- /src/test/func/external_pagemap/external_pagemap.cc: -------------------------------------------------------------------------------- 1 | #if defined(_WIN32) || !defined(TODO_REINSTATE_POSSIBLY) 2 | // This test does not make sense with malloc pass-through, skip it. 3 | // The malloc definitions are also currently incompatible with Windows headers 4 | // so skip this test on Windows as well. 5 | int main() 6 | { 7 | return 0; 8 | } 9 | #else 10 | # define SNMALLOC_EXPOSE_PAGEMAP 1 11 | # include 12 | 13 | using ExternalChunkmap = 14 | ExternalGlobalPagemapTemplate; 15 | 16 | int main() 17 | { 18 | auto& p = ExternalChunkmap::pagemap(); 19 | auto& global = GlobalChunkmap::pagemap(); 20 | SNMALLOC_CHECK(&p == &global); 21 | // Get a valid heap address 22 | uintptr_t addr = unsafe_to_uintptr(malloc(42)); 23 | // Make this very strongly aligned 24 | addr &= ~0xfffffULL; 25 | void* page = p.page_for_address(addr); 26 | SNMALLOC_CHECK(page == p.page_for_address(addr + 128)); 27 | size_t idx = p.index_for_address(addr); 28 | size_t idx2 = p.index_for_address(addr + SUPERSLAB_SIZE); 29 | // If the pagemap ends up storing things that are not uint8_t, this test 30 | // will need modifying. 31 | SNMALLOC_CHECK(idx2 = ((idx + 1) % OS_PAGE_SIZE)); 32 | } 33 | #endif 34 | -------------------------------------------------------------------------------- /src/test/func/multi_atexit/multi_atexit.cc: -------------------------------------------------------------------------------- 1 | #ifndef __has_feature 2 | # define __has_feature(x) 0 3 | #endif 4 | 5 | // These test partially override the libc malloc/free functions to test 6 | // interesting corner cases. This breaks the sanitizers as they will be 7 | // partially overridden. So we disable the tests if any of the sanitizers are 8 | // enabled. 9 | #if defined(__linux__) && !__has_feature(address_sanitizer) && \ 10 | !defined(__SANITIZE_ADDRESS__) && !defined(__SANITIZE_THREAD__) && \ 11 | !defined(SNMALLOC_THREAD_SANITIZER_ENABLED) 12 | # define RUN_TEST 13 | #endif 14 | 15 | #ifdef RUN_TEST 16 | # include 17 | # include 18 | 19 | void do_nothing() {} 20 | 21 | // We only selectively override these functions. Otherwise, malloc may be called 22 | // before atexit triggers the first initialization attempt. 23 | 24 | extern "C" void* calloc(size_t num, size_t size) 25 | { 26 | return snmalloc::libc::calloc(num, size); 27 | } 28 | 29 | extern "C" void free(void* p) 30 | { 31 | if (snmalloc::is_owned(p)) 32 | return snmalloc::libc::free(p); 33 | // otherwise, just leak the memory 34 | } 35 | 36 | #endif 37 | 38 | int main() 39 | { 40 | #ifdef RUN_TEST 41 | for (int i = 0; i < 8192; ++i) 42 | atexit(do_nothing); 43 | #endif 44 | } -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/pagemapregisterrange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../mem/metadata.h" 4 | #include "../pal/pal.h" 5 | #include "empty_range.h" 6 | #include "range_helpers.h" 7 | 8 | namespace snmalloc 9 | { 10 | template 11 | struct PagemapRegisterRange 12 | { 13 | template> 14 | class Type : public ContainsParent 15 | { 16 | using ContainsParent::parent; 17 | 18 | public: 19 | constexpr Type() = default; 20 | 21 | static constexpr bool Aligned = ParentRange::Aligned; 22 | 23 | static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; 24 | 25 | using ChunkBounds = typename ParentRange::ChunkBounds; 26 | 27 | CapPtr alloc_range(size_t size) 28 | { 29 | auto base = parent.alloc_range(size); 30 | 31 | if (base != nullptr) 32 | { 33 | auto result = Pagemap::register_range(base, size); 34 | if (!result) 35 | { 36 | // If register_range fails, typically there is no recovery from this 37 | // so just return nullptr. 38 | return CapPtr(nullptr); 39 | } 40 | } 41 | 42 | return base; 43 | } 44 | }; 45 | }; 46 | } // namespace snmalloc 47 | -------------------------------------------------------------------------------- /src/snmalloc/mem/secondary/default.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "snmalloc/ds_core/defines.h" 4 | #include "snmalloc/ds_core/mitigations.h" 5 | 6 | #include 7 | 8 | namespace snmalloc 9 | { 10 | class DefaultSecondaryAllocator 11 | { 12 | public: 13 | // This flag is used to turn off checks on fast paths if the secondary 14 | // allocator does not own the memory at all. 15 | static constexpr inline bool pass_through = true; 16 | 17 | SNMALLOC_FAST_PATH 18 | static void initialize() {} 19 | 20 | template 21 | SNMALLOC_FAST_PATH static void* allocate(SizeAlign&&) 22 | { 23 | return nullptr; 24 | } 25 | 26 | SNMALLOC_FAST_PATH 27 | static void deallocate(void* pointer) 28 | { 29 | // If pointer is not null, then dealloc has been call on something 30 | // it shouldn't be called on. 31 | // TODO: Should this be tested even in the !CHECK_CLIENT case? 32 | snmalloc_check_client( 33 | mitigations(sanity_checks), 34 | pointer == nullptr, 35 | "Not allocated by snmalloc."); 36 | } 37 | 38 | SNMALLOC_FAST_PATH 39 | static size_t alloc_size(const void*) 40 | { 41 | SNMALLOC_ASSERT( 42 | false && 43 | "secondary alloc_size should never be invoked with default setup"); 44 | return 0; 45 | } 46 | }; 47 | } // namespace snmalloc 48 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_dragonfly.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #if defined(__DragonFly__) && !defined(_KERNEL) 4 | # include "pal_bsd.h" 5 | 6 | namespace snmalloc 7 | { 8 | /** 9 | * DragonflyBSD-specific platform abstraction layer. 10 | * 11 | * This adds DragonFlyBSD-specific aligned allocation to the BSD 12 | * implementation. 13 | */ 14 | class PALDragonfly : public PALBSD 15 | { 16 | public: 17 | /** 18 | * Bitmap of PalFeatures flags indicating the optional features that this 19 | * PAL supports. 20 | * 21 | * The DragonflyBSD PAL does not currently add any features beyond 22 | * of those of the BSD Pal. 23 | * Like FreeBSD, MAP_NORESERVE is implicit. 24 | * This field is declared explicitly to remind anyone modifying this class 25 | * to add new features that they should add any required feature flags. 26 | */ 27 | static constexpr uint64_t pal_features = PALPOSIX::pal_features; 28 | 29 | /** 30 | * Source of Entropy 31 | * 32 | * DragonflyBSD does not have getentropy, so use getrandom instead. 33 | */ 34 | static uint64_t get_entropy64() 35 | { 36 | uint64_t result; 37 | if (getrandom(&result, sizeof(result), 0) != sizeof(result)) 38 | error("Failed to get system randomness"); 39 | return result; 40 | } 41 | }; 42 | } // namespace snmalloc 43 | #endif 44 | -------------------------------------------------------------------------------- /benchmark/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:24.04 2 | 3 | # Pull mimalloc-bench 4 | RUN apt-get update && apt-get install -y --no-install-recommends git gpg ca-certificates python3-numpy 5 | RUN git clone https://github.com/daanx/mimalloc-bench &&\ 6 | cd mimalloc-bench && \ 7 | git reset --hard a4ce904286365c7adfba54f0eea3a2df3fc95bd1 8 | 9 | WORKDIR /mimalloc-bench 10 | # Install dependencies 11 | RUN ./build-bench-env.sh packages 12 | 13 | # Tidy up apt cache 14 | RUN apt-get clean && rm -rf /var/lib/apt/lists/* 15 | 16 | # Build benchmarks 17 | RUN ./build-bench-env.sh bench 18 | 19 | RUN ./build-bench-env.sh redis 20 | 21 | RUN ./build-bench-env.sh rocksdb \ 22 | && find /mimalloc-bench/extern/rocksdb-8.1.1 -name "*.o" -delete 23 | 24 | RUN ./build-bench-env.sh lean \ 25 | && find /mimalloc-bench/extern/lean -name "*.o" -delete 26 | 27 | RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt 28 | 29 | # Build allocator 30 | RUN mkdir -p /snmalloc 31 | COPY . /snmalloc 32 | 33 | RUN mkdir -p /snmalloc/build 34 | WORKDIR /snmalloc/build 35 | RUN cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. 36 | RUN ninja libsnmallocshim.so 37 | 38 | # Run benchmarks 39 | ARG benchs=allt 40 | ARG repeats=1 41 | WORKDIR /mimalloc-bench/out/bench 42 | RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats 43 | 44 | WORKDIR / 45 | RUN python3 /mimalloc-bench/scripts/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv -------------------------------------------------------------------------------- /.clang-tidy: -------------------------------------------------------------------------------- 1 | Checks: '-clang-analyzer-security.insecureAPI.bzero,clang-diagnostic-*,google-readability-casting,readability-else-after-return,performance-unnecessary-copy-initialization,bugprone-use-after-move,modernize-use-nullptr,modernize-redundant-void-arg,modernize-return-braced-init-list,modernize-use-default-member-init,modernize-use-equals-default,modernize-use-equals-delete,modernize-use-nodiscard,modernize-use-override,cppcoreguidelines-avoid-goto,misc-unconventional-assign-operator,cppcoreguidelines-narrowing-conversions,bugprone-assert-side-effect,bugprone-bool-pointer-implicit-conversion,bugprone-copy-constructor-init,bugprone-forward-declaration-namespace,bugprone-forwarding-reference-overload,bugprone-macro-parentheses,bugprone-macro-repeated-side-effects,bugprone-move-forwarding-reference,bugprone-misplaced-widening-cast,bugprone-swapped-arguments,bugprone-undelegated-constructor,bugprone-unused-raii,cert-dcl21-cpp,llvm-namespace-comment,misc-static-assert,misc-redundant-expression,modernize-loop-convert,modernize-use-using,performance-noexcept-move-constructor,readability-non-const-parameter' 2 | # It would be nice to enable: 3 | # - readability-magic-numbers 4 | # - modernize-avoid-c-arrays 5 | # - cppcoreguidelines-pro-bounds-array-to-pointer-decay (assert breaks it). 6 | # - readability-braces-around-statements (mostly works, but is very confused by constexpr if). 7 | CheckOptions: 8 | - key: modernize-use-default-member-init.UseAssignment 9 | value: '1' 10 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/indirectrange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../ds/ds.h" 4 | #include "empty_range.h" 5 | 6 | namespace snmalloc 7 | { 8 | /** 9 | * Stores a references to the parent range so that it can be shared 10 | * without `static` scope. 11 | * 12 | * This could be used to allow multiple allocators on a single region of 13 | * memory. 14 | */ 15 | struct IndirectRange 16 | { 17 | template> 18 | class Type : public RefParent 19 | { 20 | using RefParent::parent; 21 | 22 | public: 23 | static constexpr bool Aligned = ParentRange::Aligned; 24 | 25 | static_assert( 26 | ParentRange::ConcurrencySafe, 27 | "IndirectRange requires a concurrency safe parent."); 28 | 29 | static constexpr bool ConcurrencySafe = true; 30 | 31 | using ChunkBounds = typename ParentRange::ChunkBounds; 32 | 33 | constexpr Type() = default; 34 | 35 | CapPtr alloc_range(size_t size) 36 | { 37 | return parent->alloc_range(size); 38 | } 39 | 40 | void dealloc_range(CapPtr base, size_t size) 41 | { 42 | parent->dealloc_range(base, size); 43 | } 44 | 45 | /** 46 | * Points the parent reference to the given range. 47 | */ 48 | void set_parent(ParentRange* p) 49 | { 50 | parent = p; 51 | } 52 | }; 53 | }; 54 | } // namespace snmalloc 55 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/palrange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "../pal/pal.h" 3 | 4 | namespace snmalloc 5 | { 6 | template 7 | class PalRange 8 | { 9 | public: 10 | static constexpr bool Aligned = pal_supports; 11 | 12 | // Note we have always assumed the Pals to provide a concurrency safe 13 | // API. If in the future this changes, then this would 14 | // need to be changed. 15 | static constexpr bool ConcurrencySafe = true; 16 | 17 | using ChunkBounds = capptr::bounds::Arena; 18 | 19 | constexpr PalRange() = default; 20 | 21 | capptr::Arena alloc_range(size_t size) 22 | { 23 | if (bits::next_pow2_bits(size) >= bits::BITS - 1) 24 | { 25 | return nullptr; 26 | } 27 | 28 | if constexpr (pal_supports) 29 | { 30 | SNMALLOC_ASSERT(size >= PAL::minimum_alloc_size); 31 | auto result = capptr::Arena::unsafe_from( 32 | PAL::template reserve_aligned(size)); 33 | 34 | #ifdef SNMALLOC_TRACING 35 | message<1024>("Pal range alloc: {} ({})", result.unsafe_ptr(), size); 36 | #endif 37 | return result; 38 | } 39 | else 40 | { 41 | auto result = capptr::Arena::unsafe_from(PAL::reserve(size)); 42 | 43 | #ifdef SNMALLOC_TRACING 44 | message<1024>("Pal range alloc: {} ({})", result.unsafe_ptr(), size); 45 | #endif 46 | 47 | return result; 48 | } 49 | } 50 | }; 51 | } // namespace snmalloc 52 | -------------------------------------------------------------------------------- /src/test/usage.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #if defined(_WIN32) 4 | # define WIN32_LEAN_AND_MEAN 5 | # ifndef NOMINMAX 6 | # define NOMINMAX 7 | # endif 8 | # include 9 | // Needs to be included after windows.h 10 | # include 11 | #endif 12 | 13 | #include 14 | #include 15 | 16 | namespace usage 17 | { 18 | void print_memory() 19 | { 20 | #if defined(_WIN32) 21 | PROCESS_MEMORY_COUNTERS_EX pmc; 22 | 23 | if (!GetProcessMemoryInfo( 24 | GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) 25 | return; 26 | 27 | std::cout << "Memory info:" << std::endl 28 | << "\tPageFaultCount: " << pmc.PageFaultCount << std::endl 29 | << "\tPeakWorkingSetSize: " << pmc.PeakWorkingSetSize << std::endl 30 | << "\tWorkingSetSize: " << pmc.WorkingSetSize << std::endl 31 | << "\tQuotaPeakPagedPoolUsage: " << pmc.QuotaPeakPagedPoolUsage 32 | << std::endl 33 | << "\tQuotaPagedPoolUsage: " << pmc.QuotaPagedPoolUsage 34 | << std::endl 35 | << "\tQuotaPeakNonPagedPoolUsage: " 36 | << pmc.QuotaPeakNonPagedPoolUsage << std::endl 37 | << "\tQuotaNonPagedPoolUsage: " << pmc.QuotaNonPagedPoolUsage 38 | << std::endl 39 | << "\tPagefileUsage: " << pmc.PagefileUsage << std::endl 40 | << "\tPeakPagefileUsage: " << pmc.PeakPagefileUsage << std::endl 41 | << "\tPrivateUsage: " << pmc.PrivateUsage << std::endl; 42 | #endif 43 | } 44 | }; 45 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/lockrange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../ds/ds.h" 4 | #include "empty_range.h" 5 | 6 | namespace snmalloc 7 | { 8 | /** 9 | * Protect the ParentRange with a spin lock. 10 | * 11 | * Accesses via the ancestor() mechanism will bypass the lock and so 12 | * should be used only where the resulting data races are acceptable. 13 | */ 14 | struct LockRange 15 | { 16 | template> 17 | class Type : public ContainsParent 18 | { 19 | using ContainsParent::parent; 20 | 21 | /** 22 | * This is infrequently used code, a spin lock simplifies the code 23 | * considerably, and should never be on the fast path. 24 | */ 25 | CombiningLock spin_lock{}; 26 | 27 | public: 28 | static constexpr bool Aligned = ParentRange::Aligned; 29 | 30 | using ChunkBounds = typename ParentRange::ChunkBounds; 31 | 32 | static constexpr bool ConcurrencySafe = true; 33 | 34 | constexpr Type() = default; 35 | 36 | CapPtr alloc_range(size_t size) 37 | { 38 | CapPtr result; 39 | with(spin_lock, [&]() { 40 | { 41 | result = parent.alloc_range(size); 42 | } 43 | }); 44 | return result; 45 | } 46 | 47 | void dealloc_range(CapPtr base, size_t size) 48 | { 49 | with(spin_lock, [&]() { parent.dealloc_range(base, size); }); 50 | } 51 | }; 52 | }; 53 | } // namespace snmalloc 54 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_bsd.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "pal_posix.h" 4 | 5 | #include 6 | 7 | namespace snmalloc 8 | { 9 | /** 10 | * Generic *BSD PAL mixin. This provides features that are common to the BSD 11 | * family. 12 | */ 13 | template 14 | class PALBSD : public PALPOSIX 15 | { 16 | public: 17 | /** 18 | * Bitmap of PalFeatures flags indicating the optional features that this 19 | * PAL supports. 20 | * 21 | * The generic BSD PAL does not add any features that are not supported by 22 | * generic POSIX systems, but explicitly declares this variable to remind 23 | * anyone who extends this class that they may need to modify this field. 24 | */ 25 | static constexpr uint64_t pal_features = PALPOSIX::pal_features; 26 | 27 | /** 28 | * Notify platform that we will not be using these pages. 29 | * 30 | * BSD systems provide the `MADV_FREE` flag to `madvise`, which allows the 31 | * operating system to replace the pages with CoW copies of a zero page at 32 | * any point between the call and the next write to that page. 33 | */ 34 | static void notify_not_using(void* p, size_t size) noexcept 35 | { 36 | SNMALLOC_ASSERT(is_aligned_block(p, size)); 37 | 38 | if constexpr (Debug) 39 | memset(p, 0x5a, size); 40 | 41 | madvise(p, size, MADV_FREE); 42 | 43 | if constexpr (mitigations(pal_enforce_access)) 44 | { 45 | mprotect(p, size, PROT_NONE); 46 | } 47 | } 48 | }; 49 | } // namespace snmalloc 50 | -------------------------------------------------------------------------------- /src/snmalloc/aal/aal_sparc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #if defined(__arch64__) // More reliable than __sparc64__ 4 | # define SNMALLOC_VA_BITS_64 5 | #else 6 | # define SNMALLOC_VA_BITS_32 7 | #endif 8 | 9 | namespace snmalloc 10 | { 11 | /** 12 | * Sparc architecture abstraction layer. 13 | */ 14 | class AAL_Sparc 15 | { 16 | public: 17 | /** 18 | * Bitmap of AalFeature flags 19 | */ 20 | static constexpr uint64_t aal_features = IntegerPointers; 21 | 22 | static constexpr enum AalName aal_name = Sparc; 23 | 24 | #ifdef SNMALLOC_VA_BITS_64 25 | /** 26 | * Even Ultra-Sparc I supports 8192 and onwards 27 | */ 28 | static constexpr size_t smallest_page_size = 0x2000; 29 | #else 30 | static constexpr size_t smallest_page_size = 0x1000; 31 | #endif 32 | 33 | /** 34 | * On Sparc ideally pause instructions ought to be 35 | * optimised per Sparc processor but here a version 36 | * as least common denominator to avoid numerous ifdef, 37 | * reading Conditions Code Register here 38 | */ 39 | static inline void pause() 40 | { 41 | __asm__ volatile("rd %%ccr, %%g0" ::: "memory"); 42 | } 43 | 44 | static inline void prefetch(void* ptr) 45 | { 46 | #ifdef SNMALLOC_VA_BITS_64 47 | __asm__ volatile("prefetch [%0], 0" ::"r"(ptr)); 48 | #else 49 | UNUSED(ptr); 50 | #endif 51 | } 52 | 53 | static inline uint64_t tick() 54 | { 55 | uint64_t tick; 56 | __asm__ volatile("rd %%asr4, %0" : "=r"(tick)); 57 | return tick; 58 | } 59 | }; 60 | 61 | using AAL_Arch = AAL_Sparc; 62 | } // namespace snmalloc 63 | -------------------------------------------------------------------------------- /src/test/func/two_alloc_types/main.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | extern "C" void* oe_memset_s(void* p, size_t p_size, int c, size_t size) 8 | { 9 | snmalloc::UNUSED(p_size); 10 | return memset(p, c, size); 11 | } 12 | 13 | extern "C" int oe_random(void* p, size_t p_size) 14 | { 15 | snmalloc::UNUSED(p_size, p); 16 | // Stub for random data. 17 | return 0; 18 | } 19 | 20 | extern "C" void oe_abort() 21 | { 22 | abort(); 23 | } 24 | 25 | extern "C" void oe_allocator_init(void* base, void* end); 26 | extern "C" void* host_malloc(size_t); 27 | extern "C" void host_free(void*); 28 | 29 | extern "C" void* enclave_malloc(size_t); 30 | extern "C" void enclave_free(void*); 31 | 32 | using namespace snmalloc; 33 | 34 | int main() 35 | { 36 | setup(); 37 | 38 | // 26 is large enough to produce a nested allocator. 39 | // many other sizes would work. 40 | size_t length = bits::one_at_bit(26); 41 | auto oe_base = host_malloc(length); 42 | 43 | auto oe_end = pointer_offset(oe_base, length); 44 | oe_allocator_init(oe_base, oe_end); 45 | 46 | std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl; 47 | 48 | auto a = host_malloc(128); 49 | auto b = enclave_malloc(128); 50 | 51 | std::cout << "Host alloc " << a << std::endl; 52 | std::cout << "Enclave alloc " << b << std::endl; 53 | 54 | host_free(a); 55 | std::cout << "Host freed!" << std::endl; 56 | enclave_free(b); 57 | std::cout << "Enclace freed!" << std::endl; 58 | } 59 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_open_enclave.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "pal_noalloc.h" 4 | 5 | #ifdef OPEN_ENCLAVE 6 | extern "C" void* oe_memset_s(void* p, size_t p_size, int c, size_t size); 7 | extern "C" int oe_random(void* data, size_t size); 8 | extern "C" [[noreturn]] void oe_abort(); 9 | 10 | namespace snmalloc 11 | { 12 | class OpenEnclaveErrorHandler 13 | { 14 | public: 15 | static void print_stack_trace() {} 16 | 17 | [[noreturn]] static void error(const char* const str) 18 | { 19 | UNUSED(str); 20 | oe_abort(); 21 | } 22 | 23 | static constexpr size_t address_bits = Aal::address_bits; 24 | static constexpr size_t page_size = Aal::smallest_page_size; 25 | }; 26 | 27 | using OpenEnclaveBasePAL = PALNoAlloc; 28 | 29 | class PALOpenEnclave : public OpenEnclaveBasePAL 30 | { 31 | public: 32 | /** 33 | * Bitmap of PalFeatures flags indicating the optional features that this 34 | * PAL supports. 35 | */ 36 | static constexpr uint64_t pal_features = 37 | OpenEnclaveBasePAL::pal_features | Entropy; 38 | 39 | template 40 | static void zero(void* p, size_t size) noexcept 41 | { 42 | oe_memset_s(p, size, 0, size); 43 | } 44 | 45 | /** 46 | * Source of Entropy 47 | */ 48 | static uint64_t get_entropy64() 49 | { 50 | uint64_t result = 0; 51 | if (oe_random(&result, sizeof(result)) != OE_OK) 52 | error("Failed to get system randomness"); 53 | return result; 54 | } 55 | }; 56 | } 57 | #endif 58 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/subrange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "../mem/mem.h" 3 | #include "empty_range.h" 4 | 5 | namespace snmalloc 6 | { 7 | /** 8 | * Creates an area inside a large allocation that is larger by 9 | * 2^RATIO_BITS. Will not return a the block at the start or 10 | * the end of the large allocation. 11 | */ 12 | template 13 | struct SubRange 14 | { 15 | template> 16 | class Type : public ContainsParent 17 | { 18 | using ContainsParent::parent; 19 | 20 | public: 21 | constexpr Type() = default; 22 | 23 | static constexpr bool Aligned = ParentRange::Aligned; 24 | 25 | static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; 26 | 27 | using ChunkBounds = typename ParentRange::ChunkBounds; 28 | 29 | CapPtr alloc_range(size_t sub_size) 30 | { 31 | SNMALLOC_ASSERT(bits::is_pow2(sub_size)); 32 | 33 | auto full_size = sub_size << RATIO_BITS; 34 | auto overblock = parent.alloc_range(full_size); 35 | if (overblock == nullptr) 36 | return nullptr; 37 | 38 | size_t offset_mask = full_size - sub_size; 39 | // Don't use first or last block in the larger reservation 40 | // Loop required to get uniform distribution. 41 | size_t offset; 42 | do 43 | { 44 | offset = get_entropy64() & offset_mask; 45 | } while ((offset == 0) || (offset == offset_mask)); 46 | 47 | return pointer_offset(overblock, offset); 48 | } 49 | }; 50 | }; 51 | } // namespace snmalloc 52 | -------------------------------------------------------------------------------- /src/test/func/release-rounding/rounding.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | using namespace snmalloc; 5 | 6 | // Check for all sizeclass that we correctly round every offset within 7 | // a superslab to the correct value, by comparing with the standard 8 | // unoptimised version using division. 9 | // Also check we correctly determine multiples using optimized check. 10 | 11 | int main(int argc, char** argv) 12 | { 13 | setup(); 14 | 15 | UNUSED(argc); 16 | UNUSED(argv); 17 | 18 | bool failed = false; 19 | 20 | for (size_t size_class = 0; size_class < NUM_SMALL_SIZECLASSES; size_class++) 21 | { 22 | size_t rsize = sizeclass_to_size((uint8_t)size_class); 23 | size_t max_offset = sizeclass_to_slab_size(size_class); 24 | sizeclass_t sc = sizeclass_t::from_small_class(size_class); 25 | for (size_t offset = 0; offset < max_offset; offset++) 26 | { 27 | size_t mod = offset % rsize; 28 | bool mod_0 = (offset % rsize) == 0; 29 | 30 | size_t opt_mod = index_in_object(sc, offset); 31 | if (mod != opt_mod) 32 | { 33 | std::cout << "rsize " << rsize << " offset " << offset << " opt " 34 | << opt_mod << " correct " << mod << std::endl 35 | << std::flush; 36 | failed = true; 37 | } 38 | 39 | bool opt_mod_0 = is_start_of_object(sc, offset); 40 | if (opt_mod_0 != mod_0) 41 | { 42 | std::cout << "rsize " << rsize << " offset " << offset 43 | << " opt_mod0 " << opt_mod_0 << " correct " << mod_0 44 | << std::endl 45 | << std::flush; 46 | failed = true; 47 | } 48 | } 49 | if (failed) 50 | abort(); 51 | } 52 | return 0; 53 | } 54 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_bsd_aligned.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "pal_bsd.h" 4 | 5 | namespace snmalloc 6 | { 7 | /** 8 | * FreeBSD-specific platform abstraction layer. 9 | * 10 | * This adds aligned allocation using `MAP_ALIGNED` to the generic BSD 11 | * implementation. This flag is supported by NetBSD and FreeBSD. 12 | */ 13 | template 14 | class PALBSD_Aligned : public PALBSD 15 | { 16 | public: 17 | /** 18 | * Bitmap of PalFeatures flags indicating the optional features that this 19 | * PAL supports. 20 | * 21 | * This class adds support for aligned allocation. 22 | */ 23 | static constexpr uint64_t pal_features = 24 | AlignedAllocation | PALBSD::pal_features; 25 | 26 | static SNMALLOC_CONSTINIT_STATIC size_t minimum_alloc_size = 27 | aal_supports ? 1 << 24 : 4096; 28 | 29 | /** 30 | * Reserve memory at a specific alignment. 31 | */ 32 | template 33 | static void* reserve_aligned(size_t size) noexcept 34 | { 35 | // Alignment must be a power of 2. 36 | SNMALLOC_ASSERT(bits::is_pow2(size)); 37 | SNMALLOC_ASSERT(size >= minimum_alloc_size); 38 | 39 | int log2align = static_cast(bits::next_pow2_bits(size)); 40 | 41 | auto prot = state_using || !mitigations(pal_enforce_access) ? 42 | PROT_READ | PROT_WRITE : 43 | PROT_NONE; 44 | 45 | void* p = mmap( 46 | nullptr, 47 | size, 48 | prot, 49 | MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED(log2align) | 50 | OS::extra_mmap_flags(state_using), 51 | -1, 52 | 0); 53 | 54 | if (p == MAP_FAILED) 55 | return nullptr; 56 | 57 | return p; 58 | } 59 | }; 60 | } // namespace snmalloc 61 | -------------------------------------------------------------------------------- /src/snmalloc/aal/aal_x86_sgx.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifdef _MSC_VER 4 | # include 5 | # include 6 | #endif 7 | 8 | #if defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \ 9 | defined(_M_AMD64) 10 | # define SNMALLOC_VA_BITS_64 11 | #else 12 | # define SNMALLOC_VA_BITS_32 13 | #endif 14 | 15 | namespace snmalloc 16 | { 17 | /** 18 | * x86-specific architecture abstraction layer minimised for use 19 | * inside SGX enclaves. 20 | */ 21 | class AAL_x86_sgx 22 | { 23 | public: 24 | /** 25 | * Bitmap of AalFeature flags 26 | */ 27 | static constexpr uint64_t aal_features = IntegerPointers; 28 | 29 | static constexpr enum AalName aal_name = X86_SGX; 30 | 31 | static constexpr size_t smallest_page_size = 0x1000; 32 | 33 | /** 34 | * On pipelined processors, notify the core that we are in a spin loop and 35 | * that speculative execution past this point may not be a performance gain. 36 | */ 37 | static inline void pause() 38 | { 39 | #ifdef _MSC_VER 40 | _mm_pause(); 41 | #else 42 | asm volatile("pause"); 43 | #endif 44 | } 45 | 46 | /** 47 | * Issue a prefetch hint at the specified address. 48 | */ 49 | static inline void prefetch(void* ptr) 50 | { 51 | #ifdef _MSC_VER 52 | _mm_prefetch(reinterpret_cast(ptr), _MM_HINT_T0); 53 | #else 54 | asm volatile("prefetcht0 %0" ::"m"(ptr)); 55 | #endif 56 | } 57 | 58 | /** 59 | * Return a cycle counter value. 60 | * Not guaranteed inside an enclave, so just always return 0. 61 | * This is only used for benchmarking inside snmalloc. 62 | */ 63 | static inline uint64_t tick() 64 | { 65 | return 0; 66 | } 67 | }; 68 | 69 | using AAL_Arch = AAL_x86_sgx; 70 | } // namespace snmalloc 71 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/logrange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "empty_range.h" 4 | #include "range_helpers.h" 5 | 6 | namespace snmalloc 7 | { 8 | /** 9 | * RangeName is an integer to specify which range is being logged. Strings can 10 | * be used as template parameters. 11 | * 12 | * ParentRange is what the range is logging calls to. 13 | */ 14 | template 15 | struct LogRange 16 | { 17 | template> 18 | class Type : public ContainsParent 19 | { 20 | using ContainsParent::parent; 21 | 22 | public: 23 | static constexpr bool Aligned = ParentRange::Aligned; 24 | 25 | static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; 26 | 27 | using ChunkBounds = typename ParentRange::ChunkBounds; 28 | 29 | constexpr Type() = default; 30 | 31 | CapPtr alloc_range(size_t size) 32 | { 33 | #ifdef SNMALLOC_TRACING 34 | message<1024>("Call alloc_range({}) on {}", size, RangeName); 35 | #endif 36 | auto range = parent.alloc_range(size); 37 | #ifdef SNMALLOC_TRACING 38 | message<1024>( 39 | "{} = alloc_range({}) in {}", range.unsafe_ptr(), size, RangeName); 40 | #endif 41 | return range; 42 | } 43 | 44 | void dealloc_range(CapPtr base, size_t size) 45 | { 46 | #ifdef SNMALLOC_TRACING 47 | message<1024>( 48 | "dealloc_range({}, {}}) on {}", base.unsafe_ptr(), size, RangeName); 49 | #endif 50 | parent.dealloc_range(base, size); 51 | #ifdef SNMALLOC_TRACING 52 | message<1024>( 53 | "Done dealloc_range({}, {}})! on {}", 54 | base.unsafe_ptr(), 55 | size, 56 | RangeName); 57 | #endif 58 | } 59 | }; 60 | }; 61 | } // namespace snmalloc 62 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_solaris.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #if defined(__sun) 4 | # include "pal_posix.h" 5 | 6 | namespace snmalloc 7 | { 8 | /** 9 | * Platform abstraction layer for Solaris. This provides features for this 10 | * system. 11 | */ 12 | class PALSolaris : public PALPOSIX 13 | { 14 | public: 15 | /** 16 | * Bitmap of PalFeatures flags indicating the optional features that this 17 | * PAL supports. 18 | * 19 | */ 20 | static constexpr uint64_t pal_features = 21 | AlignedAllocation | PALPOSIX::pal_features; 22 | 23 | static constexpr size_t page_size = 24 | Aal::aal_name == Sparc ? 0x2000 : 0x1000; 25 | static constexpr size_t minimum_alloc_size = page_size; 26 | 27 | /** 28 | * Solaris requires an explicit no-reserve flag in `mmap` to guarantee lazy 29 | * commit. 30 | */ 31 | static constexpr int default_mmap_flags = MAP_NORESERVE; 32 | 33 | /** 34 | * Reserve memory at a specific alignment. 35 | */ 36 | template 37 | static void* reserve_aligned(size_t size) noexcept 38 | { 39 | // Alignment must be a power of 2. 40 | SNMALLOC_ASSERT(bits::is_pow2(size)); 41 | SNMALLOC_ASSERT(size >= minimum_alloc_size); 42 | UNUSED(state_using); 43 | 44 | uintptr_t alignment = 45 | static_cast(bits::align_up(size, page_size)); 46 | 47 | auto prot = 48 | !mitigations(pal_enforce_access) ? PROT_READ | PROT_WRITE : PROT_NONE; 49 | 50 | void* p = mmap( 51 | (void*)alignment, 52 | size, 53 | prot, 54 | MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGN | default_mmap_flags, 55 | -1, 56 | 0); 57 | 58 | if (p == MAP_FAILED) 59 | return nullptr; 60 | 61 | return p; 62 | } 63 | }; 64 | } // namespace snmalloc 65 | #endif 66 | -------------------------------------------------------------------------------- /src/test/func/multi_threadatexit/multi_threadatexit.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #ifndef __has_feature 4 | # define __has_feature(x) 0 5 | #endif 6 | 7 | // These test partially override the libc malloc/free functions to test 8 | // interesting corner cases. This breaks the sanitizers as they will be 9 | // partially overridden. So we disable the tests if any of the sanitizers are 10 | // enabled. 11 | #if defined(__linux__) && !__has_feature(address_sanitizer) && \ 12 | !defined(__SANITIZE_ADDRESS__) && !defined(__SANITIZE_THREAD__) && \ 13 | !defined(SNMALLOC_THREAD_SANITIZER_ENABLED) 14 | # define RUN_TEST 15 | #endif 16 | 17 | #ifdef RUN_TEST 18 | # include 19 | # include 20 | 21 | template 22 | void thread_destruct() 23 | { 24 | snmalloc::message("thread_destruct<{}, {}> start", N, M); 25 | static thread_local snmalloc::OnDestruct destruct{ 26 | []() { snmalloc::message("thread_destruct<{}, {}> destructor", N, M); }}; 27 | snmalloc::message("thread_destruct<{}, {}> end", N, M); 28 | 29 | if constexpr (N > M + 1) 30 | { 31 | // destructor 32 | thread_destruct(); 33 | thread_destruct<(M + N) / 2, M>(); 34 | } 35 | } 36 | 37 | // We only selectively override these functions. Otherwise, malloc may be called 38 | // before atexit triggers the first initialization attempt. 39 | 40 | extern "C" void* calloc(size_t num, size_t size) 41 | { 42 | snmalloc::message("calloc({}, {})", num, size); 43 | return snmalloc::libc::calloc(num, size); 44 | } 45 | 46 | extern "C" void free(void* p) 47 | { 48 | snmalloc::message("free({})", p); 49 | if (snmalloc::is_owned(p)) 50 | return snmalloc::libc::free(p); 51 | // otherwise, just leak the memory 52 | } 53 | 54 | int main() 55 | { 56 | std::thread(thread_destruct<1000>).join(); 57 | } 58 | #else 59 | int main() 60 | { 61 | return 0; 62 | } 63 | #endif 64 | -------------------------------------------------------------------------------- /src/snmalloc/aal/aal_arm.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) 4 | # define SNMALLOC_VA_BITS_64 5 | # ifdef _MSC_VER 6 | # include 7 | # endif 8 | #else 9 | # define SNMALLOC_VA_BITS_32 10 | # ifdef _MSC_VER 11 | # include 12 | # endif 13 | #endif 14 | 15 | #include 16 | 17 | namespace snmalloc 18 | { 19 | /** 20 | * ARM-specific architecture abstraction layer. 21 | */ 22 | class AAL_arm 23 | { 24 | public: 25 | /** 26 | * Bitmap of AalFeature flags 27 | */ 28 | static constexpr uint64_t aal_features = IntegerPointers 29 | #if defined(SNMALLOC_VA_BITS_32) || !defined(__APPLE__) 30 | | NoCpuCycleCounters 31 | #endif 32 | ; 33 | 34 | static constexpr enum AalName aal_name = ARM; 35 | 36 | static constexpr size_t smallest_page_size = 0x1000; 37 | 38 | /** 39 | * On pipelined processors, notify the core that we are in a spin loop and 40 | * that speculative execution past this point may not be a performance gain. 41 | */ 42 | static inline void pause() 43 | { 44 | #ifdef _MSC_VER 45 | __yield(); 46 | #else 47 | __asm__ volatile("yield"); 48 | #endif 49 | } 50 | 51 | static inline void prefetch(void* ptr) 52 | { 53 | #ifdef _MSC_VER 54 | __prefetch(ptr); 55 | #elif __has_builtin(__builtin_prefetch) && !defined(SNMALLOC_NO_AAL_BUILTINS) 56 | __builtin_prefetch(ptr); 57 | #elif defined(SNMALLOC_VA_BITS_64) 58 | __asm__ volatile("prfm pstl1keep, [%0]" : "=r"(ptr)); 59 | #else 60 | __asm__ volatile("pld\t[%0]" : "=r"(ptr)); 61 | #endif 62 | } 63 | 64 | #if defined(SNMALLOC_VA_BITS_64) && defined(__APPLE__) 65 | static inline uint64_t tick() noexcept 66 | { 67 | uint64_t t; 68 | __asm__ volatile("mrs %0, cntvct_el0" : "=r"(t)); 69 | return t; 70 | } 71 | #endif 72 | }; 73 | 74 | using AAL_Arch = AAL_arm; 75 | } // namespace snmalloc 76 | -------------------------------------------------------------------------------- /src/test/opt.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace opt 9 | { 10 | class Opt 11 | { 12 | private: 13 | int argc; 14 | const char* const* argv; 15 | 16 | public: 17 | Opt(int argc, const char* const* argv) : argc(argc), argv(argv) {} 18 | 19 | bool has(const char* opt) 20 | { 21 | for (int i = 1; i < argc; i++) 22 | { 23 | if (!strcmp(opt, argv[i])) 24 | return true; 25 | } 26 | 27 | return false; 28 | } 29 | 30 | template 31 | T is(const char* opt, T def) 32 | { 33 | size_t len = strlen(opt); 34 | 35 | for (int i = 1; i < argc; i++) 36 | { 37 | const char* p = param(opt, len, i); 38 | 39 | if (p != nullptr) 40 | { 41 | char* end = nullptr; 42 | T r; 43 | 44 | if (std::is_unsigned::value) 45 | r = (T)strtoull(p, &end, 10); 46 | else 47 | r = (T)strtoll(p, &end, 10); 48 | 49 | if ((r == 0) && (end == p)) 50 | return def; 51 | 52 | return r; 53 | } 54 | } 55 | 56 | return def; 57 | } 58 | 59 | const char* is(const char* opt, const char* def) 60 | { 61 | size_t len = strlen(opt); 62 | 63 | for (int i = 1; i < argc; i++) 64 | { 65 | const char* p = param(opt, len, i); 66 | 67 | if (p != nullptr) 68 | return p; 69 | } 70 | 71 | return def; 72 | } 73 | 74 | private: 75 | const char* param(const char* opt, size_t len, int i) 76 | { 77 | if (strncmp(opt, argv[i], len)) 78 | return nullptr; 79 | 80 | switch (argv[i][len]) 81 | { 82 | case '\0': 83 | return (i < (argc - 1)) ? argv[i + 1] : nullptr; 84 | case '=': 85 | return &argv[i][len + 1]; 86 | default: 87 | return nullptr; 88 | } 89 | } 90 | }; 91 | } 92 | -------------------------------------------------------------------------------- /src/snmalloc/mem/pooled.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../ds/ds.h" 4 | #include "backend_concept.h" 5 | 6 | namespace snmalloc 7 | { 8 | template 9 | struct Range 10 | { 11 | CapPtr base; 12 | size_t length; 13 | }; 14 | 15 | template 16 | class PoolState; 17 | 18 | // clang-format off 19 | #ifdef __cpp_concepts 20 | template 21 | concept Constructable = requires() { 22 | { C::make() } -> ConceptSame>; 23 | }; 24 | #endif // __cpp_concepts 25 | 26 | 27 | /** 28 | * Required to be implemented by all types that are pooled. 29 | * 30 | * The constructor of any inherited type must take a Range& as its first 31 | * argument. This represents the leftover from pool allocation rounding up to 32 | * the nearest power of 2. It is valid to ignore this argument, but can be 33 | * used to optimise meta-data usage at startup. 34 | */ 35 | template 36 | class Pooled 37 | { 38 | public: 39 | template< 40 | typename TT, 41 | SNMALLOC_CONCEPT(Constructable) Construct, 42 | PoolState& get_state()> 43 | friend class Pool; 44 | 45 | /// Used by the pool for chaining together entries when not in use. 46 | capptr::Alloc next{nullptr}; 47 | /// Used by the pool to keep the list of all entries ever created. 48 | capptr::Alloc list_next; 49 | stl::Atomic in_use{false}; 50 | 51 | public: 52 | void set_in_use() 53 | { 54 | if (in_use.exchange(true)) 55 | error("Critical error: double use of Pooled Type!"); 56 | } 57 | 58 | void reset_in_use() 59 | { 60 | in_use.store(false); 61 | } 62 | 63 | bool debug_is_in_use() 64 | { 65 | bool result = in_use.exchange(true); 66 | if (!result) 67 | in_use.store(false); 68 | return result; 69 | } 70 | }; 71 | } // namespace snmalloc 72 | -------------------------------------------------------------------------------- /src/snmalloc/override/rust.cc: -------------------------------------------------------------------------------- 1 | #define SNMALLOC_NAME_MANGLE(a) sn_##a 2 | 3 | // The libc API provided by malloc.cc will always be mangled per above. 4 | #ifdef SNMALLOC_RUST_LIBC_API 5 | # include "malloc.cc" 6 | #else 7 | # include "snmalloc/snmalloc.h" 8 | #endif 9 | 10 | #include 11 | 12 | #ifndef SNMALLOC_EXPORT 13 | # define SNMALLOC_EXPORT 14 | #endif 15 | 16 | using namespace snmalloc; 17 | 18 | extern "C" SNMALLOC_EXPORT void* 19 | SNMALLOC_NAME_MANGLE(rust_alloc)(size_t alignment, size_t size) 20 | { 21 | return alloc(aligned_size(alignment, size)); 22 | } 23 | 24 | extern "C" SNMALLOC_EXPORT void* 25 | SNMALLOC_NAME_MANGLE(rust_alloc_zeroed)(size_t alignment, size_t size) 26 | { 27 | return alloc(aligned_size(alignment, size)); 28 | } 29 | 30 | extern "C" SNMALLOC_EXPORT void 31 | SNMALLOC_NAME_MANGLE(rust_dealloc)(void* ptr, size_t alignment, size_t size) 32 | { 33 | dealloc(ptr, aligned_size(alignment, size)); 34 | } 35 | 36 | extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(rust_realloc)( 37 | void* ptr, size_t alignment, size_t old_size, size_t new_size) 38 | { 39 | size_t aligned_old_size = aligned_size(alignment, old_size), 40 | aligned_new_size = aligned_size(alignment, new_size); 41 | if ( 42 | size_to_sizeclass_full(aligned_old_size).raw() == 43 | size_to_sizeclass_full(aligned_new_size).raw()) 44 | return ptr; 45 | void* p = alloc(aligned_new_size); 46 | if (p) 47 | { 48 | memcpy(p, ptr, old_size < new_size ? old_size : new_size); 49 | dealloc(ptr, aligned_old_size); 50 | } 51 | return p; 52 | } 53 | 54 | extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(rust_statistics)( 55 | size_t* current_memory_usage, size_t* peak_memory_usage) 56 | { 57 | *current_memory_usage = Alloc::Config::Backend::get_current_usage(); 58 | *peak_memory_usage = Alloc::Config::Backend::get_peak_usage(); 59 | } 60 | 61 | extern "C" SNMALLOC_EXPORT size_t 62 | SNMALLOC_NAME_MANGLE(rust_usable_size)(const void* ptr) 63 | { 64 | return alloc_size(ptr); 65 | } 66 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/commitrange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "../pal/pal.h" 3 | #include "empty_range.h" 4 | #include "range_helpers.h" 5 | 6 | namespace snmalloc 7 | { 8 | template 9 | struct CommitRange 10 | { 11 | template 12 | class Type : public ContainsParent 13 | { 14 | using ContainsParent::parent; 15 | 16 | public: 17 | static constexpr bool Aligned = ParentRange::Aligned; 18 | 19 | static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; 20 | 21 | using ChunkBounds = typename ParentRange::ChunkBounds; 22 | static_assert( 23 | ChunkBounds::address_space_control == 24 | capptr::dimension::AddressSpaceControl::Full); 25 | 26 | constexpr Type() = default; 27 | 28 | CapPtr alloc_range(size_t size) 29 | { 30 | SNMALLOC_ASSERT_MSG( 31 | (size % PAL::page_size) == 0, 32 | "size ({}) must be a multiple of page size ({})", 33 | size, 34 | PAL::page_size); 35 | auto range = parent.alloc_range(size); 36 | if (range != nullptr) 37 | { 38 | auto result = 39 | PAL::template notify_using(range.unsafe_ptr(), size); 40 | if (!result) 41 | { 42 | // If notify_using fails, we deallocate the range and return 43 | // nullptr. 44 | parent.dealloc_range(range, size); 45 | return CapPtr(nullptr); 46 | } 47 | } 48 | return range; 49 | } 50 | 51 | void dealloc_range(CapPtr base, size_t size) 52 | { 53 | SNMALLOC_ASSERT_MSG( 54 | (size % PAL::page_size) == 0, 55 | "size ({}) must be a multiple of page size ({})", 56 | size, 57 | PAL::page_size); 58 | PAL::notify_not_using(base.unsafe_ptr(), size); 59 | parent.dealloc_range(base, size); 60 | } 61 | }; 62 | }; 63 | } // namespace snmalloc 64 | -------------------------------------------------------------------------------- /src/snmalloc/ds_aal/singleton.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "prevent_fork.h" 4 | #include "snmalloc/stl/atomic.h" 5 | 6 | namespace snmalloc 7 | { 8 | /* 9 | * In some use cases we need to run before any of the C++ runtime has been 10 | * initialised. This singleton class is designed to not depend on the 11 | * runtime. 12 | */ 13 | template 14 | class Singleton 15 | { 16 | enum class State 17 | { 18 | Uninitialised, 19 | Initialising, 20 | Initialised 21 | }; 22 | 23 | inline static stl::Atomic initialised{State::Uninitialised}; 24 | inline static Object obj; 25 | 26 | public: 27 | /** 28 | * If argument is non-null, then it is assigned the value 29 | * true, if this is the first call to get. 30 | * At most one call will be first. 31 | */ 32 | inline SNMALLOC_SLOW_PATH static Object& get(bool* first = nullptr) 33 | { 34 | // If defined should be initially false; 35 | SNMALLOC_ASSERT(first == nullptr || *first == false); 36 | 37 | auto state = initialised.load(stl::memory_order_acquire); 38 | if (SNMALLOC_UNLIKELY(state == State::Uninitialised)) 39 | { 40 | // A unix fork while initialising a singleton can lead to deadlock. 41 | // Protect against this by not allowing a fork while attempting 42 | // initialisation. 43 | PreventFork pf; 44 | snmalloc::UNUSED(pf); 45 | if (initialised.compare_exchange_strong( 46 | state, State::Initialising, stl::memory_order_relaxed)) 47 | { 48 | init(&obj); 49 | initialised.store(State::Initialised, stl::memory_order_release); 50 | if (first != nullptr) 51 | *first = true; 52 | } 53 | } 54 | 55 | while (SNMALLOC_UNLIKELY(state != State::Initialised)) 56 | { 57 | Aal::pause(); 58 | state = initialised.load(stl::memory_order_acquire); 59 | } 60 | 61 | return obj; 62 | } 63 | }; 64 | } // namespace snmalloc 65 | -------------------------------------------------------------------------------- /src/test/xoroshiro.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace xoroshiro 7 | { 8 | namespace detail 9 | { 10 | template 11 | class XorOshiro 12 | { 13 | private: 14 | static constexpr unsigned STATE_BITS = 8 * sizeof(STATE); 15 | static constexpr unsigned RESULT_BITS = 8 * sizeof(RESULT); 16 | 17 | static_assert( 18 | STATE_BITS >= RESULT_BITS, 19 | "STATE must have at least as many bits as RESULT"); 20 | 21 | STATE x; 22 | STATE y; 23 | 24 | static inline STATE rotl(STATE x, STATE k) 25 | { 26 | return (x << k) | (x >> (STATE_BITS - k)); 27 | } 28 | 29 | public: 30 | XorOshiro(STATE x_ = 5489, STATE y_ = 0) : x(x_), y(y_) 31 | { 32 | // If both zero, then this does not work 33 | if (x_ == 0 && y_ == 0) 34 | abort(); 35 | 36 | next(); 37 | } 38 | 39 | void set_state(STATE x_, STATE y_ = 0) 40 | { 41 | // If both zero, then this does not work 42 | if (x_ == 0 && y_ == 0) 43 | abort(); 44 | 45 | x = x_; 46 | y = y_; 47 | next(); 48 | } 49 | 50 | RESULT next() 51 | { 52 | STATE r = x + y; 53 | y ^= x; 54 | x = rotl(x, A) ^ y ^ (y << B); 55 | y = rotl(y, C); 56 | // If both zero, then this does not work 57 | if (x == 0 && y == 0) 58 | abort(); 59 | return r >> (STATE_BITS - RESULT_BITS); 60 | } 61 | }; 62 | } 63 | 64 | using p128r64 = detail::XorOshiro; 65 | using p128r32 = detail::XorOshiro; 66 | using p64r32 = detail::XorOshiro; 67 | using p64r16 = detail::XorOshiro; 68 | using p32r16 = detail::XorOshiro; 69 | using p32r8 = detail::XorOshiro; 70 | using p16r8 = detail::XorOshiro; 71 | } 72 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_netbsd.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifdef __NetBSD__ 4 | # include "pal_bsd_aligned.h" 5 | 6 | # include 7 | # include 8 | 9 | /** 10 | * We skip the pthread cancellation checkpoints by reaching directly 11 | * the following syscalls so we avoid the possible pthread 12 | * allocation initialization timing issues. 13 | * @{ 14 | */ 15 | extern "C" ssize_t _sys_writev(int fd, const struct iovec* iov, int iovcnt); 16 | extern "C" int _sys_fsync(int fd); 17 | 18 | /// @} 19 | 20 | namespace snmalloc 21 | { 22 | /** 23 | * NetBSD-specific platform abstraction layer. 24 | * 25 | * This adds NetBSD-specific aligned allocation to the generic BSD 26 | * implementation. 27 | */ 28 | class PALNetBSD : public PALBSD_Aligned 29 | { 30 | public: 31 | /** 32 | * Bitmap of PalFeatures flags indicating the optional features that this 33 | * PAL supports. 34 | * 35 | * The NetBSD PAL does not currently add any features beyond those of a 36 | * generic BSD with support for arbitrary alignment from `mmap`. This 37 | * field is declared explicitly to remind anyone modifying this class to 38 | * add new features that they should add any required feature flags. 39 | * 40 | * As NetBSD does not have the getentropy call, get_entropy64 will 41 | * currently fallback to C++ libraries std::random_device. 42 | */ 43 | static constexpr uint64_t pal_features = 44 | PALBSD_Aligned::pal_features | Entropy; 45 | 46 | /** 47 | * Temporary solution for NetBSD < 10 48 | * random_device seems unimplemented in clang for this platform 49 | * otherwise using getrandom 50 | */ 51 | static uint64_t get_entropy64() 52 | { 53 | # if defined(SYS_getrandom) 54 | uint64_t result; 55 | if (getrandom(&result, sizeof(result), 0) != sizeof(result)) 56 | error("Failed to get system randomness"); 57 | return result; 58 | # else 59 | return PALPOSIX::dev_urandom(); 60 | # endif 61 | } 62 | }; 63 | } // namespace snmalloc 64 | #endif 65 | -------------------------------------------------------------------------------- /src/snmalloc/aal/aal_riscv.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #if __riscv_xlen == 64 4 | # define SNMALLOC_VA_BITS_64 5 | #elif __riscv_xlen == 32 6 | # define SNMALLOC_VA_BITS_32 7 | #endif 8 | 9 | namespace snmalloc 10 | { 11 | /** 12 | * RISC-V architecture layer, phrased as generically as possible. Specific 13 | * implementations may need to adjust some of these. 14 | */ 15 | class AAL_RISCV 16 | { 17 | public: 18 | static constexpr uint64_t aal_features = IntegerPointers; 19 | 20 | static constexpr size_t smallest_page_size = 0x1000; 21 | 22 | static constexpr AalName aal_name = RISCV; 23 | 24 | static void inline pause() 25 | { 26 | /* 27 | * The "Zihintpause" extension claims to be the right thing to do here, 28 | * and it is expected to be used in analogous places, e.g., Linux's 29 | * cpu_relax(), but... 30 | * 31 | * its specification is somewhat unusual, in that it talks about the rate 32 | * at which a HART's instructions retire rather than the rate at which 33 | * they are dispatched (Intel's PAUSE instruction explicitly promises 34 | * that it "de-pipelines" the spin-wait loop, for example) or anything 35 | * about memory semantics (Intel's PAUSE docs talk about a possible 36 | * memory order violation and pipeline flush upon loop exit). 37 | * 38 | * we don't yet have examples of what implementations have done. 39 | * 40 | * it's not yet understood by C frontends or assembler, meaning we'd have 41 | * to spell it out by hand, as 42 | * __asm__ volatile(".byte 0xF; .byte 0x0; .byte 0x0; .byte 0x1"); 43 | * 44 | * All told, we just leave this function empty for the moment. The good 45 | * news is that, if and when we do add a PAUSE, the instruction is encoded 46 | * by stealing some dead space of the FENCE instruction and so should be 47 | * available everywhere even if it doesn't do anything on a particular 48 | * microarchitecture. 49 | */ 50 | } 51 | }; 52 | 53 | using AAL_Arch = AAL_RISCV; 54 | } 55 | -------------------------------------------------------------------------------- /src/test/func/client_meta/client_meta.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * This test performs a very simple use of the client_meta data feature in 3 | * snmalloc. 4 | */ 5 | 6 | #include "test/setup.h" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace snmalloc 15 | { 16 | // Create an allocator that stores an std::atomic> per allocation. 17 | using Config = snmalloc::StandardConfigClientMeta< 18 | ArrayClientMetaDataProvider>>; 19 | } 20 | 21 | #define SNMALLOC_PROVIDE_OWN_CONFIG 22 | #include 23 | 24 | int main() 25 | { 26 | #if defined(SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION) 27 | // This test does not make sense in GWP-ASan mode. 28 | return 0; 29 | #else 30 | // Allocate a bunch of objects, and store the index into the meta-data. 31 | std::vector ptrs; 32 | for (size_t i = 0; i < 10000; i++) 33 | { 34 | auto p = snmalloc::libc::malloc(1024); 35 | auto& meta = snmalloc::get_client_meta_data(p); 36 | meta = i; 37 | ptrs.push_back(p); 38 | memset(p, (uint8_t)i, 1024); 39 | } 40 | 41 | // Check meta-data contains expected value, and that the memory contains 42 | // the expected pattern. 43 | for (size_t i = 0; i < 10000; i++) 44 | { 45 | auto p = ptrs[i]; 46 | auto& meta = snmalloc::get_client_meta_data(p); 47 | if (meta != i) 48 | { 49 | std::cout << "Failed at index " << i << std::endl; 50 | abort(); 51 | } 52 | for (size_t j = 0; j < 1024; j++) 53 | { 54 | if (reinterpret_cast(p)[j] != (uint8_t)i) 55 | { 56 | std::cout << "Failed at index " << i << " byte " << j << std::endl; 57 | abort(); 58 | } 59 | } 60 | snmalloc::libc::free(p); 61 | } 62 | 63 | // Access in a read-only way meta-data associated with the stack. 64 | // This would fail if it was accessed for write. 65 | auto& meta = snmalloc::get_client_meta_data_const(&ptrs); 66 | std::cout << "meta for stack" << meta << std::endl; 67 | 68 | return 0; 69 | #endif 70 | } 71 | -------------------------------------------------------------------------------- /src/test/func/fixed_region/fixed_region.cc: -------------------------------------------------------------------------------- 1 | #include "test/setup.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #ifdef assert 8 | # undef assert 9 | #endif 10 | #define assert please_use_SNMALLOC_ASSERT 11 | 12 | using namespace snmalloc; 13 | 14 | using CustomGlobals = FixedRangeConfig>; 15 | using FixedAlloc = Allocator; 16 | 17 | int main() 18 | { 19 | setup(); 20 | 21 | // 28 is large enough to produce a nested allocator. 22 | // It is also large enough for the example to run in. 23 | // For 1MiB superslabs, SUPERSLAB_BITS + 4 is not big enough for the example. 24 | auto size = bits::one_at_bit(28); 25 | auto oe_base = DefaultPal::reserve(size); 26 | DefaultPal::notify_using(oe_base, size); 27 | auto oe_end = pointer_offset(oe_base, size); 28 | std::cout << "Allocated region " << oe_base << " - " 29 | << pointer_offset(oe_base, size) << std::endl; 30 | 31 | CustomGlobals::init(nullptr, oe_base, size); 32 | auto a = get_scoped_allocator(); 33 | 34 | size_t object_size = 128; 35 | size_t count = 0; 36 | size_t i = 0; 37 | while (true) 38 | { 39 | auto r1 = a->alloc(object_size); 40 | 41 | count += object_size; 42 | i++; 43 | 44 | // Run until we exhaust the fixed region. 45 | // This should return null. 46 | if (r1 == nullptr) 47 | break; 48 | 49 | if (!snmalloc::is_owned(r1)) 50 | { 51 | a->dealloc(r1); 52 | continue; 53 | } 54 | 55 | if (i == 1024) 56 | { 57 | i = 0; 58 | std::cout << "."; 59 | } 60 | 61 | if (oe_base > r1) 62 | { 63 | std::cout << "Allocated: " << r1 << std::endl; 64 | abort(); 65 | } 66 | if (oe_end < r1) 67 | { 68 | std::cout << "Allocated: " << r1 << std::endl; 69 | abort(); 70 | } 71 | } 72 | 73 | std::cout << "Total allocated: " << count << " out of " << size << std::endl; 74 | std::cout << "Overhead: 1/" << (double)size / (double)(size - count) 75 | << std::endl; 76 | } 77 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/statsrange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "empty_range.h" 4 | #include "range_helpers.h" 5 | #include "snmalloc/stl/atomic.h" 6 | 7 | namespace snmalloc 8 | { 9 | /** 10 | * Used to measure memory usage. 11 | */ 12 | struct StatsRange 13 | { 14 | template> 15 | class Type : public ContainsParent 16 | { 17 | using ContainsParent::parent; 18 | 19 | static inline stl::Atomic current_usage{}; 20 | static inline stl::Atomic peak_usage{}; 21 | 22 | public: 23 | static constexpr bool Aligned = ParentRange::Aligned; 24 | 25 | static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; 26 | 27 | using ChunkBounds = typename ParentRange::ChunkBounds; 28 | 29 | constexpr Type() = default; 30 | 31 | CapPtr alloc_range(size_t size) 32 | { 33 | auto result = parent.alloc_range(size); 34 | if (result != nullptr) 35 | { 36 | auto prev = current_usage.fetch_add(size); 37 | auto curr = peak_usage.load(); 38 | while (curr < prev + size) 39 | { 40 | if (peak_usage.compare_exchange_weak(curr, prev + size)) 41 | break; 42 | } 43 | } 44 | return result; 45 | } 46 | 47 | void dealloc_range(CapPtr base, size_t size) 48 | { 49 | current_usage -= size; 50 | parent.dealloc_range(base, size); 51 | } 52 | 53 | size_t get_current_usage() 54 | { 55 | return current_usage.load(); 56 | } 57 | 58 | size_t get_peak_usage() 59 | { 60 | return peak_usage.load(); 61 | } 62 | }; 63 | }; 64 | 65 | template 66 | class StatsCombiner 67 | { 68 | StatsR1 r1{}; 69 | StatsR2 r2{}; 70 | 71 | public: 72 | size_t get_current_usage() 73 | { 74 | return r1.get_current_usage() + r2.get_current_usage(); 75 | } 76 | 77 | size_t get_peak_usage() 78 | { 79 | return r1.get_peak_usage() + r2.get_peak_usage(); 80 | } 81 | }; 82 | } // namespace snmalloc 83 | -------------------------------------------------------------------------------- /src/test/perf/startup/startup.cc: -------------------------------------------------------------------------------- 1 | #include "test/opt.h" 2 | #include "test/setup.h" 3 | #include "test/usage.h" 4 | #include "test/xoroshiro.h" 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | using namespace snmalloc; 13 | 14 | std::vector counters{}; 15 | 16 | template 17 | class ParallelTest 18 | { 19 | private: 20 | std::atomic flag = false; 21 | std::atomic ready = 0; 22 | uint64_t start; 23 | uint64_t end; 24 | std::atomic complete = 0; 25 | size_t cores; 26 | F f; 27 | 28 | void run(size_t id) 29 | { 30 | auto prev = ready.fetch_add(1); 31 | if (prev + 1 == cores) 32 | { 33 | start = Aal::tick(); 34 | flag = true; 35 | } 36 | while (!flag) 37 | Aal::pause(); 38 | 39 | f(id); 40 | 41 | prev = complete.fetch_add(1); 42 | if (prev + 1 == cores) 43 | { 44 | end = Aal::tick(); 45 | } 46 | } 47 | 48 | public: 49 | ParallelTest(F&& f, size_t cores) : cores(cores), f(std::forward(f)) 50 | { 51 | std::thread* t = new std::thread[cores]; 52 | 53 | for (size_t i = 0; i < cores; i++) 54 | { 55 | t[i] = std::thread(&ParallelTest::run, this, i); 56 | } 57 | // Wait for all the threads. 58 | for (size_t i = 0; i < cores; i++) 59 | { 60 | t[i].join(); 61 | } 62 | 63 | delete[] t; 64 | } 65 | 66 | uint64_t time() 67 | { 68 | return end - start; 69 | } 70 | }; 71 | 72 | int main() 73 | { 74 | auto nthreads = std::thread::hardware_concurrency(); 75 | counters.resize(nthreads); 76 | 77 | ParallelTest test( 78 | [](size_t id) { 79 | auto start = Aal::tick(); 80 | snmalloc::dealloc(snmalloc::alloc(1)); 81 | auto end = Aal::tick(); 82 | counters[id] = end - start; 83 | }, 84 | nthreads); 85 | 86 | std::cout << "Taken: " << test.time() << std::endl; 87 | std::sort(counters.begin(), counters.end()); 88 | uint64_t start = 0; 89 | for (auto counter : counters) 90 | { 91 | std::cout << "Thread time " << counter << " (" << counter - start << ")" 92 | << std::endl; 93 | start = counter; 94 | } 95 | } -------------------------------------------------------------------------------- /src/test/func/fixed_region_alloc/fixed_region_alloc.cc: -------------------------------------------------------------------------------- 1 | #include "test/setup.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #ifdef assert 8 | # undef assert 9 | #endif 10 | #define assert please_use_SNMALLOC_ASSERT 11 | 12 | using namespace snmalloc; 13 | 14 | using CustomGlobals = FixedRangeConfig; 15 | using FixedAlloc = Allocator; 16 | 17 | // This is a variation on the fixed_alloc test 18 | // The only difference is that we use the normal PAL here 19 | // and make sure we actually perform the right commit calls 20 | int main() 21 | { 22 | setup(); 23 | 24 | // 28 is large enough to produce a nested allocator. 25 | // It is also large enough for the example to run in. 26 | // For 1MiB superslabs, SUPERSLAB_BITS + 4 is not big enough for the example. 27 | auto size = bits::one_at_bit(28); 28 | auto oe_base = DefaultPal::reserve(size); 29 | auto oe_end = pointer_offset(oe_base, size); 30 | std::cout << "Allocated region " << oe_base << " - " 31 | << pointer_offset(oe_base, size) << std::endl; 32 | 33 | CustomGlobals::init(nullptr, oe_base, size); 34 | auto a = get_scoped_allocator(); 35 | 36 | size_t object_size = 128; 37 | size_t count = 0; 38 | size_t i = 0; 39 | while (true) 40 | { 41 | auto r1 = a->alloc(object_size); 42 | 43 | count += object_size; 44 | i++; 45 | 46 | // Run until we exhaust the fixed region. 47 | // This should return null. 48 | if (r1 == nullptr) 49 | break; 50 | 51 | if (!snmalloc::is_owned(r1)) 52 | { 53 | a->dealloc(r1); 54 | continue; 55 | } 56 | 57 | if (i == 1024) 58 | { 59 | i = 0; 60 | std::cout << "."; 61 | } 62 | 63 | if (oe_base > r1) 64 | { 65 | std::cout << "Allocated: " << r1 << std::endl; 66 | abort(); 67 | } 68 | if (oe_end < r1) 69 | { 70 | std::cout << "Allocated: " << r1 << std::endl; 71 | abort(); 72 | } 73 | } 74 | 75 | std::cout << "Total allocated: " << count << " out of " << size << std::endl; 76 | std::cout << "Overhead: 1/" << (double)size / (double)(size - count) 77 | << std::endl; 78 | } 79 | -------------------------------------------------------------------------------- /src/snmalloc/stl/gnu/utility.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "snmalloc/ds_core/defines.h" 4 | #include "snmalloc/stl/type_traits.h" 5 | 6 | // This is used by clang to provide better analysis of lifetimes. 7 | #if __has_cpp_attribute(_Clang::__lifetimebound__) 8 | # define SNMALLOC_LIFETIMEBOUND [[_Clang::__lifetimebound__]] 9 | #else 10 | # define SNMALLOC_LIFETIMEBOUND 11 | #endif 12 | 13 | namespace snmalloc 14 | { 15 | namespace stl 16 | { 17 | template 18 | [[nodiscard]] inline constexpr T&& 19 | forward(remove_reference_t& ref) noexcept 20 | { 21 | return static_cast(ref); 22 | } 23 | 24 | template 25 | [[nodiscard]] inline constexpr T&& 26 | forward(SNMALLOC_LIFETIMEBOUND remove_reference_t&& ref) noexcept 27 | { 28 | static_assert( 29 | !is_lvalue_reference_v, "cannot forward an rvalue as an lvalue"); 30 | return static_cast(ref); 31 | } 32 | 33 | template 34 | [[nodiscard]] inline constexpr remove_reference_t&& 35 | move(SNMALLOC_LIFETIMEBOUND T&& ref) noexcept 36 | { 37 | #ifdef __clang__ 38 | using U [[gnu::nodebug]] = remove_reference_t; 39 | #else 40 | using U = remove_reference_t; 41 | #endif 42 | return static_cast(ref); 43 | } 44 | 45 | #pragma GCC diagnostic push 46 | #pragma GCC diagnostic ignored "-Wdeprecated" 47 | // First try instantiation with reference types, then fall back to value 48 | // types. Use int to prioritize reference types. 49 | template 50 | T&& declval_impl(int); 51 | template 52 | T declval_impl(long); 53 | #pragma GCC diagnostic pop 54 | 55 | template 56 | constexpr inline decltype(declval_impl(0)) declval() noexcept 57 | { 58 | static_assert( 59 | !is_same_v, "declval cannot be used in an evaluation context"); 60 | } 61 | 62 | template 63 | struct Pair 64 | { 65 | T1 first; 66 | T2 second; 67 | }; 68 | 69 | template 70 | constexpr SNMALLOC_FAST_PATH A exchange(A& obj, B&& new_value) 71 | { 72 | A old_value = move(obj); 73 | obj = forward(new_value); 74 | return old_value; 75 | } 76 | } // namespace stl 77 | } // namespace snmalloc 78 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_openbsd.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #if defined(__OpenBSD__) && !defined(_KERNEL) 4 | # include "pal_bsd.h" 5 | 6 | # include 7 | 8 | namespace snmalloc 9 | { 10 | /** 11 | * OpenBSD platform abstraction layer. 12 | * 13 | * OpenBSD behaves exactly like a generic BSD platform but this class exists 14 | * as a place to add OpenBSD-specific behaviour later, if required. 15 | */ 16 | class PALOpenBSD : public PALBSD 17 | { 18 | public: 19 | /** 20 | * The features exported by this PAL. 21 | * 22 | * Currently, these are identical to the generic BSD PAL. This field is 23 | * declared explicitly to remind anyone who modifies this class that they 24 | * should add any required features. 25 | */ 26 | static constexpr uint64_t pal_features = 27 | PALBSD::pal_features | WaitOnAddress; 28 | 29 | using WaitingWord = int; 30 | 31 | template 32 | static void wait_on_address(std::atomic& addr, T expected) 33 | { 34 | int backup = errno; 35 | static_assert( 36 | sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord), 37 | "T must be the same size and alignment as WaitingWord"); 38 | while (addr.load(std::memory_order_relaxed) == expected) 39 | { 40 | futex( 41 | (uint32_t*)&addr, 42 | FUTEX_WAIT_PRIVATE, 43 | static_cast(expected), 44 | nullptr, 45 | nullptr); 46 | } 47 | errno = backup; 48 | } 49 | 50 | template 51 | static void notify_one_on_address(std::atomic& addr) 52 | { 53 | static_assert( 54 | sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord), 55 | "T must be the same size and alignment as WaitingWord"); 56 | futex((uint32_t*)&addr, FUTEX_WAKE_PRIVATE, 1, nullptr, nullptr); 57 | } 58 | 59 | template 60 | static void notify_all_on_address(std::atomic& addr) 61 | { 62 | static_assert( 63 | sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord), 64 | "T must be the same size and alignment as WaitingWord"); 65 | futex((uint32_t*)&addr, FUTEX_WAKE_PRIVATE, INT_MAX, nullptr, nullptr); 66 | } 67 | }; 68 | } // namespace snmalloc 69 | #endif 70 | -------------------------------------------------------------------------------- /src/test/perf/singlethread/singlethread.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | using namespace snmalloc; 7 | 8 | template 9 | void test_alloc_dealloc(size_t count, size_t size, bool write) 10 | { 11 | { 12 | MeasureTime m; 13 | m << "Count: " << std::setw(6) << count << ", Size: " << std::setw(6) 14 | << size 15 | << ", ZeroMem: " << std::is_same_v << ", Write: " << write; 16 | 17 | std::unordered_set set; 18 | 19 | // alloc 1.5x objects 20 | for (size_t i = 0; i < ((count * 3) / 2); i++) 21 | { 22 | void* p = snmalloc::alloc(size); 23 | SNMALLOC_CHECK(set.find(p) == set.end()); 24 | 25 | if (write) 26 | *(int*)p = 4; 27 | 28 | set.insert(p); 29 | } 30 | 31 | // free 0.25x of the objects 32 | for (size_t i = 0; i < (count / 4); i++) 33 | { 34 | auto it = set.begin(); 35 | void* p = *it; 36 | set.erase(it); 37 | SNMALLOC_CHECK(set.find(p) == set.end()); 38 | snmalloc::dealloc(p, size); 39 | } 40 | 41 | // alloc 1x objects 42 | for (size_t i = 0; i < count; i++) 43 | { 44 | void* p = snmalloc::alloc(size); 45 | SNMALLOC_CHECK(set.find(p) == set.end()); 46 | 47 | if (write) 48 | *(int*)p = 4; 49 | 50 | set.insert(p); 51 | } 52 | 53 | // free everything 54 | while (!set.empty()) 55 | { 56 | auto it = set.begin(); 57 | snmalloc::dealloc(*it, size); 58 | set.erase(it); 59 | } 60 | } 61 | 62 | snmalloc::debug_check_empty(); 63 | } 64 | 65 | int main(int, char**) 66 | { 67 | setup(); 68 | 69 | for (size_t size = 16; size <= 128; size <<= 1) 70 | { 71 | test_alloc_dealloc(1 << 15, size, false); 72 | test_alloc_dealloc(1 << 15, size, true); 73 | test_alloc_dealloc(1 << 15, size, false); 74 | test_alloc_dealloc(1 << 15, size, true); 75 | } 76 | 77 | for (size_t size = 1 << 12; size <= 1 << 17; size <<= 1) 78 | { 79 | test_alloc_dealloc(1 << 10, size, false); 80 | test_alloc_dealloc(1 << 10, size, true); 81 | test_alloc_dealloc(1 << 10, size, false); 82 | test_alloc_dealloc(1 << 10, size, true); 83 | } 84 | 85 | return 0; 86 | } 87 | -------------------------------------------------------------------------------- /src/test/func/protect_fork/protect_fork.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #ifndef SNMALLOC_PTHREAD_ATFORK_WORKS 5 | int main() 6 | { 7 | std::cout << "Test did not run" << std::endl; 8 | return 0; 9 | } 10 | #else 11 | 12 | # include 13 | # include 14 | 15 | void simulate_allocation() 16 | { 17 | snmalloc::PreventFork pf; 18 | } 19 | 20 | int main() 21 | { 22 | // Counter for the number of threads that are blocking the fork 23 | std::atomic block = false; 24 | // Specifies that the forking thread has observed that all the blocking 25 | // threads are in place. 26 | std::atomic forking = false; 27 | 28 | size_t N = 3; 29 | 30 | snmalloc::message<1024>("Testing PreventFork"); 31 | 32 | snmalloc::message<1024>("Adding alternative calls to pthread_atfork"); 33 | pthread_atfork(simulate_allocation, simulate_allocation, simulate_allocation); 34 | 35 | snmalloc::message<1024>("Initialising PreventFork singleton"); 36 | { 37 | // Cause initialisation of the PreventFork singleton to call pthread_atfork. 38 | snmalloc::PreventFork pf; 39 | } 40 | 41 | snmalloc::message<1024>("Adding alternative calls to pthread_atfork"); 42 | pthread_atfork(simulate_allocation, simulate_allocation, simulate_allocation); 43 | 44 | snmalloc::message<1024>("Creating other threads"); 45 | for (size_t i = 0; i < N; i++) 46 | { 47 | std::thread t([&block, &forking, i]() { 48 | { 49 | snmalloc::PreventFork pf; 50 | snmalloc::message<1024>("Thread {} blocking fork", i); 51 | block++; 52 | std::this_thread::sleep_for(std::chrono::milliseconds(100)); 53 | while (!forking) 54 | std::this_thread::yield(); 55 | std::this_thread::sleep_for(std::chrono::milliseconds(100)); 56 | snmalloc::message<1024>("Thread {} releasing block", i); 57 | block--; 58 | } 59 | }); 60 | 61 | t.detach(); 62 | } 63 | 64 | snmalloc::message<1024>("Waiting for all threads to block fork"); 65 | while (block != N) 66 | std::this_thread::yield(); 67 | 68 | snmalloc::message<1024>("Forking"); 69 | forking = true; 70 | fork(); 71 | 72 | if (block) 73 | { 74 | snmalloc::message<1024>("PreventFork failed"); 75 | return 1; 76 | } 77 | snmalloc::message<1024>("PreventFork passed"); 78 | return 0; 79 | } 80 | 81 | #endif -------------------------------------------------------------------------------- /src/snmalloc/mem/secondary/gwp_asan.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "gwp_asan/guarded_pool_allocator.h" 4 | #include "snmalloc/ds_core/defines.h" 5 | #include "snmalloc/mem/sizeclasstable.h" 6 | #if defined(SNMALLOC_BACKTRACE_HEADER) 7 | # include SNMALLOC_BACKTRACE_HEADER 8 | #endif 9 | 10 | namespace snmalloc 11 | { 12 | class GwpAsanSecondaryAllocator 13 | { 14 | static inline gwp_asan::GuardedPoolAllocator singleton; 15 | static inline size_t max_allocation_size; 16 | 17 | public: 18 | static constexpr inline bool pass_through = false; 19 | 20 | static void initialize() noexcept 21 | { 22 | // for now, we use default options 23 | gwp_asan::options::Options opt; 24 | opt.setDefaults(); 25 | #ifdef SNMALLOC_BACKTRACE_HEADER 26 | opt.Backtrace = [](uintptr_t* buf, size_t length) { 27 | return static_cast( 28 | ::backtrace(reinterpret_cast(buf), static_cast(length))); 29 | }; 30 | #endif 31 | singleton.init(opt); 32 | max_allocation_size = 33 | singleton.getAllocatorState()->maximumAllocationSize(); 34 | } 35 | 36 | // Use thunk to avoid extra computation when allocation decision can be made 37 | // before size and alignment are computed. 38 | template 39 | SNMALLOC_FAST_PATH static void* allocate(SizeAlign&& getter) 40 | { 41 | // TODO: this `shouldSample` is only triggered on snmalloc's slowpath, 42 | // which may reduce the chance of error detection. We may reconsider 43 | // the logic to improve the precision in future commits. 44 | if (SNMALLOC_UNLIKELY(singleton.shouldSample())) 45 | { 46 | auto [size, align] = getter(); 47 | if (size > max_allocation_size) 48 | return nullptr; 49 | return singleton.allocate(size, align); 50 | } 51 | return nullptr; 52 | } 53 | 54 | SNMALLOC_FAST_PATH 55 | static void deallocate(void* pointer) 56 | { 57 | snmalloc_check_client( 58 | mitigations(sanity_checks), 59 | singleton.pointerIsMine(pointer), 60 | "Not allocated by snmalloc or secondary allocator"); 61 | 62 | singleton.deallocate(pointer); 63 | } 64 | 65 | SNMALLOC_FAST_PATH 66 | static size_t alloc_size(const void* pointer) 67 | { 68 | return singleton.getSize(pointer); 69 | } 70 | }; 71 | } // namespace snmalloc 72 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/authmap.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "commonconfig.h" 4 | #include "pagemapregisterrange.h" 5 | 6 | namespace snmalloc 7 | { 8 | /** 9 | * A dummy authmap that simply passes pointers through. For use on 10 | * non-StrictProvenance architectures. 11 | */ 12 | struct DummyAuthmap 13 | { 14 | static SNMALLOC_FAST_PATH void init() {} 15 | 16 | static SNMALLOC_FAST_PATH bool register_range(capptr::Arena, size_t) 17 | { 18 | return true; 19 | } 20 | 21 | template 22 | static SNMALLOC_FAST_PATH capptr::Arena amplify(capptr::Alloc c) 23 | { 24 | return capptr::Arena::unsafe_from(c.unsafe_ptr()); 25 | } 26 | }; 27 | 28 | /** 29 | * Wrap a concrete Pagemap to store Arena pointers, and use these when 30 | * amplifying a pointer. 31 | */ 32 | template 33 | struct BasicAuthmap 34 | { 35 | static_assert( 36 | stl::is_same_v, typename ConcreteMap::EntryType>, 37 | "BasicAuthmap's ConcreteMap must have capptr::Arena element type!"); 38 | 39 | private: 40 | SNMALLOC_REQUIRE_CONSTINIT 41 | static inline ConcreteMap concreteAuthmap; 42 | 43 | public: 44 | static SNMALLOC_FAST_PATH void init() 45 | { 46 | concreteAuthmap.template init(); 47 | } 48 | 49 | static SNMALLOC_FAST_PATH bool 50 | register_range(capptr::Arena base, size_t size) 51 | { 52 | concreteAuthmap.register_range(address_cast(base), size); 53 | 54 | address_t base_addr = address_cast(base); 55 | for (address_t a = base_addr; a < base_addr + size; 56 | a += ConcreteMap::GRANULARITY) 57 | { 58 | concreteAuthmap.set(a, base); 59 | } 60 | return true; 61 | } 62 | 63 | template 64 | static SNMALLOC_FAST_PATH capptr::Arena amplify(capptr::Alloc c) 65 | { 66 | return Aal::capptr_rebound( 67 | concreteAuthmap.template get(address_cast(c)), 68 | c); 69 | } 70 | }; 71 | 72 | /** 73 | * Pick between the two above implementations based on StrictProvenance 74 | */ 75 | template 76 | using DefaultAuthmap = stl::conditional_t< 77 | aal_supports, 78 | BasicAuthmap, 79 | DummyAuthmap>; 80 | 81 | } // namespace snmalloc 82 | -------------------------------------------------------------------------------- /src/test/func/thread_alloc_external/thread_alloc_external.cc: -------------------------------------------------------------------------------- 1 | #ifdef SNMALLOC_USE_PTHREAD_DESTRUCTORS 2 | # undef SNMALLOC_USE_PTHREAD_DESTRUCTORS 3 | #endif 4 | #ifdef SNMALLOC_USE_CXX11_THREAD_ATEXIT_DIRECT 5 | # undef SNMALLOC_USE_CXX11_THREAD_ATEXIT_DIRECT 6 | #endif 7 | #ifdef SNMALLOC_USE_CXX11_DESTRUCTORS 8 | # undef SNMALLOC_USE_CXX11_DESTRUCTORS 9 | #endif 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | // Specify using own 16 | #define SNMALLOC_EXTERNAL_THREAD_ALLOC 17 | 18 | #include 19 | 20 | namespace snmalloc 21 | { 22 | using Config = snmalloc::StandardConfigClientMeta; 23 | using Alloc = snmalloc::Allocator; 24 | } 25 | 26 | using namespace snmalloc; 27 | 28 | class ThreadAllocExternal 29 | { 30 | public: 31 | static Alloc*& get_inner() 32 | { 33 | static thread_local Alloc* alloc; 34 | return alloc; 35 | } 36 | 37 | static Alloc& get() 38 | { 39 | return *get_inner(); 40 | } 41 | }; 42 | 43 | #include 44 | 45 | void allocator_thread_init(void) 46 | { 47 | void* aptr; 48 | { 49 | // Create bootstrap allocator 50 | auto a = snmalloc::ScopedAllocator(); 51 | // Create storage for the thread-local allocator 52 | aptr = a->alloc(sizeof(snmalloc::Alloc)); 53 | } 54 | // Initialize the thread-local allocator 55 | ThreadAllocExternal::get_inner() = new (aptr) snmalloc::Alloc(); 56 | } 57 | 58 | void allocator_thread_cleanup(void) 59 | { 60 | // Teardown the thread-local allocator 61 | ThreadAlloc::teardown(); 62 | // Need a bootstrap allocator to deallocate the thread-local allocator 63 | auto a = snmalloc::ScopedAllocator(); 64 | // Deallocate the storage for the thread local allocator 65 | a->dealloc(ThreadAllocExternal::get_inner()); 66 | } 67 | 68 | int main() 69 | { 70 | setup(); 71 | allocator_thread_init(); 72 | 73 | for (size_t i = 0; i < 1000; i++) 74 | { 75 | auto r1 = snmalloc::alloc(i); 76 | 77 | snmalloc::dealloc(r1); 78 | } 79 | 80 | snmalloc::debug_teardown(); 81 | 82 | // This checks that the scoped allocator does not call 83 | // register clean up, as this configuration will fault 84 | // if that occurs. 85 | auto a2 = ScopedAllocator(); 86 | for (size_t i = 0; i < 1000; i++) 87 | { 88 | auto r1 = a2->alloc(i); 89 | 90 | a2->dealloc(r1); 91 | } 92 | 93 | allocator_thread_cleanup(); 94 | } 95 | -------------------------------------------------------------------------------- /.github/workflows/benchmark.yml: -------------------------------------------------------------------------------- 1 | name: snmalloc Benchmarking CI 2 | 3 | # The following should ensure that the workflow only runs a single set of actions 4 | # for each PR. But it will not apply this to pushes to the main branch. 5 | concurrency: 6 | group: benchmarking${{ github.ref }} 7 | cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} 8 | 9 | # Controls when the workflow will run 10 | on: 11 | schedule: 12 | - cron: "0 0 * * 1" # Runs every Monday at midnight UTC 13 | # Triggers the workflow on push to main or any branch starting bench. 14 | # The bench/** branches can be used for test before merge any PR that 15 | # might regress performance. 16 | push: 17 | branches: 18 | - 'main' 19 | - 'bench/**' 20 | 21 | # Allows you to run this workflow manually from the Actions tab 22 | workflow_dispatch: 23 | 24 | jobs: 25 | benchmark: 26 | runs-on: [self-hosted, 1ES.Pool=snmalloc-perf] 27 | 28 | steps: 29 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 30 | - name: Checkout 31 | uses: actions/checkout@v3 32 | 33 | # Setup docker buildx 34 | - name: Setup Docker Buildx 35 | uses: docker/setup-buildx-action@v2 36 | 37 | - name: Build and run benchmarks in Docker 38 | id: docker_build 39 | uses: docker/build-push-action@v4 40 | with: 41 | context: . 42 | file: benchmark/Dockerfile 43 | push: false 44 | load: true 45 | tags: snmalloc-bench 46 | build-args: | 47 | benchs=allt 48 | repeats=5 49 | cache-from: type=gha 50 | cache-to: type=gha,mode=max 51 | 52 | # Extracts the benchmark results from the Docker container 53 | - name: Extract Benchmark Results 54 | run: | 55 | docker cp `docker run -d ${{ steps.docker_build.outputs.imageid }}`:/bencher.dev.sn.json . 56 | 57 | # Uploads the benchmark results as an artifact 58 | - name: Upload Benchmark Results 59 | uses: actions/upload-artifact@v4 60 | with: 61 | name: benchmark-results 62 | path: bencher.dev.sn.json 63 | 64 | # Upload to graphing service 65 | - uses: bencherdev/bencher@main 66 | - name: Upload benchmark results to Bencher 67 | run: | 68 | bencher run \ 69 | --project snmalloc \ 70 | --token '${{ secrets.BENCHER_DEV_API_TOKEN }}' \ 71 | --branch main \ 72 | --adapter json \ 73 | --err \ 74 | --file bencher.dev.sn.json -------------------------------------------------------------------------------- /BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake") 2 | 3 | filegroup( 4 | name = "srcs", 5 | srcs = glob( 6 | [ 7 | "src/snmalloc/**/*", 8 | "src/test/*.h", 9 | "CMakeLists.txt", 10 | ], 11 | ), 12 | visibility = ["//visibility:private"], 13 | ) 14 | 15 | config_setting( 16 | name = "release_with_debug", 17 | values = { 18 | "compilation_mode": "fastbuild", 19 | }, 20 | ) 21 | 22 | config_setting( 23 | name = "release", 24 | values = { 25 | "compilation_mode": "opt", 26 | }, 27 | ) 28 | 29 | config_setting( 30 | name = "debug", 31 | values = { 32 | "compilation_mode": "dbg", 33 | }, 34 | ) 35 | 36 | CMAKE_FLAGS = { 37 | "CMAKE_INTERPROCEDURAL_OPTIMIZATION": "TRUE", 38 | "SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE": "ON", 39 | "SNMALLOC_USE_SELF_VENDORED_STL": "OFF", 40 | "SNMALLOC_IPO": "ON", 41 | "USE_SNMALLOC_STATS": "ON", 42 | } | select({ 43 | ":release_with_debug": {"CMAKE_BUILD_TYPE": "RelWithDebInfo"}, 44 | ":release": {"CMAKE_BUILD_TYPE": "Release"}, 45 | ":debug": {"CMAKE_BUILD_TYPE": "Debug"}, 46 | "//conditions:default": {"CMAKE_BUILD_TYPE": "Release"}, 47 | }) 48 | 49 | cmake( 50 | name = "snmalloc", 51 | cache_entries = CMAKE_FLAGS, 52 | generate_args = ["-G Ninja"], 53 | lib_source = ":srcs", 54 | out_shared_libs = select({ 55 | "@bazel_tools//src/conditions:darwin": [ 56 | "libsnmallocshim-checks-memcpy-only.dylib", 57 | "libsnmallocshim-checks.dylib", 58 | "libsnmallocshim.dylib", 59 | ], 60 | "//conditions:default": [], 61 | }), 62 | out_static_libs = [ 63 | "libsnmallocshim-static.a", 64 | "libsnmalloc-new-override.a", 65 | ], 66 | postfix_script = "ninja", 67 | visibility = ["//visibility:public"], 68 | ) 69 | 70 | cmake( 71 | name = "snmalloc-rs", 72 | cache_entries = CMAKE_FLAGS | { 73 | "SNMALLOC_RUST_SUPPORT": "ON", 74 | }, 75 | generate_args = ["-G Ninja"], 76 | lib_source = ":srcs", 77 | out_shared_libs = select({ 78 | "@bazel_tools//src/conditions:darwin": [ 79 | "libsnmallocshim-checks-memcpy-only.dylib", 80 | "libsnmallocshim-checks.dylib", 81 | "libsnmallocshim.dylib", 82 | ], 83 | "//conditions:default": [], 84 | }), 85 | out_static_libs = [ 86 | "libsnmallocshim-static.a", 87 | "libsnmalloc-new-override.a", 88 | ], 89 | postfix_script = "ninja", 90 | visibility = ["//visibility:public"], 91 | ) 92 | -------------------------------------------------------------------------------- /src/snmalloc/global/scopedalloc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /** 4 | * This header requires that Alloc has been defined. 5 | */ 6 | 7 | namespace snmalloc 8 | { 9 | /** 10 | * RAII wrapper around an `Alloc`. This class gets an allocator from the 11 | * global pool and wraps it so that `Alloc` methods can be called 12 | * directly via the `->` operator on this class. When this object is 13 | * destroyed, it returns the allocator to the global pool. 14 | * 15 | * This does not depend on thread-local storage working, so can be used for 16 | * bootstrapping. 17 | */ 18 | template 19 | struct ScopedAllocator 20 | { 21 | /** 22 | * The allocator that this wrapper will use. 23 | */ 24 | SAlloc* alloc; 25 | 26 | /** 27 | * Constructor. Claims an allocator from the global pool 28 | */ 29 | ScopedAllocator() 30 | { 31 | alloc = AllocPool::acquire(); 32 | }; 33 | 34 | /** 35 | * Copying is not supported, it could easily lead to accidental sharing of 36 | * allocators. 37 | */ 38 | ScopedAllocator(const ScopedAllocator&) = delete; 39 | 40 | /** 41 | * Moving is not supported, though it would be easy to add if there's a use 42 | * case for it. 43 | */ 44 | ScopedAllocator(ScopedAllocator&&) = delete; 45 | 46 | /** 47 | * Copying is not supported, it could easily lead to accidental sharing of 48 | * allocators. 49 | */ 50 | ScopedAllocator& operator=(const ScopedAllocator&) = delete; 51 | 52 | /** 53 | * Moving is not supported, though it would be easy to add if there's a use 54 | * case for it. 55 | */ 56 | ScopedAllocator& operator=(ScopedAllocator&&) = delete; 57 | 58 | /** 59 | * Destructor. Returns the allocator to the pool. 60 | */ 61 | ~ScopedAllocator() 62 | { 63 | if (alloc != nullptr) 64 | { 65 | alloc->flush(); 66 | AllocPool::release(alloc); 67 | alloc = nullptr; 68 | } 69 | } 70 | 71 | /** 72 | * Arrow operator, allows methods exposed by `Alloc` to be called on the 73 | * wrapper. 74 | */ 75 | SAlloc* operator->() 76 | { 77 | return alloc; 78 | } 79 | }; 80 | 81 | /** 82 | * Returns a new scoped allocator. When the `ScopedAllocator` goes out of 83 | * scope, the underlying `Alloc` will be returned to the pool. 84 | */ 85 | template 86 | inline ScopedAllocator get_scoped_allocator() 87 | { 88 | return {}; 89 | } 90 | } // namespace snmalloc 91 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/defaultpagemapentry.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../mem/mem.h" 4 | 5 | namespace snmalloc 6 | { 7 | /** 8 | * Example of type stored in the pagemap. 9 | * The following class could be replaced by: 10 | * 11 | * ``` 12 | * using DefaultPagemapEntry = FrontendMetaEntry; 13 | * ``` 14 | * 15 | * The full form here provides an example of how to extend the pagemap 16 | * entries. It also guarantees that the front end never directly 17 | * constructs meta entries, it only ever reads them or modifies them in 18 | * place. 19 | */ 20 | template 21 | class DefaultPagemapEntryT : public FrontendMetaEntry 22 | { 23 | /** 24 | * The private initialising constructor is usable only by this back end. 25 | */ 26 | template< 27 | SNMALLOC_CONCEPT(IsPAL) A1, 28 | typename A2, 29 | typename A3, 30 | typename A4, 31 | typename A5> 32 | friend class BackendAllocator; 33 | 34 | /** 35 | * The private default constructor is usable only by the pagemap. 36 | */ 37 | template 38 | friend class FlatPagemap; 39 | 40 | /** 41 | * The only constructor that creates newly initialised meta entries. 42 | * This is callable only by the back end. The front end may copy, 43 | * query, and update these entries, but it may not create them 44 | * directly. This contract allows the back end to store any arbitrary 45 | * metadata in meta entries when they are first constructed. 46 | */ 47 | SNMALLOC_FAST_PATH 48 | DefaultPagemapEntryT(SlabMetadata* meta, uintptr_t ras) 49 | : FrontendMetaEntry(meta, ras) 50 | {} 51 | 52 | /** 53 | * Copy assignment is used only by the pagemap. 54 | */ 55 | DefaultPagemapEntryT& operator=(const DefaultPagemapEntryT& other) 56 | { 57 | FrontendMetaEntry::operator=(other); 58 | return *this; 59 | } 60 | 61 | /** 62 | * Default constructor. This must be callable from the pagemap. 63 | */ 64 | SNMALLOC_FAST_PATH DefaultPagemapEntryT() = default; 65 | }; 66 | 67 | template 68 | class DefaultSlabMetadata : public FrontendSlabMetadata< 69 | DefaultSlabMetadata, 70 | ClientMetaDataProvider> 71 | {}; 72 | 73 | template 74 | using DefaultPagemapEntry = 75 | DefaultPagemapEntryT>; 76 | 77 | } // namespace snmalloc 78 | -------------------------------------------------------------------------------- /src/snmalloc/ds/mpmcstack.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../ds_core/ds_core.h" 4 | #include "aba.h" 5 | #include "allocconfig.h" 6 | 7 | namespace snmalloc 8 | { 9 | template 10 | class MPMCStack 11 | { 12 | using ABAT = ABA; 13 | 14 | private: 15 | alignas(CACHELINE_SIZE) ABAT stack; 16 | 17 | #ifdef SNMALLOC_THREAD_SANITIZER_ENABLED 18 | __attribute__((no_sanitize("thread"))) static T* 19 | racy_read(stl::Atomic& ptr) 20 | { 21 | // reinterpret_cast is required as TSAN still instruments 22 | // stl::Atomic operations, even if you disable TSAN on 23 | // the function. 24 | return *reinterpret_cast(&ptr); 25 | } 26 | #else 27 | static T* racy_read(stl::Atomic& ptr) 28 | { 29 | return ptr.load(stl::memory_order_relaxed); 30 | } 31 | #endif 32 | 33 | public: 34 | constexpr MPMCStack() = default; 35 | 36 | void push(T* item) 37 | { 38 | static_assert( 39 | stl::is_same_v>, 40 | "T->next must be an stl::Atomic"); 41 | 42 | return push(item, item); 43 | } 44 | 45 | void push(T* first, T* last) 46 | { 47 | // Pushes an item on the stack. 48 | auto cmp = stack.read(); 49 | 50 | do 51 | { 52 | auto top = cmp.ptr(); 53 | last->next.store(top, stl::memory_order_release); 54 | } while (!cmp.store_conditional(first)); 55 | } 56 | 57 | T* pop() 58 | { 59 | // Returns the next item. If the returned value is decommitted, it is 60 | // possible for the read of top->next to segfault. 61 | auto cmp = stack.read(); 62 | T* top; 63 | T* next; 64 | 65 | do 66 | { 67 | top = cmp.ptr(); 68 | 69 | if (top == nullptr) 70 | break; 71 | 72 | // The following read can race with non-atomic accesses 73 | // this is undefined behaviour. There is no way to use 74 | // CAS sensibly that conforms to the standard with optimistic 75 | // concurrency. 76 | next = racy_read(top->next); 77 | } while (!cmp.store_conditional(next)); 78 | 79 | return top; 80 | } 81 | 82 | T* pop_all() 83 | { 84 | // Returns all items as a linked list, leaving an empty stack. 85 | auto cmp = stack.read(); 86 | T* top; 87 | 88 | do 89 | { 90 | top = cmp.ptr(); 91 | 92 | if (top == nullptr) 93 | break; 94 | } while (!cmp.store_conditional(nullptr)); 95 | 96 | return top; 97 | } 98 | }; 99 | } // namespace snmalloc 100 | -------------------------------------------------------------------------------- /src/test/func/memory_usage/memory_usage.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Memory usage test 3 | * Query memory usage repeatedly 4 | */ 5 | #include 6 | #include 7 | #include 8 | 9 | #define SNMALLOC_NAME_MANGLE(a) our_##a 10 | #include "../../../snmalloc/override/malloc-extensions.cc" 11 | #include "../../../snmalloc/override/malloc.cc" 12 | 13 | using namespace snmalloc; 14 | 15 | bool print_memory_usage() 16 | { 17 | static malloc_info_v1 last_memory_usage; 18 | malloc_info_v1 next_memory_usage; 19 | 20 | get_malloc_info_v1(&next_memory_usage); 21 | 22 | if ( 23 | (next_memory_usage.current_memory_usage != 24 | last_memory_usage.current_memory_usage) || 25 | (next_memory_usage.peak_memory_usage != 26 | last_memory_usage.peak_memory_usage)) 27 | { 28 | std::cout << "Memory Usages Changed to (" 29 | << next_memory_usage.current_memory_usage << ", " 30 | << next_memory_usage.peak_memory_usage << ")" << std::endl; 31 | last_memory_usage = next_memory_usage; 32 | return true; 33 | } 34 | return false; 35 | } 36 | 37 | std::vector allocs{}; 38 | 39 | /** 40 | * Add allocs until the statistics have changed n times. 41 | */ 42 | void add_n_allocs(size_t n) 43 | { 44 | while (true) 45 | { 46 | auto p = our_malloc(1024); 47 | allocs.push_back(p); 48 | if (print_memory_usage()) 49 | { 50 | n--; 51 | if (n == 0) 52 | break; 53 | } 54 | } 55 | } 56 | 57 | /** 58 | * Remove allocs until the statistics have changed n times. 59 | */ 60 | void remove_n_allocs(size_t n) 61 | { 62 | while (true) 63 | { 64 | if (allocs.empty()) 65 | return; 66 | auto p = allocs.back(); 67 | our_free(p); 68 | allocs.pop_back(); 69 | if (print_memory_usage()) 70 | { 71 | n--; 72 | if (n == 0) 73 | break; 74 | } 75 | } 76 | } 77 | 78 | int main(int argc, char** argv) 79 | { 80 | UNUSED(argc); 81 | UNUSED(argv); 82 | setup(); 83 | 84 | add_n_allocs(5); 85 | std::cout << "Init complete!" << std::endl; 86 | 87 | for (int i = 0; i < 10; i++) 88 | { 89 | remove_n_allocs(1); 90 | std::cout << "Phase " << i << " remove complete!" << std::endl; 91 | add_n_allocs(2); 92 | std::cout << "Phase " << i << " add complete!" << std::endl; 93 | } 94 | 95 | for (int i = 0; i < 10; i++) 96 | { 97 | remove_n_allocs(2); 98 | std::cout << "Phase " << i << " remove complete!" << std::endl; 99 | add_n_allocs(1); 100 | std::cout << "Phase " << i << " add complete!" << std::endl; 101 | } 102 | 103 | remove_n_allocs(3); 104 | std::cout << "Teardown complete!" << std::endl; 105 | } 106 | -------------------------------------------------------------------------------- /src/snmalloc/ds_core/concept.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "snmalloc/stl/type_traits.h" 4 | 5 | /** 6 | * C++20 concepts are referenced as if they were types in declarations within 7 | * template parameters (e.g. "template ..."). That is, they 8 | * take the place of the "typename"/"class" keyword on template parameters. 9 | * If the compiler understands concepts, this macro expands as its argument; 10 | * otherwise, it expands to the keyword "typename", so snmalloc templates that 11 | * use concept-qualified parameters should use this to remain compatible across 12 | * C++ versions: "template" 13 | */ 14 | #ifdef __cpp_concepts 15 | # define SNMALLOC_CONCEPT(c) c 16 | #else 17 | # define SNMALLOC_CONCEPT(c) typename 18 | #endif 19 | 20 | #ifdef __cpp_concepts 21 | namespace snmalloc 22 | { 23 | template 24 | concept ConceptSame = stl::is_same_v; 25 | 26 | /** 27 | * Equivalence mod stl::remove_reference 28 | */ 29 | template 30 | concept ConceptSameModRef = 31 | ConceptSame, stl::remove_reference_t>; 32 | 33 | /** 34 | * Some of the types in snmalloc are circular in their definition and use 35 | * templating as a lazy language to carefully tie knots and only pull on the 36 | * whole mess once it's assembled. Unfortunately, concepts amount to eagerly 37 | * demanding the result of the computation. If concepts come into play during 38 | * the circular definition, they may see an incomplete type and so fail (with 39 | * "incomplete type ... used in type trait expression" or similar). However, 40 | * it turns out that SFINAE gives us a way to detect whether a template 41 | * parameter refers to an incomplete type, and short circuit evaluation means 42 | * we can bail on concept checking if we find ourselves in this situation. 43 | * 44 | * See https://devblogs.microsoft.com/oldnewthing/20190710-00/?p=102678 45 | * 46 | * Unfortunately, C++20 concepts are not first-order things and, in 47 | * particular, cannot themselves be template parameters. So while we would 48 | * love to write a generic Lazy combinator, 49 | * 50 | * template concept C, typename T> 51 | * concept Lazy = !is_type_complete_v || C(); 52 | * 53 | * this will instead have to be inlined at every definition (and referred to 54 | * explicitly at call sites) until C++23 or later. 55 | */ 56 | template 57 | constexpr bool is_type_complete_v{false}; 58 | 59 | template 60 | constexpr bool 61 | is_type_complete_v)>>{ 62 | false}; 63 | } // namespace snmalloc 64 | #endif 65 | -------------------------------------------------------------------------------- /security.md: -------------------------------------------------------------------------------- 1 | ## Security 2 | 3 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 4 | 5 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets Microsoft's [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. 6 | 7 | ## Reporting Security Issues 8 | 9 | **Please do not report security vulnerabilities through public GitHub issues.** 10 | 11 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). 12 | 13 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). 14 | 15 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 16 | 17 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 18 | 19 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 20 | * Full paths of source file(s) related to the manifestation of the issue 21 | * The location of the affected source code (tag/branch/commit or direct URL) 22 | * Any special configuration required to reproduce the issue 23 | * Step-by-step instructions to reproduce the issue 24 | * Proof-of-concept or exploit code (if possible) 25 | * Impact of the issue, including how an attacker might exploit the issue 26 | 27 | This information will help us triage your report more quickly. 28 | 29 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. 30 | 31 | ## Preferred Languages 32 | 33 | We prefer all communications to be in English. 34 | 35 | ## Policy 36 | 37 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). 38 | -------------------------------------------------------------------------------- /src/test/func/multi_setspecific/multi_setspecific.cc: -------------------------------------------------------------------------------- 1 | #ifndef __has_feature 2 | # define __has_feature(x) 0 3 | #endif 4 | 5 | // These test partially override the libc malloc/free functions to test 6 | // interesting corner cases. This breaks the sanitizers as they will be 7 | // partially overridden. So we disable the tests if any of the sanitizers are 8 | // enabled. 9 | #if defined(__linux__) && !__has_feature(address_sanitizer) && \ 10 | !defined(__SANITIZE_ADDRESS__) && !defined(__SANITIZE_THREAD__) && \ 11 | !defined(SNMALLOC_THREAD_SANITIZER_ENABLED) 12 | # define RUN_TEST 13 | #endif 14 | 15 | #ifdef RUN_TEST 16 | # include 17 | # include 18 | # include 19 | # include 20 | 21 | // A key in the second "second level" block of the pthread key table. 22 | // First second level block is statically allocated. 23 | // This is be 33. 24 | pthread_key_t key; 25 | 26 | void thread_setspecific() 27 | { 28 | // If the following line is uncommented then the test will pass. 29 | // free(calloc(1, 1)); 30 | pthread_setspecific(key, (void*)1); 31 | } 32 | 33 | // We only selectively override these functions. Otherwise, malloc may be called 34 | // before atexit triggers the first initialization attempt. 35 | 36 | extern "C" void* calloc(size_t num, size_t size) 37 | { 38 | snmalloc::message("calloc({}, {})", num, size); 39 | return snmalloc::libc::calloc(num, size); 40 | } 41 | 42 | extern "C" void free(void* p) 43 | { 44 | snmalloc::message("free({})", p); 45 | // Just leak it 46 | if (snmalloc::is_owned(p)) 47 | return snmalloc::libc::free(p); 48 | } 49 | 50 | void callback(void*) 51 | { 52 | snmalloc::message("callback"); 53 | } 54 | 55 | int main() 56 | { 57 | // The first 32 keys are statically allocated, so we need to create 33 keys 58 | // to create a key for which pthread_setspecific will call the calloc. 59 | for (size_t i = 0; i < 33; i++) 60 | { 61 | pthread_key_create(&key, callback); 62 | } 63 | 64 | // The first calloc occurs here, after the first [0, 32] keys have been 65 | // created thus snmalloc will choose the key 33, `key` contains the key `32` 66 | // and snmalloc `33`. Both of these keys are not in the statically allocated 67 | // part of the pthread key space. 68 | std::thread(thread_setspecific).join(); 69 | 70 | // There should be a single allocator that can be extracted. 71 | if (snmalloc::AllocPool::extract() == nullptr) 72 | { 73 | // The thread has not torn down its allocator. 74 | snmalloc::report_fatal_error( 75 | "Teardown of thread allocator has not occurred."); 76 | return 1; 77 | } 78 | 79 | return 0; 80 | } 81 | #else 82 | int main() 83 | { 84 | // This test is not run, but it is used to check that the code compiles. 85 | return 0; 86 | } 87 | #endif 88 | -------------------------------------------------------------------------------- /src/snmalloc/backend_helpers/staticconditionalrange.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "../pal/pal.h" 3 | #include "empty_range.h" 4 | #include "range_helpers.h" 5 | 6 | namespace snmalloc 7 | { 8 | template 9 | struct StaticConditionalRange 10 | { 11 | // This is a range that can bypass the OptionalRange if it is disabled. 12 | // Disabling is global, and not local. 13 | // This is used to allow disabling thread local buddy allocators when the 14 | // initial fixed size heap is small. 15 | // 16 | // The range builds a more complex parent 17 | // Pipe 18 | // and uses the ancestor functions to bypass the OptionalRange if the flag 19 | // has been set. 20 | template 21 | class Type : public ContainsParent> 22 | { 23 | // This contains connects the optional range to the parent range. 24 | using ActualParentRange = Pipe; 25 | 26 | using ContainsParent::parent; 27 | 28 | // Global flag specifying if the optional range should be disabled. 29 | static inline bool disable_range_{false}; 30 | 31 | public: 32 | // Both parent and grandparent must be aligned for this range to be 33 | // aligned. 34 | static constexpr bool Aligned = 35 | ActualParentRange::Aligned && ParentRange::Aligned; 36 | 37 | // Both parent and grandparent must be aligned for this range to be 38 | // concurrency safe. 39 | static constexpr bool ConcurrencySafe = 40 | ActualParentRange::ConcurrencySafe && ParentRange::ConcurrencySafe; 41 | 42 | using ChunkBounds = typename ActualParentRange::ChunkBounds; 43 | 44 | static_assert( 45 | stl::is_same_v, 46 | "Grandparent and optional parent range chunk bounds must be equal"); 47 | 48 | constexpr Type() = default; 49 | 50 | CapPtr alloc_range(size_t size) 51 | { 52 | if (disable_range_) 53 | { 54 | // Use ancestor to bypass the optional range. 55 | return this->template ancestor()->alloc_range(size); 56 | } 57 | 58 | return parent.alloc_range(size); 59 | } 60 | 61 | void dealloc_range(CapPtr base, size_t size) 62 | { 63 | if (disable_range_) 64 | { 65 | // Use ancestor to bypass the optional range. 66 | this->template ancestor()->dealloc_range(base, size); 67 | return; 68 | } 69 | parent.dealloc_range(base, size); 70 | } 71 | 72 | static void disable_range() 73 | { 74 | disable_range_ = true; 75 | } 76 | }; 77 | }; 78 | } // namespace snmalloc 79 | -------------------------------------------------------------------------------- /src/test/func/statistics/stats.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | template 6 | void debug_check_empty_1() 7 | { 8 | std::cout << "debug_check_empty_1 " << size << std::endl; 9 | bool result; 10 | 11 | auto r = snmalloc::alloc(size); 12 | 13 | snmalloc::debug_check_empty(&result); 14 | if (result != false) 15 | { 16 | std::cout << "debug_check_empty failed to detect leaked memory:" << size 17 | << std::endl; 18 | abort(); 19 | } 20 | 21 | snmalloc::dealloc(r); 22 | 23 | snmalloc::debug_check_empty(&result); 24 | if (result != true) 25 | { 26 | std::cout << "debug_check_empty failed to say empty:" << size << std::endl; 27 | abort(); 28 | } 29 | 30 | r = snmalloc::alloc(size); 31 | 32 | snmalloc::debug_check_empty(&result); 33 | if (result != false) 34 | { 35 | std::cout << "debug_check_empty failed to detect leaked memory:" << size 36 | << std::endl; 37 | abort(); 38 | } 39 | 40 | snmalloc::dealloc(r); 41 | 42 | snmalloc::debug_check_empty(&result); 43 | if (result != true) 44 | { 45 | std::cout << "debug_check_empty failed to say empty:" << size << std::endl; 46 | abort(); 47 | } 48 | } 49 | 50 | template 51 | void debug_check_empty_2() 52 | { 53 | std::cout << "debug_check_empty_2 " << size << std::endl; 54 | bool result; 55 | std::vector allocs; 56 | // 1GB of allocations 57 | size_t count = snmalloc::bits::min(2048, 1024 * 1024 * 1024 / size); 58 | 59 | for (size_t i = 0; i < count; i++) 60 | { 61 | if (i % (count / 16) == 0) 62 | { 63 | std::cout << "." << std::flush; 64 | } 65 | auto r = snmalloc::alloc(size); 66 | allocs.push_back(r); 67 | snmalloc::debug_check_empty(&result); 68 | if (result != false) 69 | { 70 | std::cout << "False empty after " << i << " allocations of " << size 71 | << std::endl; 72 | abort(); 73 | } 74 | } 75 | std::cout << std::endl; 76 | 77 | for (size_t i = 0; i < count; i++) 78 | { 79 | if (i % (count / 16) == 0) 80 | { 81 | std::cout << "." << std::flush; 82 | } 83 | snmalloc::debug_check_empty(&result); 84 | if (result != false) 85 | { 86 | std::cout << "False empty after " << i << " deallocations of " << size 87 | << std::endl; 88 | abort(); 89 | } 90 | snmalloc::dealloc(allocs[i]); 91 | } 92 | std::cout << std::endl; 93 | snmalloc::debug_check_empty(); 94 | } 95 | 96 | int main() 97 | { 98 | debug_check_empty_1<16>(); 99 | debug_check_empty_1<16384>(); 100 | debug_check_empty_1<65536>(); 101 | debug_check_empty_1<1024 * 1024 * 32>(); 102 | 103 | debug_check_empty_2<32>(); 104 | debug_check_empty_2<16384>(); 105 | debug_check_empty_2<65535>(); 106 | debug_check_empty_2<1024 * 1024 * 32>(); 107 | 108 | return 0; 109 | } 110 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_consts.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../ds_core/ds_core.h" 4 | #include "snmalloc/stl/atomic.h" 5 | 6 | namespace snmalloc 7 | { 8 | /** 9 | * Flags in a bitfield of optional features that a PAL may support. These 10 | * should be set in the PAL's `pal_features` static constexpr field. 11 | */ 12 | enum PalFeatures : uint64_t 13 | { 14 | /** 15 | * This PAL supports low memory notifications. It must implement a 16 | * `register_for_low_memory_callback` method that allows callbacks to be 17 | * registered when the platform detects low-memory and an 18 | * `expensive_low_memory_check()` method that returns a `bool` indicating 19 | * whether low memory conditions are still in effect. 20 | */ 21 | LowMemoryNotification = (1 << 0), 22 | 23 | /** 24 | * This PAL natively supports allocation with a guaranteed alignment. If 25 | * this is not supported, then we will over-allocate and round the 26 | * allocation. 27 | * 28 | * A PAL that does supports this must expose a `request()` method that takes 29 | * a size and alignment. A PAL that does *not* support it must expose a 30 | * `request()` method that takes only a size. 31 | */ 32 | AlignedAllocation = (1 << 1), 33 | 34 | /** 35 | * This PAL natively supports lazy commit of pages. This means have large 36 | * allocations and not touching them does not increase memory usage. This is 37 | * exposed in the Pal. 38 | */ 39 | LazyCommit = (1 << 2), 40 | 41 | /** 42 | * This Pal does not support allocation. All memory used with this Pal 43 | * should be pre-allocated. 44 | */ 45 | NoAllocation = (1 << 3), 46 | 47 | /** 48 | * This Pal provides a source of Entropy 49 | */ 50 | Entropy = (1 << 4), 51 | 52 | /** 53 | * This Pal provides a millisecond time source 54 | */ 55 | Time = (1 << 5), 56 | 57 | /** 58 | * This Pal provides selective core dumps, so 59 | * modify which parts get dumped. 60 | */ 61 | CoreDump = (1 << 6), 62 | 63 | /** 64 | * This Pal provides a way for parking threads at a specific address. 65 | */ 66 | WaitOnAddress = (1 << 7), 67 | }; 68 | 69 | /** 70 | * Flag indicating whether requested memory should be zeroed. 71 | */ 72 | enum ZeroMem 73 | { 74 | /** 75 | * Memory should not be zeroed, contents are undefined. 76 | */ 77 | NoZero, 78 | 79 | /** 80 | * Memory must be zeroed. This can be lazily allocated via a copy-on-write 81 | * mechanism as long as any load from the memory returns zero. 82 | */ 83 | YesZero 84 | }; 85 | 86 | /** 87 | * Default Tag ID for the Apple class 88 | */ 89 | static const int PALAnonDefaultID = 241; 90 | 91 | /** 92 | * Query whether the PAL supports a specific feature. 93 | */ 94 | template 95 | static constexpr bool pal_supports = (PAL::pal_features & F) == F; 96 | } // namespace snmalloc 97 | -------------------------------------------------------------------------------- /src/snmalloc/stl/gnu/array.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "snmalloc/ds_core/defines.h" 4 | 5 | #include 6 | 7 | namespace snmalloc 8 | { 9 | namespace stl 10 | { 11 | template 12 | struct Array 13 | { 14 | // N is potentially 0, so we mark it with __extension__ attribute. 15 | // Expose this to public to allow aggregate initialization 16 | __extension__ T _storage[N]; 17 | 18 | [[nodiscard]] constexpr SNMALLOC_FAST_PATH size_t size() const 19 | { 20 | return N; 21 | } 22 | 23 | constexpr T& operator[](size_t i) 24 | { 25 | return _storage[i]; 26 | } 27 | 28 | constexpr const T& operator[](size_t i) const 29 | { 30 | return _storage[i]; 31 | } 32 | 33 | using value_type = T; 34 | using size_type = size_t; 35 | using iterator = T*; 36 | using const_iterator = const T*; 37 | 38 | [[nodiscard]] constexpr SNMALLOC_FAST_PATH iterator begin() 39 | { 40 | return &_storage[0]; 41 | } 42 | 43 | [[nodiscard]] constexpr SNMALLOC_FAST_PATH const_iterator begin() const 44 | { 45 | return &_storage[0]; 46 | } 47 | 48 | [[nodiscard]] constexpr SNMALLOC_FAST_PATH iterator end() 49 | { 50 | return &_storage[N]; 51 | } 52 | 53 | [[nodiscard]] constexpr SNMALLOC_FAST_PATH const_iterator end() const 54 | { 55 | return &_storage[N]; 56 | } 57 | 58 | [[nodiscard]] constexpr SNMALLOC_FAST_PATH T* data() 59 | { 60 | return &_storage[0]; 61 | } 62 | 63 | [[nodiscard]] constexpr SNMALLOC_FAST_PATH const T* data() const 64 | { 65 | return &_storage[0]; 66 | } 67 | }; 68 | 69 | template 70 | constexpr T* begin(Array& a) 71 | { 72 | return a.begin(); 73 | } 74 | 75 | template 76 | constexpr T* end(Array& a) 77 | { 78 | return a.end(); 79 | } 80 | 81 | template 82 | constexpr const T* begin(const Array& a) 83 | { 84 | return a.begin(); 85 | } 86 | 87 | template 88 | constexpr const T* end(const Array& a) 89 | { 90 | return a.end(); 91 | } 92 | 93 | template 94 | constexpr T* begin(T (&a)[N]) 95 | { 96 | return &a[0]; 97 | } 98 | 99 | template 100 | constexpr T* end(T (&a)[N]) 101 | { 102 | return &a[N]; 103 | } 104 | 105 | template 106 | constexpr const T* begin(const T (&a)[N]) 107 | { 108 | return &a[0]; 109 | } 110 | 111 | template 112 | constexpr const T* end(const T (&a)[N]) 113 | { 114 | return &a[N]; 115 | } 116 | } // namespace stl 117 | } // namespace snmalloc 118 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_noalloc.h: -------------------------------------------------------------------------------- 1 | #ifndef SNMALLOC_PAL_NO_ALLOC_H 2 | #define SNMALLOC_PAL_NO_ALLOC_H 3 | 4 | #pragma once 5 | 6 | #include "../aal/aal.h" 7 | #include "pal_concept.h" 8 | #include "pal_consts.h" 9 | #include "pal_timer_default.h" 10 | 11 | #include 12 | 13 | namespace snmalloc 14 | { 15 | #ifdef __cpp_concepts 16 | /** 17 | * The minimal subset of a PAL that we need for delegation 18 | */ 19 | template 20 | concept PALNoAllocBase = IsPAL_static_sizes && IsPAL_error; 21 | #endif 22 | 23 | /** 24 | * Platform abstraction layer that does not allow allocation. 25 | * 26 | * This is a minimal PAL for pre-reserved memory regions, where the 27 | * address-space manager is initialised with all of the memory that it will 28 | * ever use. 29 | */ 30 | template 31 | struct PALNoAlloc : public BasePAL 32 | { 33 | /** 34 | * Bitmap of PalFeatures flags indicating the optional features that this 35 | * PAL supports. 36 | */ 37 | static constexpr uint64_t pal_features = NoAllocation; 38 | 39 | static constexpr size_t page_size = BasePAL::page_size; 40 | 41 | static constexpr size_t address_bits = BasePAL::address_bits; 42 | 43 | /** 44 | * Print a stack trace. 45 | */ 46 | static void print_stack_trace() 47 | { 48 | BasePAL::print_stack_trace(); 49 | } 50 | 51 | /** 52 | * Report a message to the user. 53 | */ 54 | static void message(const char* const str) noexcept 55 | { 56 | BasePAL::message(str); 57 | } 58 | 59 | /** 60 | * Report a fatal error an exit. 61 | */ 62 | [[noreturn]] static void error(const char* const str) noexcept 63 | { 64 | BasePAL::error(str); 65 | } 66 | 67 | /** 68 | * Notify platform that we will not be using these pages. 69 | * 70 | * This is a no-op in this stub. 71 | */ 72 | static void notify_not_using(void*, size_t) noexcept {} 73 | 74 | /** 75 | * Notify platform that we will be using these pages. 76 | * 77 | * This is a no-op in this stub, except for zeroing memory if required. 78 | */ 79 | template 80 | static bool notify_using(void* p, size_t size) noexcept 81 | { 82 | if constexpr (zero_mem == YesZero) 83 | { 84 | zero(p, size); 85 | } 86 | else 87 | { 88 | UNUSED(p, size); 89 | } 90 | return true; 91 | } 92 | 93 | /** 94 | * OS specific function for zeroing memory. 95 | * 96 | * This just calls memset - we don't assume that we have access to any 97 | * virtual-memory functions. 98 | */ 99 | template 100 | static void zero(void* p, size_t size) noexcept 101 | { 102 | memset(p, 0, size); 103 | } 104 | }; 105 | } // namespace snmalloc 106 | 107 | #endif 108 | -------------------------------------------------------------------------------- /src/snmalloc/pal/pal_freebsd_kernel.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../ds_core/ds_core.h" 4 | 5 | #if defined(FreeBSD_KERNEL) 6 | extern "C" 7 | { 8 | # include 9 | # include 10 | # include 11 | # include 12 | # include 13 | # include 14 | } 15 | 16 | namespace snmalloc 17 | { 18 | class PALFreeBSDKernel 19 | { 20 | vm_offset_t get_vm_offset(uint_ptr_t p) 21 | { 22 | return static_cast(reinterpret_cast(p)); 23 | } 24 | 25 | public: 26 | /** 27 | * Bitmap of PalFeatures flags indicating the optional features that this 28 | * PAL supports. 29 | */ 30 | static constexpr uint64_t pal_features = AlignedAllocation; 31 | 32 | /** 33 | * Report a message to console, followed by a newline. 34 | */ 35 | static void message(const char* const str) noexcept 36 | { 37 | printf("%s\n", str); 38 | } 39 | 40 | [[noreturn]] void error(const char* const str) 41 | { 42 | panic("snmalloc error: %s", str); 43 | } 44 | 45 | /// Notify platform that we will not be using these pages 46 | static void notify_not_using(void* p, size_t size) 47 | { 48 | vm_offset_t addr = get_vm_offset(p); 49 | kmem_unback(kernel_object, addr, size); 50 | } 51 | 52 | /// Notify platform that we will be using these pages 53 | template 54 | static bool notify_using(void* p, size_t size) 55 | { 56 | vm_offset_t addr = get_vm_offset(p); 57 | int flags = M_WAITOK | ((zero_mem == YesZero) ? M_ZERO : 0); 58 | if (kmem_back(kernel_object, addr, size, flags) != KERN_SUCCESS) 59 | { 60 | return false; 61 | } 62 | return true; 63 | } 64 | 65 | /// OS specific function for zeroing memory 66 | template 67 | static void zero(void* p, size_t size) 68 | { 69 | ::bzero(p, size); 70 | } 71 | 72 | template 73 | static void* reserve_aligned(size_t size) noexcept 74 | { 75 | SNMALLOC_ASSERT(bits::is_pow2(size)); 76 | SNMALLOC_ASSERT(size >= minimum_alloc_size); 77 | size_t align = size; 78 | 79 | vm_offset_t addr; 80 | if (vmem_xalloc( 81 | kernel_arena, 82 | size, 83 | align, 84 | 0, 85 | 0, 86 | VMEM_ADDR_MIN, 87 | VMEM_ADDR_MAX, 88 | M_BESTFIT, 89 | &addr)) 90 | { 91 | return nullptr; 92 | } 93 | if (state_using) 94 | { 95 | if ( 96 | kmem_back(kernel_object, addr, size, M_ZERO | M_WAITOK) != 97 | KERN_SUCCESS) 98 | { 99 | vmem_xfree(kernel_arena, addr, size); 100 | return nullptr; 101 | } 102 | } 103 | return get_vm_offset(addr); 104 | } 105 | }; 106 | } 107 | #endif 108 | -------------------------------------------------------------------------------- /src/snmalloc/mem/ticker.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../ds_core/ds_core.h" 4 | 5 | #include 6 | 7 | namespace snmalloc 8 | { 9 | /** 10 | * This class will attempt to call the PAL every 50ms to check the time. 11 | * If the caller of check_tick, does so more frequently, it will attempt 12 | * to back-off to only query the time, every n calls to check_tick, where 13 | * `n` adapts to the current frequency of calling. 14 | * 15 | * The aim is to reduce the time spent querying the time as this might be 16 | * an expensive operation if time has been virtualised. 17 | */ 18 | template 19 | class Ticker 20 | { 21 | /** 22 | * Calls to check_tick required before the time is next queried 23 | */ 24 | uint64_t count_down = 1; 25 | 26 | /** 27 | * Number of ticks next time we check the time. 28 | * That is, 29 | * counted - count_down 30 | * Is how many ticks, since last_epoch_ms was updated. 31 | */ 32 | uint64_t counted = 1; 33 | 34 | /** 35 | * Last time we queried the clock. 36 | */ 37 | uint64_t last_query_ms = 0; 38 | 39 | /** 40 | * Slow path that actually queries clock and sets up 41 | * how many calls for the next time we hit the slow path. 42 | */ 43 | template 44 | SNMALLOC_SLOW_PATH T check_tick_slow(T p = nullptr) noexcept 45 | { 46 | uint64_t now_ms = PAL::time_in_ms(); 47 | 48 | // Set up clock. 49 | if (last_query_ms == 0) 50 | { 51 | last_query_ms = now_ms; 52 | count_down = 1; 53 | counted = 1; 54 | return p; 55 | } 56 | 57 | uint64_t duration_ms = now_ms - last_query_ms; 58 | last_query_ms = now_ms; 59 | 60 | // Check is below clock resolution 61 | if (duration_ms == 0) 62 | { 63 | // Exponential back off 64 | count_down = counted; 65 | counted *= 2; 66 | return p; 67 | } 68 | 69 | constexpr size_t deadline_in_ms = 50; 70 | 71 | // Estimate number of ticks to get to the new deadline, based on the 72 | // current interval 73 | auto new_deadline_in_ticks = 74 | ((1 + counted) * deadline_in_ms) / duration_ms; 75 | 76 | counted = new_deadline_in_ticks; 77 | count_down = new_deadline_in_ticks; 78 | 79 | return p; 80 | } 81 | 82 | public: 83 | template 84 | SNMALLOC_FAST_PATH T check_tick(T p = nullptr) 85 | { 86 | if constexpr (pal_supports) 87 | { 88 | // Check before decrement, so that later calcations can use 89 | // count_down == 0 for check on the next call. 90 | // This is used if the ticks are way below the frequency of 91 | // heart beat. 92 | if (--count_down == 0) 93 | { 94 | return check_tick_slow(p); 95 | } 96 | } 97 | return p; 98 | } 99 | }; 100 | } // namespace snmalloc 101 | -------------------------------------------------------------------------------- /docs/security/README.md: -------------------------------------------------------------------------------- 1 | # Hardening snmalloc 2 | 3 | The key focus of the 0.6.0 release of snmalloc is security. 4 | This was inspired by a few different things coming together. 5 | 6 | First, we had been discussing with the Microsoft Security Response various research on allocator hardening. 7 | Saar Amar had been categorising exploits and what features an allocator should have. 8 | As part of this, we both realised the existing structures of snmalloc made certain things hard to harden, but more interesting we had some ideas for stuff that could advance the state of the art. 9 | 10 | Secondly, we had been experimenting with adding support to snmalloc for [CHERI](http://www.chericpu.org). 11 | This support illustrated many places where snmalloc (like most allocators) does pointer tricks that go against the grain of CHERI. 12 | There were refactorings that would make CHERI support much more natural, but those refactorings were quite involved. 13 | Fortunately, they were the very refactorings we needed for the other allocator hardening research we wanted to conduct. 14 | 15 | The core aim of our refactoring for 0.6.0 is to provide hardening that can be switched on all the time even in allocation heavy work loads. 16 | We have been super keen to keep fast paths fast, and not lose the awesome performance. 17 | Here we illustrate the performance using the application benchmarks from mimalloc-bench: 18 | 19 | ![Performance graph](./data/perfgraph.png) 20 | 21 | The primary comparison point in the graphs is to show the introduced overheads of the checks by comparing `sn-0.6.0` with `sn-0.6.0-checks`. 22 | Here you can see the switching the hardening on leads to regressions under 5%. This is running on a 72-core VM in Azure with each run benchmark repeated 20 times. 23 | 24 | We have also included a few other allocators. 25 | Firstly, [jemalloc](https://github.com/jemalloc/jemalloc) v5.2.1 (labelled `je`) as a baseline for a world-class allocator, and two secure allocators [mimalloc](https://github.com/microsoft/mimalloc) v1.7.6 with its security features enabled (labelled mi-secure), and [SCUDO](https://www.llvm.org/docs/ScudoHardenedAllocator.html) (Commit hash bf0bcd5e, labelled `scudo`). 26 | The precise hardening features in these allocators is different to snmalloc, hence the performance is not directly comparable. 27 | We present them to show the hardenings snmalloc hit a lower performance penalty. 28 | 29 | To really understand the performance security trade-off, you need to understand the hardening features we have implemented. We have a series of short explanations to explain these mechanisms, and what protections we get: 30 | 31 | * [Enabling variable sized slabs](./VariableSizedChunks.md) 32 | * [Enabling guarded `memcpy`](./GuardedMemcpy.md) 33 | * [Protecting free lists from user corruption](./FreelistProtection.md) 34 | * [Randomisation of allocations](./Randomisation.md) 35 | 36 | To try out the hardening features of snmalloc on Elf platforms (e.g. Linux, BSD) you can simply [build](../BUILDING.md) and then preload with: 37 | ``` 38 | LD_PRELOAD=[snmalloc_build]/libsnmalloc-checks.so ./my_app 39 | ``` 40 | -------------------------------------------------------------------------------- /src/snmalloc/backend/standard_range.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | #pragma once 4 | 5 | #include "../backend/backend.h" 6 | #include "base_constants.h" 7 | 8 | namespace snmalloc 9 | { 10 | /** 11 | * Default configuration that does not provide any meta-data protection. 12 | * 13 | * PAL is the underlying PAL that is used to Commit memory ranges. 14 | * 15 | * Base is where memory is sourced from. 16 | * 17 | * MinSizeBits is the minimum request size that can be passed to Base. 18 | * On Windows this 16 as VirtualAlloc cannot reserve less than 64KiB. 19 | * Alternative configurations might make this 2MiB so that huge pages 20 | * can be used. 21 | */ 22 | template< 23 | typename PAL, 24 | typename Pagemap, 25 | typename Base = EmptyRange<>, 26 | size_t MinSizeBits = MinBaseSizeBits()> 27 | struct StandardLocalState : BaseLocalStateConstants 28 | { 29 | // Global range of memory, expose this so can be filled by init. 30 | using GlobalR = Pipe< 31 | Base, 32 | LargeBuddyRange< 33 | GlobalCacheSizeBits, 34 | bits::BITS - 1, 35 | Pagemap, 36 | MinSizeBits>, 37 | LogRange<2>, 38 | GlobalRange>; 39 | 40 | // Track stats of the committed memory 41 | using Stats = Pipe, StatsRange>; 42 | 43 | private: 44 | static constexpr size_t page_size_bits = 45 | bits::next_pow2_bits_const(PAL::page_size); 46 | 47 | public: 48 | // Source for object allocations and metadata 49 | // Use buddy allocators to cache locally. 50 | using LargeObjectRange = Pipe< 51 | Stats, 52 | StaticConditionalRange>>; 57 | 58 | private: 59 | using ObjectRange = Pipe; 60 | 61 | ObjectRange object_range; 62 | 63 | public: 64 | // Expose a global range for the initial allocation of meta-data. 65 | using GlobalMetaRange = Pipe; 66 | 67 | /** 68 | * Where we turn for allocations of user chunks. 69 | * 70 | * Reach over the SmallBuddyRange that's at the near end of the ObjectRange 71 | * pipe, rather than having that range adapter dynamically branch to its 72 | * parent. 73 | */ 74 | LargeObjectRange* get_object_range() 75 | { 76 | return object_range.template ancestor(); 77 | } 78 | 79 | /** 80 | * The backend has its own need for small objects without using the 81 | * frontend allocators; this range manages those. 82 | */ 83 | ObjectRange& get_meta_range() 84 | { 85 | // Use the object range to service meta-data requests. 86 | return object_range; 87 | } 88 | 89 | static void set_small_heap() 90 | { 91 | // This disables the thread local caching of large objects. 92 | LargeObjectRange::disable_range(); 93 | } 94 | }; 95 | } // namespace snmalloc 96 | -------------------------------------------------------------------------------- /src/test/perf/external_pointer/externalpointer.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | using namespace snmalloc; 8 | 9 | namespace test 10 | { 11 | static constexpr size_t count_log = 20; 12 | static constexpr size_t count = 1 << count_log; 13 | // Pre allocate all the objects 14 | size_t* objects[count]; 15 | 16 | NOINLINE void setup(xoroshiro::p128r64& r) 17 | { 18 | for (size_t i = 0; i < count; i++) 19 | { 20 | size_t rand = (size_t)r.next(); 21 | size_t offset = bits::clz(rand); 22 | if constexpr (DefaultPal::address_bits > 32) 23 | { 24 | if (offset > 30) 25 | offset = 30; 26 | } 27 | else if (offset > 20) 28 | offset = 20; 29 | 30 | size_t size = (rand & 15) << offset; 31 | if (size < 16) 32 | size = 16; 33 | // store object 34 | objects[i] = (size_t*)snmalloc::alloc(size); 35 | if (objects[i] == nullptr) 36 | abort(); 37 | // Store allocators size for this object 38 | *objects[i] = snmalloc::alloc_size(objects[i]); 39 | } 40 | } 41 | 42 | NOINLINE void teardown() 43 | { 44 | // Deallocate everything 45 | for (size_t i = 0; i < count; i++) 46 | { 47 | snmalloc::dealloc(objects[i]); 48 | } 49 | 50 | snmalloc::debug_check_empty(); 51 | } 52 | 53 | void test_external_pointer(xoroshiro::p128r64& r) 54 | { 55 | // This is very slow on Windows at the moment. Until this is fixed, help 56 | // CI terminate. 57 | #if defined(NDEBUG) && !defined(_MSC_VER) 58 | static constexpr size_t iterations = 10000000; 59 | #else 60 | # ifdef _MSC_VER 61 | // Windows Debug build is very slow on this test. 62 | // Reduce complexity to balance CI times. 63 | static constexpr size_t iterations = 50000; 64 | # else 65 | static constexpr size_t iterations = 100000; 66 | # endif 67 | #endif 68 | setup(r); 69 | 70 | { 71 | MeasureTime m; 72 | m << "External pointer queries "; 73 | for (size_t i = 0; i < iterations; i++) 74 | { 75 | size_t rand = (size_t)r.next(); 76 | size_t oid = rand & (((size_t)1 << count_log) - 1); 77 | size_t* external_ptr = objects[oid]; 78 | if (!snmalloc::is_owned(external_ptr)) 79 | continue; 80 | size_t size = *external_ptr; 81 | size_t offset = (size >> 4) * (rand & 15); 82 | void* interior_ptr = pointer_offset(external_ptr, offset); 83 | void* calced_external = snmalloc::external_pointer(interior_ptr); 84 | if (calced_external != external_ptr) 85 | { 86 | abort(); 87 | } 88 | } 89 | } 90 | 91 | teardown(); 92 | } 93 | } 94 | 95 | int main(int, char**) 96 | { 97 | setup(); 98 | 99 | xoroshiro::p128r64 r; 100 | 101 | size_t nn = snmalloc::Debug ? 30 : 3; 102 | 103 | for (size_t n = 0; n < nn; n++) 104 | test::test_external_pointer(r); 105 | return 0; 106 | } 107 | -------------------------------------------------------------------------------- /src/test/func/first_operation/first_operation.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * The first operation a thread performs takes a different path to every 3 | * subsequent operation as it must lazily initialise the thread local allocator. 4 | * This tests performs all sizes of allocation, and deallocation as the first 5 | * operation. 6 | */ 7 | 8 | #include "test/setup.h" 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | void alloc1(size_t size) 15 | { 16 | void* r = snmalloc::alloc(size); 17 | snmalloc::dealloc(r); 18 | } 19 | 20 | void alloc2(size_t size) 21 | { 22 | void* r = snmalloc::alloc(size); 23 | snmalloc::dealloc(r, size); 24 | } 25 | 26 | void check_calloc(void* p, size_t size) 27 | { 28 | if (p != nullptr) 29 | { 30 | for (size_t i = 0; i < size; i++) 31 | { 32 | if (((uint8_t*)p)[i] != 0) 33 | { 34 | std::cout << "Calloc contents:" << std::endl; 35 | for (size_t j = 0; j < size; j++) 36 | { 37 | std::cout << std::hex << (size_t)((uint8_t*)p)[j] << " "; 38 | if (j % 32 == 0) 39 | std::cout << std::endl; 40 | } 41 | abort(); 42 | } 43 | // ((uint8_t*)p)[i] = 0x5a; 44 | } 45 | } 46 | } 47 | 48 | void calloc1(size_t size) 49 | { 50 | void* r = snmalloc::alloc(size); 51 | check_calloc(r, size); 52 | snmalloc::dealloc(r); 53 | } 54 | 55 | void calloc2(size_t size) 56 | { 57 | void* r = snmalloc::alloc(size); 58 | check_calloc(r, size); 59 | snmalloc::dealloc(r, size); 60 | } 61 | 62 | void dealloc1(void* p, size_t) 63 | { 64 | snmalloc::dealloc(p); 65 | } 66 | 67 | void dealloc2(void* p, size_t size) 68 | { 69 | snmalloc::dealloc(p, size); 70 | } 71 | 72 | void f(size_t size) 73 | { 74 | auto t1 = std::thread(alloc1, size); 75 | auto t2 = std::thread(alloc2, size); 76 | 77 | auto t3 = std::thread(calloc1, size); 78 | auto t4 = std::thread(calloc2, size); 79 | 80 | { 81 | auto a = snmalloc::get_scoped_allocator(); 82 | auto p1 = a->alloc(size); 83 | auto p2 = a->alloc(size); 84 | 85 | auto t5 = std::thread(dealloc1, p1, size); 86 | auto t6 = std::thread(dealloc2, p2, size); 87 | 88 | t1.join(); 89 | t2.join(); 90 | t3.join(); 91 | t4.join(); 92 | t5.join(); 93 | t6.join(); 94 | } // Drops a. 95 | snmalloc::debug_in_use(0); 96 | printf("."); 97 | fflush(stdout); 98 | } 99 | 100 | int main(int, char**) 101 | { 102 | setup(); 103 | printf("."); 104 | fflush(stdout); 105 | 106 | f(0); 107 | f(1); 108 | f(3); 109 | f(5); 110 | f(7); 111 | printf("\n"); 112 | for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) 113 | { 114 | auto shifted = [exp](size_t v) { return v << exp; }; 115 | 116 | f(shifted(1)); 117 | f(shifted(3)); 118 | f(shifted(5)); 119 | f(shifted(7)); 120 | f(shifted(1) + 1); 121 | f(shifted(3) + 1); 122 | f(shifted(5) + 1); 123 | f(shifted(7) + 1); 124 | f(shifted(1) - 1); 125 | f(shifted(3) - 1); 126 | f(shifted(5) - 1); 127 | f(shifted(7) - 1); 128 | printf("\n"); 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/test/setup.h: -------------------------------------------------------------------------------- 1 | #if defined(SNMALLOC_CI_BUILD) 2 | # include 3 | # if defined(WIN32) 4 | # include 5 | # include 6 | # include 7 | # include 8 | // Has to come after the PAL. 9 | # include 10 | # pragma comment(lib, "dbghelp.lib") 11 | 12 | void print_stack_trace() 13 | { 14 | DWORD error; 15 | HANDLE hProcess = GetCurrentProcess(); 16 | 17 | char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)]; 18 | PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer; 19 | 20 | pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO); 21 | pSymbol->MaxNameLen = MAX_SYM_NAME; 22 | 23 | SymSetOptions(SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS); 24 | 25 | if (!SymInitialize(hProcess, NULL, TRUE)) 26 | { 27 | // SymInitialize failed 28 | error = GetLastError(); 29 | printf("SymInitialize returned error : %lu\n", error); 30 | return; 31 | } 32 | 33 | void* stack[1024]; 34 | DWORD count = CaptureStackBackTrace(0, 1024, stack, NULL); 35 | IMAGEHLP_LINE64 line; 36 | line.SizeOfStruct = sizeof(IMAGEHLP_LINE64); 37 | 38 | for (int i = 0; count > 0; count--, i++) 39 | { 40 | DWORD64 dwDisplacement = 0; 41 | DWORD64 dwAddress = (DWORD64)stack[i]; 42 | 43 | if (SymFromAddr(hProcess, dwAddress, &dwDisplacement, pSymbol)) 44 | { 45 | DWORD dwDisplacement2 = 0; 46 | if (SymGetLineFromAddr64(hProcess, dwAddress, &dwDisplacement2, &line)) 47 | { 48 | std::cerr << "Frame: " << pSymbol->Name << " (" << line.FileName << ": " 49 | << line.LineNumber << ")" << std::endl; 50 | } 51 | else 52 | { 53 | std::cerr << "Frame: " << pSymbol->Name << std::endl; 54 | } 55 | } 56 | else 57 | { 58 | error = GetLastError(); 59 | std::cerr << "SymFromAddr returned error : " << error << std::endl; 60 | } 61 | } 62 | } 63 | 64 | void _cdecl error(int signal) 65 | { 66 | snmalloc::UNUSED(signal); 67 | snmalloc::DefaultPal::message("*****ABORT******"); 68 | 69 | print_stack_trace(); 70 | 71 | _exit(1); 72 | } 73 | 74 | LONG WINAPI VectoredHandler(struct _EXCEPTION_POINTERS* ExceptionInfo) 75 | { 76 | snmalloc::UNUSED(ExceptionInfo); 77 | 78 | snmalloc::DefaultPal::message("*****UNHANDLED EXCEPTION******"); 79 | 80 | print_stack_trace(); 81 | 82 | _exit(1); 83 | } 84 | 85 | void setup() 86 | { 87 | // Disable abort dialog box in CI builds. 88 | _set_error_mode(_OUT_TO_STDERR); 89 | _set_abort_behavior(0, _WRITE_ABORT_MSG); 90 | signal(SIGABRT, error); 91 | 92 | // If we have an unhandled exception print a stack trace. 93 | SetUnhandledExceptionFilter(VectoredHandler); 94 | 95 | // Disable OS level dialog boxes during CI. 96 | SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); 97 | } 98 | # else 99 | # include 100 | 101 | void error_handle(int signal) 102 | { 103 | snmalloc::UNUSED(signal); 104 | snmalloc::error("Seg Fault"); 105 | _exit(1); 106 | } 107 | 108 | void setup() 109 | { 110 | signal(SIGSEGV, error_handle); 111 | } 112 | # endif 113 | #else 114 | void setup() {} 115 | #endif 116 | -------------------------------------------------------------------------------- /src/test/perf/lotsofthreads/lotsofthread.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * This benchmark is based on 3 | * https://github.com/microsoft/mimalloc/issues/1002#issuecomment-2630410617 4 | * 5 | * It causes large batchs of memory to be freed on a remote thread, and causes 6 | * many aspects of the backend to be under-contention. 7 | * 8 | * The benchmark has a single freeing thread, and many allocating threads. The 9 | * allocating threads communicate using a shared list of memory to free, which 10 | * is protected by a mutex. This causes interesting batch behaviour which 11 | * triggered a bug in the linux backend. 12 | */ 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | using namespace std; 21 | 22 | #include 23 | #define malloc snmalloc::libc::malloc 24 | #define free snmalloc::libc::free 25 | #define malloc_usable_size snmalloc::libc::malloc_usable_size 26 | 27 | std::mutex global_tofree_list_mtx; 28 | std::vector global_tofree_list; 29 | 30 | std::atomic_int mustexit; 31 | 32 | void freeloop() 33 | { 34 | size_t max_list_bytes = 0; 35 | while (1) 36 | { 37 | std::lock_guard guard{global_tofree_list_mtx}; 38 | size_t list_bytes = 0; 39 | for (auto& p : global_tofree_list) 40 | { 41 | list_bytes += malloc_usable_size(p); 42 | free(p); 43 | } 44 | global_tofree_list.clear(); 45 | 46 | if (list_bytes > max_list_bytes) 47 | { 48 | printf("%zd bytes\n", list_bytes); 49 | max_list_bytes = list_bytes; 50 | } 51 | 52 | if (mustexit) 53 | return; 54 | } 55 | } 56 | 57 | void looper(size_t iterations) 58 | { 59 | std::vector tofree_list; 60 | auto flush = [&]() { 61 | { 62 | std::lock_guard guard{global_tofree_list_mtx}; 63 | for (auto& p : tofree_list) 64 | global_tofree_list.push_back(p); 65 | } 66 | tofree_list.clear(); 67 | }; 68 | 69 | auto do_free = [&](void* p) { 70 | tofree_list.push_back(p); 71 | if (tofree_list.size() > 100) 72 | { 73 | flush(); 74 | } 75 | }; 76 | 77 | for (size_t i = 0; i < iterations; ++i) 78 | { 79 | size_t s = snmalloc::bits::one_at_bit(i % 20); 80 | for (size_t j = 0; j < 8; j++) 81 | { 82 | auto ptr = (int*)malloc(s * sizeof(int)); 83 | if (ptr == nullptr) 84 | continue; 85 | *ptr = 1523; 86 | do_free(ptr); 87 | } 88 | } 89 | 90 | flush(); 91 | } 92 | 93 | int main() 94 | { 95 | #ifdef SNMALLOC_THREAD_SANITIZER_ENABLED 96 | size_t iterations = 50000; 97 | #elif defined(__APPLE__) && !defined(SNMALLOC_APPLE_HAS_OS_SYNC_WAIT_ON_ADDRESS) 98 | size_t iterations = 50000; 99 | #elif defined(WIN32) 100 | size_t iterations = 50000; 101 | #else 102 | size_t iterations = 200000; 103 | #endif 104 | 105 | int threadcount = 8; 106 | vector threads; 107 | 108 | for (int i = 0; i < threadcount; ++i) 109 | threads.emplace_back(looper, iterations); 110 | 111 | std::thread freeloop_thread(freeloop); 112 | 113 | for (auto& thread : threads) 114 | { 115 | thread.join(); 116 | } 117 | 118 | mustexit.store(1); 119 | freeloop_thread.join(); 120 | 121 | puts("Done!"); 122 | return 0; 123 | } -------------------------------------------------------------------------------- /src/snmalloc/aal/aal_x86.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifdef _MSC_VER 4 | # include 5 | # include 6 | #else 7 | # include 8 | # include 9 | #endif 10 | 11 | #if defined(__linux__) 12 | # include 13 | #endif 14 | 15 | #if defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \ 16 | defined(_M_AMD64) 17 | # define SNMALLOC_VA_BITS_64 18 | #else 19 | # define SNMALLOC_VA_BITS_32 20 | #endif 21 | 22 | namespace snmalloc 23 | { 24 | /** 25 | * x86-specific architecture abstraction layer. 26 | */ 27 | class AAL_x86 28 | { 29 | /** 30 | * Read the timestamp counter, guaranteeing that all previous instructions 31 | * have been retired. 32 | */ 33 | static inline uint64_t tickp() 34 | { 35 | unsigned int aux; 36 | #if defined(_MSC_VER) 37 | return __rdtscp(&aux); 38 | #else 39 | return __builtin_ia32_rdtscp(&aux); 40 | #endif 41 | } 42 | 43 | /** 44 | * Issue a fully serialising instruction. 45 | */ 46 | static inline void halt_out_of_order() 47 | { 48 | #if defined(_MSC_VER) 49 | int cpu_info[4]; 50 | __cpuid(cpu_info, 0); 51 | #else 52 | unsigned int eax, ebx, ecx, edx; 53 | __get_cpuid(0, &eax, &ebx, &ecx, &edx); 54 | #endif 55 | } 56 | 57 | public: 58 | /** 59 | * Bitmap of AalFeature flags 60 | */ 61 | static constexpr uint64_t aal_features = IntegerPointers; 62 | 63 | static constexpr enum AalName aal_name = X86; 64 | 65 | static constexpr size_t smallest_page_size = 0x1000; 66 | 67 | /** 68 | * On pipelined processors, notify the core that we are in a spin loop and 69 | * that speculative execution past this point may not be a performance gain. 70 | */ 71 | static inline void pause() 72 | { 73 | _mm_pause(); 74 | } 75 | 76 | /** 77 | * Issue a prefetch hint at the specified address. 78 | */ 79 | static inline void prefetch(void* ptr) 80 | { 81 | #if defined(_MSC_VER) 82 | _m_prefetchw(ptr); 83 | #else 84 | _mm_prefetch(reinterpret_cast(ptr), _MM_HINT_ET0); 85 | #endif 86 | } 87 | 88 | /** 89 | * Return a cycle counter value. 90 | */ 91 | static inline uint64_t tick() 92 | { 93 | #if defined(_MSC_VER) 94 | return __rdtsc(); 95 | #else 96 | return __builtin_ia32_rdtsc(); 97 | #endif 98 | } 99 | 100 | /** 101 | * Return the cycle counter value that can be used to indicate the start of 102 | * a benchmark run. This is responsible for ensuring any serialisation 103 | * that the pipeline may require. 104 | */ 105 | static inline uint64_t benchmark_time_start() 106 | { 107 | halt_out_of_order(); 108 | return AAL_Generic::tick(); 109 | } 110 | 111 | /** 112 | * Return the cycle counter value that can be used to indicate the end of a 113 | * benchmark run. This is responsible for ensuring any serialisation that 114 | * the pipeline may require. 115 | */ 116 | static inline uint64_t benchmark_time_end() 117 | { 118 | uint64_t t = tickp(); 119 | halt_out_of_order(); 120 | return t; 121 | } 122 | }; 123 | 124 | using AAL_Arch = AAL_x86; 125 | } // namespace snmalloc 126 | -------------------------------------------------------------------------------- /fuzztest.bazelrc: -------------------------------------------------------------------------------- 1 | ### DO NOT EDIT. Generated file. 2 | # 3 | # To regenerate, run the following from your project's workspace: 4 | # 5 | # bazel run @com_google_fuzztest//bazel:setup_configs > fuzztest.bazelrc 6 | # 7 | # And don't forget to add the following to your project's .bazelrc: 8 | # 9 | # try-import %workspace%/fuzztest.bazelrc 10 | 11 | ### Common options. 12 | # 13 | # Do not use directly. 14 | 15 | # Standard define for \"ifdef-ing\" any fuzz test specific code. 16 | build:fuzztest-common --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION 17 | 18 | # In fuzz tests, we want to catch assertion violations even in optimized builds. 19 | build:fuzztest-common --copt=-UNDEBUG 20 | 21 | # Enable libc++ assertions. 22 | # See https://libcxx.llvm.org/UsingLibcxx.html#enabling-the-safe-libc-mode 23 | build:fuzztest-common --copt=-D_LIBCPP_ENABLE_ASSERTIONS=1 24 | 25 | ### ASan (Address Sanitizer) build configuration. 26 | # 27 | # Use with: --config=asan 28 | 29 | build:asan --linkopt=-fsanitize=address 30 | build:asan --copt=-fsanitize=address 31 | 32 | # We rely on the following flag instead of the compiler provided 33 | # __has_feature(address_sanitizer) to know that we have an ASAN build even in 34 | # the uninstrumented runtime. 35 | build:asan --copt=-DADDRESS_SANITIZER 36 | 37 | ### FuzzTest build configuration. 38 | # 39 | # Use with: --config=fuzztest 40 | # 41 | # Note that this configuration includes the ASan configuration. 42 | 43 | build:fuzztest --config=asan 44 | build:fuzztest --config=fuzztest-common 45 | 46 | # Link statically. 47 | build:fuzztest --dynamic_mode=off 48 | 49 | # We apply coverage tracking instrumentation to everything but Centipede and the 50 | # FuzzTest framework itself (including GoogleTest and GoogleMock). 51 | build:fuzztest --copt=-fsanitize-coverage=inline-8bit-counters,trace-cmp,pc-table 52 | build:fuzztest --per_file_copt=common/.*,fuzztest/.*,centipede/.*,-centipede/.*fuzz_target,googletest/.*,googlemock/.*@-fsanitize-coverage=0 53 | 54 | ### Experimental FuzzTest build configuration. 55 | # 56 | # Use with: --config=fuzztest-experimental 57 | # 58 | # Use this instead of --config=fuzztest when building test binaries to run with 59 | # Centipede. Eventually, this will be consolidated with --config=fuzztest. 60 | # Note that this configuration doesn't include the ASan configuration. If you 61 | # want to use both, you can use --config=fuzztest-experimental --config=asan. 62 | 63 | build:fuzztest-experimental --config=fuzztest-common 64 | build:fuzztest-experimental --@com_google_fuzztest//fuzztest:centipede_integration 65 | 66 | # Generate line tables for debugging. 67 | build:fuzztest-experimental --copt=-gline-tables-only 68 | build:fuzztest-experimental --strip=never 69 | 70 | # Prevent memcmp & co from being inlined. 71 | build:fuzztest-experimental --copt=-fno-builtin 72 | 73 | # Disable heap checking. 74 | build:fuzztest-experimental --copt=-DHEAPCHECK_DISABLE 75 | 76 | # Link statically. 77 | build:fuzztest-experimental --dynamic_mode=off 78 | 79 | # We apply coverage tracking instrumentation to everything but Centipede and the 80 | # FuzzTest framework itself (including GoogleTest and GoogleMock). 81 | # TODO(b/374840534): Add -fsanitize-coverage=control-flow once we start building 82 | # with clang 16+. 83 | build:fuzztest-experimental --copt=-fsanitize-coverage=trace-pc-guard,pc-table,trace-loads,trace-cmp 84 | build:fuzztest-experimental --per_file_copt=common/.*,fuzztest/.*,centipede/.*,-centipede/.*fuzz_target,googletest/.*,googlemock/.*@-fsanitize-coverage=0 85 | 86 | -------------------------------------------------------------------------------- /src/snmalloc/aal/aal_concept.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifdef __cpp_concepts 4 | # include "../ds_core/ds_core.h" 5 | # include "aal_consts.h" 6 | 7 | # include 8 | 9 | namespace snmalloc 10 | { 11 | /** 12 | * AALs must advertise the bit vector of supported features, their name, 13 | * machine word size, and an upper bound on the address space size 14 | */ 15 | template 16 | concept IsAAL_static_members = 17 | requires() { 18 | typename stl::integral_constant; 19 | typename stl::integral_constant; 20 | typename stl::integral_constant; 21 | typename stl::integral_constant; 22 | }; 23 | 24 | /** 25 | * AALs provide a prefetch operation. 26 | */ 27 | template 28 | concept IsAAL_prefetch = requires(void* ptr) { 29 | { 30 | AAL::prefetch(ptr) 31 | } noexcept -> ConceptSame; 32 | }; 33 | 34 | /** 35 | * AALs provide a notion of high-precision timing. 36 | */ 37 | template 38 | concept IsAAL_tick = requires() { 39 | { 40 | AAL::tick() 41 | } noexcept -> ConceptSame; 42 | }; 43 | 44 | template 45 | concept IsAAL_capptr_methods = 46 | requires(capptr::Chunk auth, capptr::AllocFull ret, size_t sz) { 47 | /** 48 | * Produce a pointer with reduced authority from a more privilged pointer. 49 | * The resulting pointer will have base at auth's address and length of 50 | * exactly sz. auth+sz must not exceed auth's limit. 51 | */ 52 | { 53 | AAL::template capptr_bound(auth, sz) 54 | } noexcept -> ConceptSame>; 55 | 56 | /** 57 | * "Amplify" by copying the address of one pointer into one of higher 58 | * privilege. The resulting pointer differs from auth only in address. 59 | */ 60 | { 61 | AAL::capptr_rebound(auth, ret) 62 | } noexcept -> ConceptSame>; 63 | 64 | /** 65 | * Round up an allocation size to a size this architecture can represent. 66 | * While there may also, in general, be alignment requirements for 67 | * representability, in snmalloc so far we have not had reason to consider 68 | * these explicitly: when we use our... 69 | * 70 | * - sizeclass machinery (for user-facing data), we assume that all 71 | * sizeclasses describe architecturally representable aligned-and-sized 72 | * regions 73 | * 74 | * - Range machinery (for internal meta-data), we always choose NAPOT 75 | * regions big enough for the requested size (returning space above the 76 | * allocation within such regions for use as smaller NAPOT regions). 77 | * 78 | * That is, capptr_size_round is not needed on the user-facing fast paths, 79 | * merely internally for bootstrap and metadata management. 80 | */ 81 | { 82 | AAL::capptr_size_round(sz) 83 | } noexcept -> ConceptSame; 84 | }; 85 | 86 | template 87 | concept IsAAL = IsAAL_static_members && IsAAL_prefetch && 88 | IsAAL_tick && IsAAL_capptr_methods; 89 | 90 | } // namespace snmalloc 91 | #endif 92 | -------------------------------------------------------------------------------- /src/test/func/teardown/teardown.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * After a thread has started teardown a different path is taken for 3 | * allocation and deallocation. This tests causes the state to be torn 4 | * down early, and then use the teardown path for multiple allocations 5 | * and deallocation. 6 | */ 7 | 8 | #include "test/setup.h" 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | void trigger_teardown() 15 | { 16 | // Trigger init 17 | void* r = snmalloc::alloc(16); 18 | snmalloc::dealloc(r); 19 | // Force teardown 20 | snmalloc::debug_teardown(); 21 | } 22 | 23 | void alloc1(size_t size) 24 | { 25 | trigger_teardown(); 26 | void* r = snmalloc::alloc(size); 27 | snmalloc::dealloc(r); 28 | } 29 | 30 | void alloc2(size_t size) 31 | { 32 | trigger_teardown(); 33 | void* r = snmalloc::alloc(size); 34 | snmalloc::dealloc(r, size); 35 | } 36 | 37 | void check_calloc(void* p, size_t size) 38 | { 39 | if (p != nullptr) 40 | { 41 | for (size_t i = 0; i < size; i++) 42 | { 43 | if (((uint8_t*)p)[i] != 0) 44 | { 45 | std::cout << "Calloc contents:" << std::endl; 46 | for (size_t j = 0; j < size; j++) 47 | { 48 | std::cout << std::hex << (size_t)((uint8_t*)p)[j] << " "; 49 | if (j % 32 == 0) 50 | std::cout << std::endl; 51 | } 52 | abort(); 53 | } 54 | // ((uint8_t*)p)[i] = 0x5a; 55 | } 56 | } 57 | } 58 | 59 | void calloc1(size_t size) 60 | { 61 | trigger_teardown(); 62 | void* r = snmalloc::alloc(size); 63 | check_calloc(r, size); 64 | snmalloc::dealloc(r); 65 | } 66 | 67 | void calloc2(size_t size) 68 | { 69 | trigger_teardown(); 70 | void* r = snmalloc::alloc(size); 71 | check_calloc(r, size); 72 | snmalloc::dealloc(r, size); 73 | } 74 | 75 | void dealloc1(void* p, size_t) 76 | { 77 | trigger_teardown(); 78 | snmalloc::dealloc(p); 79 | } 80 | 81 | void dealloc2(void* p, size_t size) 82 | { 83 | trigger_teardown(); 84 | snmalloc::dealloc(p, size); 85 | } 86 | 87 | void f(size_t size) 88 | { 89 | auto t1 = std::thread(alloc1, size); 90 | auto t2 = std::thread(alloc2, size); 91 | 92 | auto t3 = std::thread(calloc1, size); 93 | auto t4 = std::thread(calloc2, size); 94 | 95 | { 96 | auto a = snmalloc::get_scoped_allocator(); 97 | auto p1 = a->alloc(size); 98 | auto p2 = a->alloc(size); 99 | 100 | auto t5 = std::thread(dealloc1, p1, size); 101 | auto t6 = std::thread(dealloc2, p2, size); 102 | 103 | t1.join(); 104 | t2.join(); 105 | t3.join(); 106 | t4.join(); 107 | t5.join(); 108 | t6.join(); 109 | } // Drops a. 110 | snmalloc::debug_in_use(0); 111 | printf("."); 112 | fflush(stdout); 113 | } 114 | 115 | int main(int, char**) 116 | { 117 | setup(); 118 | printf("."); 119 | fflush(stdout); 120 | 121 | f(0); 122 | f(1); 123 | f(3); 124 | f(5); 125 | f(7); 126 | printf("\n"); 127 | for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) 128 | { 129 | auto shifted = [exp](size_t v) { return v << exp; }; 130 | 131 | f(shifted(1)); 132 | f(shifted(3)); 133 | f(shifted(5)); 134 | f(shifted(7)); 135 | f(shifted(1) + 1); 136 | f(shifted(3) + 1); 137 | f(shifted(5) + 1); 138 | f(shifted(7) + 1); 139 | f(shifted(1) - 1); 140 | f(shifted(3) - 1); 141 | f(shifted(5) - 1); 142 | f(shifted(7) - 1); 143 | printf("\n"); 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | # BasedOnStyle: LLVM 4 | AccessModifierOffset: -2 5 | AlignAfterOpenBracket: AlwaysBreak 6 | AlignConsecutiveAssignments: false 7 | AlignConsecutiveDeclarations: false 8 | AlignEscapedNewlines: DontAlign 9 | AlignOperands: false 10 | AlignTrailingComments: false 11 | AllowAllParametersOfDeclarationOnNextLine: true 12 | AllowShortBlocksOnASingleLine: false 13 | AllowShortCaseLabelsOnASingleLine: false 14 | AllowShortFunctionsOnASingleLine: Empty 15 | AllowShortIfStatementsOnASingleLine: false 16 | AllowShortLoopsOnASingleLine: false 17 | AlwaysBreakAfterDefinitionReturnType: None 18 | AlwaysBreakAfterReturnType: None 19 | AlwaysBreakBeforeMultilineStrings: true 20 | AlwaysBreakTemplateDeclarations: true 21 | BinPackArguments: false 22 | BinPackParameters: false 23 | BraceWrapping: 24 | AfterCaseLabel: true 25 | AfterClass: true 26 | AfterControlStatement: true 27 | AfterEnum: true 28 | AfterFunction: true 29 | AfterNamespace: true 30 | AfterObjCDeclaration: true 31 | AfterStruct: true 32 | AfterUnion: true 33 | AfterExternBlock: true 34 | BeforeCatch: true 35 | BeforeElse: true 36 | IndentBraces: false 37 | SplitEmptyFunction: false 38 | SplitEmptyRecord: false 39 | SplitEmptyNamespace: false 40 | BreakBeforeBinaryOperators: None 41 | BreakBeforeBraces: Custom 42 | BreakBeforeInheritanceComma: false 43 | BreakBeforeTernaryOperators: false 44 | BreakConstructorInitializersBeforeComma: false 45 | BreakConstructorInitializers: BeforeColon 46 | BreakAfterJavaFieldAnnotations: false 47 | BreakStringLiterals: true 48 | ColumnLimit: 80 49 | CommentPragmas: '^ IWYU pragma:' 50 | CompactNamespaces: false 51 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 52 | ConstructorInitializerIndentWidth: 0 53 | ContinuationIndentWidth: 2 54 | Cpp11BracedListStyle: true 55 | DerivePointerAlignment: false 56 | DisableFormat: false 57 | ExperimentalAutoDetectBinPacking: false 58 | FixNamespaceComments: false 59 | ForEachMacros: 60 | - Q_FOREACH 61 | - BOOST_FOREACH 62 | IncludeBlocks: Regroup 63 | IncludeCategories: 64 | - Regex: '^"(llvm|llvm-c|clang|clang-c)/' 65 | Priority: 2 66 | - Regex: '^(<|"(gtest|gmock|isl|json)/)' 67 | Priority: 3 68 | - Regex: '.*' 69 | Priority: 1 70 | IncludeIsMainRegex: '(Test)?$' 71 | IndentCaseLabels: true 72 | IndentPPDirectives: AfterHash 73 | IndentWidth: 2 74 | IndentWrappedFunctionNames: false 75 | JavaScriptQuotes: Leave 76 | JavaScriptWrapImports: true 77 | KeepEmptyLinesAtTheStartOfBlocks: false 78 | MacroBlockBegin: '' 79 | MacroBlockEnd: '' 80 | MaxEmptyLinesToKeep: 1 81 | NamespaceIndentation: All 82 | ObjCBlockIndentWidth: 2 83 | ObjCSpaceAfterProperty: false 84 | ObjCSpaceBeforeProtocolList: true 85 | PenaltyBreakAssignment: 2 86 | PenaltyBreakBeforeFirstCallParameter: 19 87 | PenaltyBreakComment: 300 88 | PenaltyBreakFirstLessLess: 120 89 | PenaltyBreakString: 1000 90 | PenaltyExcessCharacter: 1000000 91 | PenaltyReturnTypeOnItsOwnLine: 60 92 | PointerAlignment: Left 93 | ReflowComments: true 94 | SeparateDefinitionBlocks: Always 95 | SortIncludes: true 96 | SortUsingDeclarations: true 97 | SpaceAfterCStyleCast: false 98 | SpaceAfterTemplateKeyword: false 99 | SpaceBeforeAssignmentOperators: true 100 | SpaceBeforeParens: ControlStatements 101 | SpaceInEmptyParentheses: false 102 | SpacesBeforeTrailingComments: 1 103 | SpacesInAngles: false 104 | SpacesInContainerLiterals: false 105 | SpacesInCStyleCastParentheses: false 106 | SpacesInParentheses: false 107 | SpacesInSquareBrackets: false 108 | Standard: Cpp11 109 | TabWidth: 2 110 | UseTab: Never 111 | ... 112 | 113 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # snmalloc 2 | 3 | snmalloc is a high-performance allocator. 4 | snmalloc can be used directly in a project as a header-only C++ library, 5 | it can be `LD_PRELOAD`ed on Elf platforms (e.g. Linux, BSD), 6 | and there is a [crate](https://crates.io/crates/snmalloc-rs) to use it from Rust. 7 | 8 | Its key design features are: 9 | 10 | * Memory that is freed by the same thread that allocated it does not require any 11 | synchronising operations. 12 | * Freeing memory in a different thread to initially allocated it, does not take 13 | any locks and instead uses a novel message passing scheme to return the 14 | memory to the original allocator, where it is recycled. This enables 1000s of remote 15 | deallocations to be performed with only a single atomic operation enabling great 16 | scaling with core count. 17 | * The allocator uses large ranges of pages to reduce the amount of meta-data 18 | required. 19 | * The fast paths are highly optimised with just two branches on the fast path 20 | for malloc (On Linux compiled with Clang). 21 | * The platform dependencies are abstracted away to enable porting to other platforms. 22 | 23 | snmalloc's design is particular well suited to the following two difficult 24 | scenarios that can be problematic for other allocators: 25 | 26 | * Allocations on one thread are freed by a different thread 27 | * Deallocations occur in large batches 28 | 29 | Both of these can cause massive reductions in performance of other allocators, but 30 | do not for snmalloc. 31 | 32 | The implementation of snmalloc has evolved significantly since the [initial paper](snmalloc.pdf). 33 | The mechanism for returning memory to remote threads has remained, but most of the meta-data layout has changed. 34 | We recommend you read [docs/security](./docs/security/README.md) to find out about the current design, and 35 | if you want to dive into the code [docs/AddressSpace.md](./docs/AddressSpace.md) provides a good overview of the allocation and deallocation paths. 36 | 37 | [![snmalloc CI](https://github.com/microsoft/snmalloc/actions/workflows/main.yml/badge.svg)](https://github.com/microsoft/snmalloc/actions/workflows/main.yml) 38 | 39 | # Hardening 40 | 41 | There is a hardened version of snmalloc, it contains 42 | 43 | * Randomisation of the allocations' relative locations, 44 | * Most meta-data is stored separately from allocations, and is protected with guard pages, 45 | * All in-band meta-data is protected with a novel encoding that can detect corruption, and 46 | * Provides a `memcpy` that automatically checks the bounds relative to the underlying malloc. 47 | 48 | A more comprehensive write up is in [docs/security](./docs/security/README.md). 49 | 50 | # Further documentation 51 | 52 | - [Instructions for building snmalloc](docs/BUILDING.md) 53 | - [Instructions for porting snmalloc](docs/PORTING.md) 54 | 55 | # Contributing 56 | 57 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 58 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 59 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 60 | 61 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 62 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 63 | provided by the bot. You will only need to do this once across all repos using our CLA. 64 | 65 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 66 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 67 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 68 | -------------------------------------------------------------------------------- /src/snmalloc/mem/backend_wrappers.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /** 3 | * Several of the functions provided by the back end are optional. This file 4 | * contains helpers that are templated on a back end and either call the 5 | * corresponding function or do nothing. This allows the rest of the front end 6 | * to assume that these functions always exist and avoid the need for `if 7 | * constexpr` clauses everywhere. The no-op versions are always inlined and so 8 | * will be optimised away. 9 | */ 10 | 11 | #include "../ds_core/ds_core.h" 12 | 13 | namespace snmalloc 14 | { 15 | /** 16 | * SFINAE helper. Matched only if `T` implements `is_initialised`. Calls 17 | * it if it exists. 18 | */ 19 | template 20 | SNMALLOC_FAST_PATH auto call_is_initialised(T*, int) 21 | -> decltype(T::is_initialised()) 22 | { 23 | return T::is_initialised(); 24 | } 25 | 26 | /** 27 | * SFINAE helper. Matched only if `T` does not implement `is_initialised`. 28 | * Unconditionally returns true if invoked. 29 | */ 30 | template 31 | SNMALLOC_FAST_PATH auto call_is_initialised(T*, long) 32 | { 33 | return true; 34 | } 35 | 36 | namespace detail 37 | { 38 | /** 39 | * SFINAE helper to detect the presence of capptr_domesticate function in 40 | * backend. Returns true if there is a function with correct name and type. 41 | */ 42 | template< 43 | SNMALLOC_CONCEPT(IsConfigDomestication) Config, 44 | typename T, 45 | SNMALLOC_CONCEPT(capptr::IsBound) B> 46 | constexpr SNMALLOC_FAST_PATH auto has_domesticate(int) -> stl::enable_if_t< 47 | stl::is_same_v< 48 | decltype(Config::capptr_domesticate( 49 | stl::declval(), 50 | stl::declval>())), 51 | CapPtr< 52 | T, 53 | typename B::template with_wildness< 54 | capptr::dimension::Wildness::Tame>>>, 55 | bool> 56 | { 57 | return true; 58 | } 59 | 60 | /** 61 | * SFINAE helper to detect the presence of capptr_domesticate function in 62 | * backend. Returns false in case where above template does not match. 63 | */ 64 | template< 65 | SNMALLOC_CONCEPT(IsConfig) Config, 66 | typename T, 67 | SNMALLOC_CONCEPT(capptr::IsBound) B> 68 | constexpr SNMALLOC_FAST_PATH bool has_domesticate(long) 69 | { 70 | return false; 71 | } 72 | } // namespace detail 73 | 74 | /** 75 | * Wrapper that calls `Config::capptr_domesticate` if and only if 76 | * Config::Options.HasDomesticate is true. If it is not implemented then 77 | * this assumes that any wild pointer can be domesticated. 78 | */ 79 | template< 80 | SNMALLOC_CONCEPT(IsConfig) Config, 81 | typename T, 82 | SNMALLOC_CONCEPT(capptr::IsBound) B> 83 | SNMALLOC_FAST_PATH_INLINE auto 84 | capptr_domesticate(typename Config::LocalState* ls, CapPtr p) 85 | { 86 | static_assert( 87 | !detail::has_domesticate(0) || 88 | Config::Options.HasDomesticate, 89 | "Back end provides domesticate function but opts out of using it "); 90 | 91 | static_assert( 92 | detail::has_domesticate(0) || 93 | !Config::Options.HasDomesticate, 94 | "Back end does not provide capptr_domesticate and requests its use"); 95 | if constexpr (Config::Options.HasDomesticate) 96 | { 97 | return Config::capptr_domesticate(ls, p); 98 | } 99 | else 100 | { 101 | UNUSED(ls); 102 | return CapPtr< 103 | T, 104 | typename B::template with_wildness>:: 105 | unsafe_from(p.unsafe_ptr()); 106 | } 107 | } 108 | } // namespace snmalloc 109 | -------------------------------------------------------------------------------- /src/snmalloc/README.md: -------------------------------------------------------------------------------- 1 | Include hierarchy 2 | ----------------- 3 | 4 | The `snmalloc/` include path contains all of the snmalloc headers. 5 | These are arranged in a hierarchy such that each of the directories may include ones below, in the following order, starting at the bottom: 6 | 7 | - `ds_core/` provides core data structures that depend on the C++ implementation and nothing else. 8 | This directory includes a number of things that abstract over different language extensions (for example, different built-in function names in different compilers). 9 | - `aal/` provides the architecture abstraction layer (AAL). 10 | This layer provides abstractions over CPU-specific intrinsics and defines things such as the virtual address-space size. 11 | There is a single AAL for an snmalloc instantiation. 12 | - `ds_aal/` provides data structures that depend on the AAL. 13 | - `pal/` provides the platform abstraction layer (PAL). 14 | This exposes OS- or environment-specific abstractions into the rest of the code. 15 | An snmalloc instantiation may use more than one PAL, including ones provided by the user. 16 | - `ds/` includes data structures that may depend on platform services or on features specific to the current CPU. 17 | - `mem/` provides the core allocator abstractions. 18 | The code here is templated over a back-end, which defines a particular embedding of snmalloc. 19 | - `backend_helpers/` provides helper classes for use in defining a back end. 20 | This includes data structures such as pagemap implementations (efficient maps from a chunk address to associated metadata) and buddy allocators for managing address-space ranges. 21 | - `backend/` provides some example implementations for snmalloc embeddings that provide a global memory allocator for an address space. 22 | Users may ignore this entirely and use the types in `mem/` with a custom back end to expose an snmalloc instance with specific behaviour. 23 | Layers above this can be used with a custom configuration by defining `SNMALLOC_PROVIDE_OWN_CONFIG` and exporting a type as `snmalloc::Config` that defines the configuration. 24 | - `global/` provides some front-end components that assume that snmalloc is available in a global configuration. 25 | - `override/` builds on top of `global/` to provide specific implementations with compatibility with external specifications (for example C `malloc`, C++ `operator new`, jemalloc's `*allocx`, or Rust's `std::alloc`). 26 | 27 | Each layer until `backend_helpers/` provides a single header with the same name as the directory. 28 | Files in higher layers should depend only on the single-file version. 29 | This allows specific files to be moved to a lower layer if appropriate, without too much code churn. 30 | 31 | There is only one exception to this rule: `backend/globalconfig.h`. 32 | This file defines either the default configuration *or* nothing, depending on whether the user has defined `SNMALLOC_PROVIDE_OWN_CONFIG`. 33 | The layers above the back end should include only this file, so that there is a single interception point for externally defined back ends. 34 | 35 | External code should include only the following files: 36 | 37 | - `snmalloc/snmalloc_core.h` includes everything up to `backend_helpers`. 38 | This provides the building blocks required to assemble an snmalloc instance, but does not assume any global configuration. 39 | - `snmalloc/snmalloc_front.h` assumes a global configuration (either user-provided or the default from `snmalloc/backend/globalconfig.h` and exposes all of the functionality that depends on both. 40 | - `snmalloc/snmalloc.h` is a convenience wrapper that includes both of the above files. 41 | - `snmalloc/override/*.cc` can be compiled as-is or included after `snmalloc/snmalloc_core.h` and a custom global allocator definition to provide specific languages' global memory allocator APIs with a custom snmalloc embedding. 42 | -------------------------------------------------------------------------------- /src/snmalloc/global/bounds_checks.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "globalalloc.h" 3 | #include "threadalloc.h" 4 | 5 | namespace snmalloc 6 | { 7 | /** 8 | * Should we check loads? This defaults to on in debug builds, off in 9 | * release (store-only checks) and can be overridden by defining the macro 10 | * `SNMALLOC_CHECK_LOADS` to true or false. 11 | */ 12 | static constexpr bool CheckReads = 13 | #ifdef SNMALLOC_CHECK_LOADS 14 | SNMALLOC_CHECK_LOADS 15 | #else 16 | Debug 17 | #endif 18 | ; 19 | 20 | /** 21 | * Should we fail fast when we encounter an error? With this set to true, we 22 | * just issue a trap instruction and crash the process once we detect an 23 | * error. With it set to false we print a helpful error message and then crash 24 | * the process. The process may be in an undefined state by the time the 25 | * check fails, so there are potentially security implications to turning this 26 | * off. It defaults to false and can be overridden by defining the macro 27 | * `SNMALLOC_FAIL_FAST` to true. 28 | * 29 | * Current default to true will help with adoption experience. 30 | */ 31 | static constexpr bool FailFast = 32 | #ifdef SNMALLOC_FAIL_FAST 33 | SNMALLOC_FAIL_FAST 34 | #else 35 | false 36 | #endif 37 | ; 38 | 39 | /** 40 | * Report an error message for a failed bounds check and then abort the 41 | * program. 42 | * `p` is the input pointer and `len` is the offset from this pointer of the 43 | * bounds. `msg` is the message that will be reported along with the 44 | * start and end of the real object's bounds. 45 | * 46 | * Note that this function never returns. We do not mark it [[NoReturn]] 47 | * so as to generate better code, because [[NoReturn]] prevents tailcails 48 | * in GCC and Clang. 49 | * 50 | * The function claims to return a FakeReturn, this is so it can be tail 51 | * called where the bound checked function returns a value, for instance, in 52 | * memcpy it is specialised to void*. 53 | */ 54 | template 55 | SNMALLOC_SLOW_PATH SNMALLOC_UNUSED_FUNCTION inline FakeReturn 56 | report_fatal_bounds_error(const void* ptr, size_t len, const char* msg) 57 | { 58 | if constexpr (FailFast) 59 | { 60 | UNUSED(ptr, len, msg); 61 | SNMALLOC_FAST_FAIL(); 62 | } 63 | else 64 | { 65 | void* p = const_cast(ptr); 66 | 67 | auto range_end = pointer_offset(p, len); 68 | auto object_end = external_pointer(p); 69 | report_fatal_error( 70 | "Fatal Error!\n{}: \n\trange [{}, {})\n\tallocation [{}, " 71 | "{})\nrange goes beyond allocation by {} bytes \n", 72 | msg, 73 | p, 74 | range_end, 75 | external_pointer(p), 76 | object_end, 77 | pointer_diff(object_end, range_end)); 78 | } 79 | } 80 | 81 | /** 82 | * Check whether a pointer + length is in the same object as the pointer. 83 | * 84 | * Returns the result of the supplied continuation 85 | * 86 | * The template parameter indicates whether the check should be performed. It 87 | * defaults to true. If it is false, it short cuts to calling the continuation 88 | * directly. 89 | */ 90 | template< 91 | bool PerformCheck = true, 92 | typename F, 93 | SNMALLOC_CONCEPT(IsConfig) Config = Config> 94 | SNMALLOC_FAST_PATH_INLINE auto check_bound( 95 | const void* ptr, size_t len, const char* msg, F f = []() {}) 96 | { 97 | if constexpr (PerformCheck) 98 | { 99 | if (SNMALLOC_LIKELY(len != 0)) 100 | { 101 | if (SNMALLOC_UNLIKELY(remaining_bytes(address_cast(ptr)) < len)) 102 | { 103 | return report_fatal_bounds_error(ptr, len, msg); 104 | } 105 | } 106 | } 107 | else 108 | { 109 | UNUSED(ptr, len, msg); 110 | } 111 | return f(); 112 | } 113 | } // namespace snmalloc 114 | -------------------------------------------------------------------------------- /src/snmalloc/override/malloc.cc: -------------------------------------------------------------------------------- 1 | #include "override.h" 2 | 3 | using namespace snmalloc; 4 | 5 | #ifndef MALLOC_USABLE_SIZE_QUALIFIER 6 | # define MALLOC_USABLE_SIZE_QUALIFIER 7 | #endif 8 | 9 | extern "C" 10 | { 11 | SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr) 12 | { 13 | return snmalloc::libc::__malloc_end_pointer(ptr); 14 | } 15 | 16 | SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) 17 | { 18 | return snmalloc::libc::malloc(size); 19 | } 20 | 21 | SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(free)(void* ptr) 22 | { 23 | snmalloc::libc::free(ptr); 24 | } 25 | 26 | SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(cfree)(void* ptr) 27 | { 28 | snmalloc::libc::free(ptr); 29 | } 30 | 31 | SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(calloc)(size_t nmemb, size_t size) 32 | { 33 | return snmalloc::libc::calloc(nmemb, size); 34 | } 35 | 36 | SNMALLOC_EXPORT 37 | size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)( 38 | MALLOC_USABLE_SIZE_QUALIFIER void* ptr) 39 | { 40 | return snmalloc::libc::malloc_usable_size(ptr); 41 | } 42 | 43 | #ifdef _WIN32 44 | SNMALLOC_EXPORT 45 | size_t SNMALLOC_NAME_MANGLE(_msize)(void* ptr) 46 | { 47 | return snmalloc::libc::malloc_usable_size(ptr); 48 | } 49 | #endif 50 | 51 | SNMALLOC_EXPORT 52 | size_t SNMALLOC_NAME_MANGLE(malloc_good_size)(size_t size) 53 | { 54 | return round_size(size); 55 | } 56 | 57 | SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(realloc)(void* ptr, size_t size) 58 | { 59 | return snmalloc::libc::realloc(ptr, size); 60 | } 61 | 62 | #if !defined(SNMALLOC_NO_REALLOCARRAY) 63 | SNMALLOC_EXPORT void* 64 | SNMALLOC_NAME_MANGLE(reallocarray)(void* ptr, size_t nmemb, size_t size) 65 | { 66 | return snmalloc::libc::reallocarray(ptr, nmemb, size); 67 | } 68 | #endif 69 | 70 | #if !defined(SNMALLOC_NO_REALLOCARR) 71 | SNMALLOC_EXPORT int 72 | SNMALLOC_NAME_MANGLE(reallocarr)(void* ptr, size_t nmemb, size_t size) 73 | { 74 | return snmalloc::libc::reallocarr(ptr, nmemb, size); 75 | } 76 | #endif 77 | 78 | SNMALLOC_EXPORT void* 79 | SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size) 80 | { 81 | return snmalloc::libc::memalign(alignment, size); 82 | } 83 | 84 | SNMALLOC_EXPORT void* 85 | SNMALLOC_NAME_MANGLE(aligned_alloc)(size_t alignment, size_t size) 86 | { 87 | return snmalloc::libc::aligned_alloc(alignment, size); 88 | } 89 | 90 | SNMALLOC_EXPORT int SNMALLOC_NAME_MANGLE(posix_memalign)( 91 | void** memptr, size_t alignment, size_t size) 92 | { 93 | return snmalloc::libc::posix_memalign(memptr, alignment, size); 94 | } 95 | 96 | #if !defined(__FreeBSD__) && !defined(__OpenBSD__) 97 | SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(valloc)(size_t size) 98 | { 99 | return snmalloc::libc::memalign(OS_PAGE_SIZE, size); 100 | } 101 | #endif 102 | 103 | SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(pvalloc)(size_t size) 104 | { 105 | return snmalloc::libc::memalign( 106 | OS_PAGE_SIZE, (size + OS_PAGE_SIZE - 1) & ~(OS_PAGE_SIZE - 1)); 107 | } 108 | 109 | #if __has_include() 110 | # include 111 | #endif 112 | #if defined(__GLIBC__) 113 | // glibc uses these hooks to replace malloc. 114 | // This is required when RTL_DEEPBIND is used and the library is 115 | // LD_PRELOADed. 116 | // See https://github.com/microsoft/snmalloc/issues/595 117 | SNMALLOC_EXPORT void (*SNMALLOC_NAME_MANGLE(__free_hook))(void* ptr) = 118 | &SNMALLOC_NAME_MANGLE(free); 119 | SNMALLOC_EXPORT void* (*SNMALLOC_NAME_MANGLE(__malloc_hook))(size_t size) = 120 | &SNMALLOC_NAME_MANGLE(malloc); 121 | SNMALLOC_EXPORT void* (*SNMALLOC_NAME_MANGLE(__realloc_hook))( 122 | void* ptr, size_t size) = &SNMALLOC_NAME_MANGLE(realloc); 123 | SNMALLOC_EXPORT void* (*SNMALLOC_NAME_MANGLE(__memalign_hook))( 124 | size_t alignment, size_t size) = &SNMALLOC_NAME_MANGLE(memalign); 125 | #endif 126 | } 127 | --------------------------------------------------------------------------------