├── .bazelrc ├── .gitignore ├── BUILD ├── CONTRIBUTING.md ├── LICENSE ├── MODULE.bazel ├── README.md ├── WORKSPACE ├── WORKSPACE.bzlmod ├── ci ├── linux_clang-latest_libcxx_bazel.sh ├── linux_clang-latest_libstdcxx_bazel.sh └── linux_gcc-latest_libstdcxx_bazel.sh ├── docs ├── README.md ├── compatibility.md ├── design.md ├── gperftools.md ├── gwp-asan.md ├── images │ ├── legacy_pageheap.png │ ├── lifetimes-counterfactual.png │ ├── lifetimes-enabled.png │ ├── pagemap.png │ ├── per-cpu-cache-internals.png │ ├── per-thread-structure.png │ ├── spanmap.gif │ └── tcmalloc_internals.png ├── lifetime-based-allocator.md ├── mismatched-sized-delete.md ├── overview.md ├── platforms.md ├── quickstart.md ├── reference.md ├── regions-are-not-optional.md ├── rseq.md ├── sampling.md ├── stats.md ├── temeraire.md └── tuning.md └── tcmalloc ├── .clang-format ├── .github ├── CODEOWNERS └── workflows │ └── ci.yml ├── BUILD ├── allocation_sample.cc ├── allocation_sample.h ├── allocation_sample_test.cc ├── allocation_sampling.cc ├── allocation_sampling.h ├── arena.cc ├── arena.h ├── arena_test.cc ├── background.cc ├── central_freelist.cc ├── central_freelist.h ├── central_freelist_benchmark.cc ├── central_freelist_fuzz.cc ├── central_freelist_test.cc ├── common.cc ├── common.h ├── copts.bzl ├── cpu_cache.cc ├── cpu_cache.h ├── cpu_cache_activate_test.cc ├── cpu_cache_test.cc ├── deallocation_profiler.cc ├── deallocation_profiler.h ├── error_reporting.cc ├── error_reporting.h ├── experiment.cc ├── experiment.h ├── experiment_config.h ├── experiment_config_test.cc ├── experiment_fuzz.cc ├── experimental_pow2_size_class.cc ├── global_stats.cc ├── global_stats.h ├── guarded_allocations.h ├── guarded_page_allocator.cc ├── guarded_page_allocator.h ├── guarded_page_allocator_benchmark.cc ├── guarded_page_allocator_profile_test.cc ├── guarded_page_allocator_test.cc ├── hinted_tracker_lists.h ├── huge_address_map.cc ├── huge_address_map.h ├── huge_address_map_test.cc ├── huge_allocator.cc ├── huge_allocator.h ├── huge_allocator_test.cc ├── huge_cache.cc ├── huge_cache.h ├── huge_cache_test.cc ├── huge_page_aware_allocator.cc ├── huge_page_aware_allocator.h ├── huge_page_aware_allocator_fuzz.cc ├── huge_page_aware_allocator_test.cc ├── huge_page_filler.h ├── huge_page_filler_fuzz.cc ├── huge_page_filler_test.cc ├── huge_page_subrelease.h ├── huge_page_subrelease_test.cc ├── huge_pages.h ├── huge_region.h ├── huge_region_fuzz.cc ├── huge_region_test.cc ├── internal ├── BUILD ├── affinity.cc ├── affinity.h ├── affinity_test.cc ├── allocation_guard.cc ├── allocation_guard.h ├── allocation_guard_test.cc ├── atomic_danger.h ├── atomic_stats_counter.h ├── cache_topology.cc ├── cache_topology.h ├── cache_topology_test.cc ├── clock.h ├── config.h ├── config_test.cc ├── cpu_utils.h ├── declarations.h ├── environment.cc ├── environment.h ├── environment_test.cc ├── explicitly_constructed.h ├── exponential_biased.h ├── exponential_biased_test.cc ├── fake_profile.h ├── gwp_asan_state.h ├── hook_list.cc ├── hook_list.h ├── linked_list.h ├── linked_list_benchmark.cc ├── linked_list_test.cc ├── linux_syscall_support.h ├── logging.cc ├── logging.h ├── logging_test.cc ├── logging_test_helper.cc ├── memory_stats.cc ├── memory_stats.h ├── memory_stats_test.cc ├── memory_tag.cc ├── memory_tag.h ├── mincore.cc ├── mincore.h ├── mincore_benchmark.cc ├── mincore_test.cc ├── mock_span.h ├── numa.cc ├── numa.h ├── numa_test.cc ├── optimization.h ├── overflow.h ├── page_size.cc ├── page_size.h ├── pageflags.cc ├── pageflags.h ├── pageflags_test.cc ├── parameter_accessors.h ├── percpu.cc ├── percpu.h ├── percpu_early_test.cc ├── percpu_rseq_aarch64.S ├── percpu_rseq_asm.S ├── percpu_rseq_unsupported.cc ├── percpu_rseq_x86_64.S ├── percpu_tcmalloc.h ├── percpu_tcmalloc_test.cc ├── percpu_test.cc ├── prefetch.h ├── prefetch_test.cc ├── proc_maps.cc ├── proc_maps.h ├── profile.proto ├── profile_builder.cc ├── profile_builder.h ├── profile_builder_fuzz.cc ├── profile_builder_no_tcmalloc_test.cc ├── profile_builder_test.cc ├── range_tracker.h ├── range_tracker_benchmark.cc ├── range_tracker_test.cc ├── residency.cc ├── residency.h ├── residency_test.cc ├── sampled_allocation.h ├── sampled_allocation_recorder.h ├── sampled_allocation_recorder_test.cc ├── sampled_allocation_test.cc ├── stacktrace_filter.h ├── stacktrace_filter_test.cc ├── sysinfo.cc ├── sysinfo.h ├── sysinfo_fuzz.cc ├── sysinfo_test.cc ├── testdata │ ├── b180635896.so │ ├── gnu-property.so │ └── profile_builder_fuzz │ │ ├── clusterfuzz-testcase-minimized-profile_builder_fuzz-5534221534363648 │ │ ├── clusterfuzz-testcase-minimized-profile_builder_fuzz-5647243657216000 │ │ ├── clusterfuzz-testcase-minimized-profile_builder_fuzz-5915530833559552 │ │ ├── clusterfuzz-testcase-minimized-profile_builder_fuzz-6685031907328000.fuzz │ │ └── crash-adc83b19e793491b1c6ea0fd8b46cd9f32e592fc ├── timeseries_tracker.h ├── timeseries_tracker_test.cc ├── util.cc └── util.h ├── internal_malloc_extension.h ├── internal_malloc_tracing_extension.h ├── legacy_size_classes.cc ├── libc_override.h ├── malloc_extension.cc ├── malloc_extension.h ├── malloc_extension_fuzz.cc ├── malloc_hook.cc ├── malloc_hook.h ├── malloc_hook_invoke.h ├── malloc_tracing_extension.cc ├── malloc_tracing_extension.h ├── metadata_allocator.h ├── metadata_object_allocator.h ├── mock_central_freelist.cc ├── mock_central_freelist.h ├── mock_huge_page_static_forwarder.cc ├── mock_huge_page_static_forwarder.h ├── mock_metadata_allocator.h ├── mock_static_forwarder.h ├── mock_transfer_cache.cc ├── mock_transfer_cache.h ├── mock_virtual_allocator.h ├── new_extension.h ├── new_extension_test.cc ├── page_allocator.cc ├── page_allocator.h ├── page_allocator_interface.cc ├── page_allocator_interface.h ├── page_allocator_test.cc ├── page_allocator_test_util.h ├── pagemap.cc ├── pagemap.h ├── pagemap_test.cc ├── pages.h ├── pages_test.cc ├── parameters.cc ├── parameters.h ├── peak_heap_tracker.cc ├── peak_heap_tracker.h ├── profile_marshaler.cc ├── profile_marshaler.h ├── profile_marshaler_test.cc ├── profile_test.cc ├── reuse_size_classes.cc ├── sampler.cc ├── sampler.h ├── segv_handler.cc ├── segv_handler.h ├── segv_handler_test.cc ├── selsan ├── BUILD ├── README.md ├── report_test.cc ├── selsan.cc ├── selsan.h └── shadow_test.cc ├── size_class_info.h ├── size_classes.cc ├── size_classes_test.cc ├── sizemap.cc ├── sizemap.h ├── sizemap_fuzz.cc ├── sizemap_test.cc ├── span.cc ├── span.h ├── span_benchmark.cc ├── span_fuzz.cc ├── span_stats.h ├── span_test.cc ├── stack_trace_table.cc ├── stack_trace_table.h ├── stack_trace_table_test.cc ├── static_vars.cc ├── static_vars.h ├── stats.cc ├── stats.h ├── stats_test.cc ├── system-alloc.cc ├── system-alloc.h ├── tcmalloc.cc ├── tcmalloc.h ├── tcmalloc_policy.h ├── testdata ├── central_freelist_fuzz │ ├── clusterfuzz-testcase-central_freelist_fuzz-6338860943802368 │ └── clusterfuzz-testcase-minimized-central_freelist_fuzz-5328515345809408 ├── clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-4796454007799808 ├── huge_page_aware_allocator_fuzz │ ├── clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-4650158169391104 │ ├── clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-5216394376773632 │ ├── clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-5397442449178624 │ ├── clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-5580915038093312 │ ├── clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-6140744194457600 │ ├── clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-6302517124005888 │ ├── testcase-4507694249082880 │ ├── testcase-4582514590875648 │ ├── testcase-5091122805276672 │ ├── testcase-5185382849773568 │ ├── testcase-5781242586923008 │ ├── testcase-5876893820977152 │ └── testcase-6591694528970752 ├── huge_page_filler_fuzz │ ├── clusterfuzz-testcase-huge_page_filler_fuzz-5476984341004288 │ ├── clusterfuzz-testcase-minimized-huge_page_filler_fuzz-5161409228701696.test │ ├── clusterfuzz-testcase-minimized-huge_page_filler_fuzz-5516474505363456.test │ ├── clusterfuzz-testcase-minimized-huge_page_filler_fuzz-6053674183688192.test │ ├── clusterfuzz-testcase-minimized-huge_page_filler_fuzz-6159120802381824 │ ├── clusterfuzz-testcase-minimized-huge_page_filler_fuzz-6512022070886400.test │ ├── clusterfuzz-testcase-minimized-huge_page_filler_fuzz-6622985612820480 │ ├── crash-869dbc1cdf6a1f79b386adf046c7df32257ef684 │ ├── crash-e9f3aa3ad83e808a5588ec529c6cdf00d5d397fc │ └── testcase-6686265543557120 ├── huge_region_fuzz │ ├── testcase-5235702354214912 │ └── testcase-5618130730156032 ├── malloc_extension_fuzz │ ├── crash-4338e5c59e1bda5104fb5f0aa5553aeb1d3d6465 │ └── crash-c20bfc5c10e885f8e5498c2907cfab82da7c0cff ├── sizemap_fuzz │ └── clusterfuzz-testcase-minimized-sizemap_fuzz-5240920228626432 ├── span_fuzz │ ├── clusterfuzz-testcase-minimized-span_fuzz-6271015625031680 │ ├── clusterfuzz-testcase-minimized-span_fuzz-6321706670620672.fuzz │ ├── crash-01d72a40d5815461b92d3f7c0f6377fd441b0034 │ ├── crash-32697afd59029eb8356fee8ba568e7f6b58d728f │ ├── crash-42b80edf9551d1095aebb6724c070ee43d490125 │ ├── crash-500955af6568b0ed234bd40d6a01af496ba15eb2 │ ├── crash-6ef2b6ae2246d1bda0190983b1007df2699e7738 │ ├── crash-746940d0368bfe3e4a94b60659eeb6cb87106618 │ └── testcase-5877384059617280 └── transfer_cache_fuzz │ └── leak-0b593173f17376c77a3a74a6644af58f77d7a366 ├── testing ├── BUILD ├── aligned_new_test.cc ├── background_test.cc ├── benchmark_main.cc ├── current_allocated_bytes_test.cc ├── deallocation_profiler_test.cc ├── default_parameters_test.cc ├── disable_numa_test.cc ├── fast_path.insecure.golden ├── fast_path.opt.golden ├── fast_path.release+insecure.golden ├── fast_path.release.golden ├── fast_path.unstable.release.golden ├── frag_test.cc ├── get_stats_test.cc ├── heap_profiling_test.cc ├── hello_main.cc ├── hooks_test.cc ├── large_alloc_size_test.cc ├── largesmall_frag_test.cc ├── limit_test.cc ├── malloc_extension_system_malloc_test.cc ├── malloc_extension_test.cc ├── malloc_tracing_extension_test.cc ├── markidle_test.cc ├── memalign_test.cc ├── memory_errors_test.cc ├── no_deps_test.cc ├── numa_locality_test.cc ├── outofmemory_test.cc ├── parallel_test.cc ├── peak_heap_profiling_test.cc ├── profile_drop_frames_test.cc ├── profile_test.cc ├── realized_fragmentation_test.cc ├── realloc_test.cc ├── reclaim_test.cc ├── releasing_test.cc ├── sample_size_class_test.cc ├── sampled_hooks_test.cc ├── sampler_test.cc ├── sampling_memusage_test.cc ├── sampling_test.cc ├── startup_size_test.cc ├── system-alloc_test.cc ├── tcmalloc_benchmark.cc ├── tcmalloc_large_test.cc ├── tcmalloc_test.cc ├── test_allocator_harness.h ├── testutil.cc ├── testutil.h ├── thread_ctor_test.cc ├── thread_ctor_test_lib.cc ├── thread_manager.h ├── threadcachesize_test.cc ├── variants_test.cc ├── want_disable_huge_region_more_often_test_helper.cc ├── want_disable_wider_slabs_test_helper.cc ├── want_hpaa_test_helper.cc └── want_pow2below64_size_classes_helper.cc ├── thread_cache.cc ├── thread_cache.h ├── thread_cache_test.cc ├── transfer_cache.cc ├── transfer_cache.h ├── transfer_cache_benchmark.cc ├── transfer_cache_fuzz.cc ├── transfer_cache_internals.h ├── transfer_cache_stats.h ├── transfer_cache_test.cc ├── variants.bzl ├── want_disable_dynamic_slabs.cc ├── want_disable_huge_region_more_often.cc ├── want_hpaa.cc ├── want_legacy_size_classes.cc ├── want_legacy_size_classes_test.cc └── want_numa_aware.cc /.bazelrc: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TCMalloc Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | build --cxxopt='-std=c++17' 16 | 17 | # Flip incompatible_disallow_empty_glob to avoid breaking forward compatibility, see 18 | # https://github.com/bazelbuild/bazel/pull/15327 for details. 19 | build --incompatible_disallow_empty_glob 20 | 21 | # Flip incompatible_config_setting_private_default_visibility to avoid breaking 22 | # forward compatibility. See https://github.com/bazelbuild/bazel/issues/12933 23 | # for details. 24 | common --incompatible_config_setting_private_default_visibility 25 | 26 | # Link Google Test against Abseil and RE2. 27 | build --define='absl=1' 28 | 29 | # AllocationGuard.CooperativeDeathTest checks for "SIGABRT received" 30 | # message printed by this signal handler. 31 | test --test_env="GTEST_INSTALL_FAILURE_SIGNAL_HANDLER=1" 32 | 33 | # Define the --config=asan-libfuzzer configuration. 34 | build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer 35 | build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer 36 | build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=asan 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Bazel 2 | bazel-* 3 | MODULE.bazel.lock 4 | -------------------------------------------------------------------------------- /BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The TCMalloc Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /MODULE.bazel: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The TCMalloc Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module( 16 | name = "tcmalloc", 17 | version = "0", 18 | compatibility_level = 0, 19 | repo_name = "com_google_tcmalloc", 20 | ) 21 | 22 | bazel_dep(name = "abseil-cpp", version = "20240722.0", repo_name = "com_google_absl") 23 | bazel_dep(name = "bazel_skylib", version = "1.7.1") 24 | bazel_dep(name = "protobuf", version = "27.5", repo_name = "com_google_protobuf") 25 | bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") 26 | bazel_dep(name = "rules_cc", version = "0.0.9") 27 | 28 | bazel_dep(name = "fuzztest", version = "20241028.0", dev_dependency = True, repo_name = "com_google_fuzztest") 29 | bazel_dep(name = "google_benchmark", version = "1.9.2", dev_dependency = True, repo_name = "com_github_google_benchmark") 30 | bazel_dep(name = "googletest", version = "1.13.0", dev_dependency = True, repo_name = "com_google_googletest") 31 | bazel_dep(name = "rules_fuzzing", version = "0.5.2", dev_dependency = True) 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TCMalloc 2 | 3 | This repository contains the TCMalloc C++ code. 4 | 5 | TCMalloc is Google's customized implementation of C's `malloc()` and C++'s 6 | `operator new` used for memory allocation within our C and C++ code. TCMalloc is 7 | a fast, multi-threaded malloc implementation. 8 | 9 | ## Building TCMalloc 10 | 11 | [Bazel](https://bazel.build) is the official build system for TCMalloc. 12 | 13 | The [TCMalloc Platforms Guide](docs/platforms.md) contains information on 14 | platform support for TCMalloc. 15 | 16 | ## Documentation 17 | 18 | All users of TCMalloc should consult the following documentation resources: 19 | 20 | * The [TCMalloc Quickstart](docs/quickstart.md) covers downloading, 21 | installing, building, and testing TCMalloc, including incorporating within 22 | your codebase. 23 | * The [TCMalloc Overview](docs/overview.md) covers the basic architecture of 24 | TCMalloc, and how that may affect configuration choices. 25 | * The [TCMalloc Reference](docs/reference.md) covers the C and C++ TCMalloc 26 | API endpoints. 27 | 28 | More advanced usages of TCMalloc may find the following documentation useful: 29 | 30 | * The [TCMalloc Tuning Guide](docs/tuning.md) covers the configuration 31 | choices in more depth, and also illustrates other ways to customize 32 | TCMalloc. This also covers important operating system-level properties for 33 | improving TCMalloc performance. 34 | * The [TCMalloc Design Doc](docs/design.md) covers how TCMalloc works 35 | underneath the hood, and why certain design choices were made. Most 36 | developers will not need this level of implementation detail. 37 | * The [TCMalloc Compatibility Guide](docs/compatibility.md) which documents 38 | our expectations for how our APIs are used. 39 | 40 | ## License 41 | 42 | The TCMalloc library is licensed under the terms of the Apache license. See 43 | LICENSE for more information. 44 | 45 | Disclaimer: This is not an officially supported Google product. 46 | -------------------------------------------------------------------------------- /WORKSPACE.bzlmod: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The TCMalloc Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # https://bazel.build/external/migration#workspace.bzlmod 16 | # 17 | # This file is intentionally empty. When bzlmod is enabled and this 18 | # file exists, the content of WORKSPACE is ignored. This prevents 19 | # bzlmod builds from unintentionally depending on the WORKSPACE file. 20 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # TCMalloc 2 | 3 | This repository contains the TCMalloc C++ code. 4 | 5 | TCMalloc is Google's customized implementation of C's `malloc()` and C++'s 6 | `operator new` used for memory allocation within our C and C++ code. TCMalloc is 7 | a fast, multi-threaded malloc implementation. 8 | 9 | ## Building TCMalloc 10 | 11 | [Bazel](https://bazel.build) is the official build system for TCMalloc. 12 | 13 | The [TCMalloc Platforms Guide](platforms.md) contains information on platform 14 | support for TCMalloc. 15 | 16 | ## Documentation 17 | 18 | All users of TCMalloc should consult the following documentation resources: 19 | 20 | * The [TCMalloc Quickstart](quickstart.md) covers downloading, installing, 21 | building, and testing TCMalloc, including incorporating within your 22 | codebase. 23 | * The [TCMalloc Overview](overview.md) covers the basic architecture of 24 | TCMalloc, and how that may affect configuration choices. 25 | * The [TCMalloc Reference](reference.md) covers the C and C++ TCMalloc API 26 | endpoints. 27 | 28 | More advanced usages of TCMalloc may find the following documentation useful: 29 | 30 | * The [TCMalloc Tuning Guide](tuning.md) covers the configuration choices in 31 | more depth, and also illustrates other ways to customize TCMalloc. 32 | * The [TCMalloc Design Doc](design.md) covers how TCMalloc works underneath 33 | the hood, and why certain design choices were made. Most developers will not 34 | need this level of implementation detail. 35 | * The [TCMalloc Compatibility Guide](compatibility.md) which documents our 36 | expectations for how our APIs are used. 37 | * The [history and differences](gperftools.md) between this repository and 38 | gperftools. 39 | 40 | ## Publications 41 | 42 | We've published several papers relating to TCMalloc optimizations: 43 | 44 | * ["Beyond malloc efficiency to fleet efficiency: a hugepage-aware memory 45 | allocator" (OSDI 2021)](https://research.google/pubs/pub50370/) relating to 46 | the development and rollout of [Temeraire](temeraire.md), TCMalloc's 47 | hugepage-aware page heap implementation. 48 | * ["Adaptive Hugepage Subrelease for Non-moving Memory Allocators in 49 | Warehouse-Scale Computers" (ISMM 50 | 2021)](https://research.google/pubs/pub50436/) relating to optimizations for 51 | releasing partial hugepages to the operating system. 52 | * ["Characterizing a Memory Allocator at Warehouse Scale" (ASPLOS 2024)](https://research.google/pubs/characterizing-a-memory-allocator-at-warehouse-scale/) 53 | relating to several optimizations developed since 2018. 54 | 55 | ## License 56 | 57 | The TCMalloc library is licensed under the terms of the Apache license. See 58 | LICENSE for more information. 59 | 60 | Disclaimer: This is not an officially supported Google product. 61 | -------------------------------------------------------------------------------- /docs/compatibility.md: -------------------------------------------------------------------------------- 1 | # TCMalloc Compatibility Guidelines 2 | 3 | This document details what we expect from well-behaved users. Any usage of 4 | TCMalloc libraries outside of these technical boundaries may result in breakage 5 | when upgrading to newer versions of TCMalloc. 6 | 7 | Put another way: don't do things that make TCMalloc API maintenance tasks 8 | harder. If you misuse TCMalloc APIs, you're on your own. 9 | 10 | Additionally, because TCMalloc depends on Abseil, Abseil's 11 | [compatibility guidelines](https://abseil.io/about/compatibility) also apply. 12 | 13 | ## What Users Must (And Must Not) Do 14 | 15 | * **Do not depend on a compiled representation of TCMalloc.** We do not 16 | promise any ABI compatibility — we intend for TCMalloc to be built 17 | from source, hopefully from head. The internal layout of our types may 18 | change at any point, without notice. Building TCMalloc in the presence of 19 | different C++ standard library types may change Abseil types, especially for 20 | pre-adopted types (`string_view`, `variant`, etc) — these will become 21 | typedefs and their ABI will change accordingly. 22 | * **Do not rely on dynamic loading/unloading.** TCMalloc does not support 23 | dynamic loading and unloading. 24 | * **You may not open namespace `tcmalloc`.** You are not allowed to define 25 | additional names in namespace `tcmalloc`, nor are you allowed to specialize 26 | anything we provide. 27 | * **You may not depend on the signatures of TCMalloc APIs.** You cannot take 28 | the address of APIs in TCMalloc (that would prevent us from adding overloads 29 | without breaking you). You cannot use metaprogramming tricks to depend on 30 | those signatures either. (This is also similar to the restrictions in the 31 | C++ standard.) 32 | * **You may not forward declare TCMalloc APIs.** This is actually a sub-point 33 | of "do not depend on the signatures of TCMalloc APIs" as well as "do not 34 | open namespace `tcmalloc`", but can be surprising. Any refactoring that 35 | changes template parameters, default parameters, or namespaces will be a 36 | breaking change in the face of forward-declarations. 37 | * **Do not depend upon internal details.** This should go without saying: if 38 | something is in a namespace or filename/path that includes the word 39 | "internal", you are not allowed to depend upon it. It's an implementation 40 | detail. You cannot friend it, you cannot include it, you cannot mention it 41 | or refer to it in any way. 42 | * **Include What You Use.** We may make changes to the internal `#include` 43 | graph for TCMalloc headers - if you use an API, please include the relevant 44 | header file directly. 45 | -------------------------------------------------------------------------------- /docs/images/legacy_pageheap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/docs/images/legacy_pageheap.png -------------------------------------------------------------------------------- /docs/images/lifetimes-counterfactual.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/docs/images/lifetimes-counterfactual.png -------------------------------------------------------------------------------- /docs/images/lifetimes-enabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/docs/images/lifetimes-enabled.png -------------------------------------------------------------------------------- /docs/images/pagemap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/docs/images/pagemap.png -------------------------------------------------------------------------------- /docs/images/per-cpu-cache-internals.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/docs/images/per-cpu-cache-internals.png -------------------------------------------------------------------------------- /docs/images/per-thread-structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/docs/images/per-thread-structure.png -------------------------------------------------------------------------------- /docs/images/spanmap.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/docs/images/spanmap.gif -------------------------------------------------------------------------------- /docs/images/tcmalloc_internals.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/docs/images/tcmalloc_internals.png -------------------------------------------------------------------------------- /docs/platforms.md: -------------------------------------------------------------------------------- 1 | # TCMalloc Platforms 2 | 3 | The TCMalloc code is supported on the following platforms. By "platforms", we 4 | mean the union of operating system, architecture (e.g. little-endian vs. 5 | big-endian), compiler, and standard library. 6 | 7 | ## Language Requirements 8 | 9 | TCMalloc requires a code base that supports C++17 and our code is 10 | C++17-compliant. C code is required to be compliant to C11. 11 | 12 | We guarantee that our code will compile under the following compilation flags: 13 | 14 | Linux: 15 | 16 | * gcc 9.2+, clang 9.0+: `-std=c++17` 17 | 18 | (TL;DR; All code at this time must be built under C++17. We will update this 19 | list if circumstances change.) 20 | 21 | ## Supported Platforms 22 | 23 | The document below lists each platform, broken down by Operating System, 24 | Architecture, Specific Compiler, and Standard Library implementation. 25 | 26 | ### Linux 27 | 28 | **Supported** 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 |
Operating SystemEndianness/Word SizeProcessor ArchitecturesCompilers*Standard Libraries
Linuxlittle-endian, 64-bitx86, AArch64gcc 9.2+
clang 9.0+		libstdc++
libc++
50 | 51 | \* We test on gcc 9.2, though gcc versions (which support C++17) prior to that 52 | release should also work. 53 | 54 | **Best Effort** 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 |
Operating SystemEndianness/Word SizeProcessor ArchitecturesCompilers*Standard Libraries
Linuxlittle-endian, 64-bitPPCgcc 9.2+
clang 9.0+		libstdc++
libc++
76 | -------------------------------------------------------------------------------- /tcmalloc/.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | BasedOnStyle: Google 4 | DerivePointerAlignment: false 5 | PointerAlignment: Left 6 | ... 7 | -------------------------------------------------------------------------------- /tcmalloc/.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Default owners 2 | * @ckennelly 3 | 4 | # Documentation 5 | docs/* @manshreck 6 | -------------------------------------------------------------------------------- /tcmalloc/.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The TCMalloc Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | name: ci 15 | 16 | on: 17 | push: 18 | branches: 19 | - master 20 | 21 | pull_request: 22 | 23 | jobs: 24 | Linux: 25 | runs-on: ubuntu-latest 26 | strategy: 27 | fail-fast: false 28 | matrix: 29 | compiler: 30 | - g++ 31 | - clang++ 32 | 33 | name: "Build/Test ${{matrix.compiler}}" 34 | steps: 35 | - name: Cancel previous 36 | uses: styfle/cancel-workflow-action@0.8.0 37 | with: 38 | access_token: ${{ github.token }} 39 | 40 | - name: Prepare 41 | run: | 42 | sudo apt-get update -qq 43 | sudo apt install -y g++ clang 44 | 45 | - uses: actions/checkout@v2 46 | with: 47 | fetch-depth: 0 48 | 49 | - name: Create Cache Timestamp 50 | id: cache_timestamp 51 | uses: nanzm/get-time-action@v1.1 52 | with: 53 | format: 'YYYY-MM-DD-HH-mm-ss' 54 | 55 | - name: Mount bazel cache 56 | uses: actions/cache@v2 57 | with: 58 | path: "/home/runner/.cache/bazel" 59 | key: bazelcache_${{matrix.compiler}}_${{ steps.cache_timestamp.outputs.time }} 60 | restore-keys: bazelcache_${{matrix.compiler}}_ 61 | 62 | - name: Tests 63 | run: CXX=${{matrix.compiler}} bazel test --test_output=errors //... 64 | -------------------------------------------------------------------------------- /tcmalloc/allocation_sample.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/allocation_sample.h" 16 | 17 | #include 18 | #include 19 | 20 | #include "absl/time/clock.h" 21 | #include "absl/time/time.h" 22 | #include "tcmalloc/internal/config.h" 23 | #include "tcmalloc/internal_malloc_extension.h" 24 | #include "tcmalloc/malloc_extension.h" 25 | #include "tcmalloc/stack_trace_table.h" 26 | 27 | GOOGLE_MALLOC_SECTION_BEGIN 28 | namespace tcmalloc::tcmalloc_internal { 29 | 30 | AllocationSample::AllocationSample(AllocationSampleList* list, absl::Time start) 31 | : list_(list), start_(start) { 32 | mallocs_ = std::make_unique(ProfileType::kAllocations); 33 | mallocs_->SetStartTime(start_); 34 | list->Add(this); 35 | } 36 | 37 | AllocationSample::~AllocationSample() { 38 | if (mallocs_ == nullptr) { 39 | return; 40 | } 41 | 42 | // deleted before ending profile, do it for them 43 | list_->Remove(this); 44 | } 45 | 46 | Profile AllocationSample::Stop() && { 47 | // We need to remove ourselves from list_ before we mutate mallocs_; 48 | // 49 | // A concurrent call to AllocationSampleList::ReportMalloc can access mallocs_ 50 | // until we remove it from list_. 51 | if (mallocs_) { 52 | list_->Remove(this); 53 | mallocs_->SetDuration(absl::Now() - start_); 54 | } 55 | return ProfileAccessor::MakeProfile(std::move(mallocs_)); 56 | } 57 | 58 | } // namespace tcmalloc::tcmalloc_internal 59 | GOOGLE_MALLOC_SECTION_END 60 | -------------------------------------------------------------------------------- /tcmalloc/common.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/common.h" 16 | 17 | #include "tcmalloc/internal/config.h" 18 | #include "tcmalloc/internal/optimization.h" 19 | 20 | GOOGLE_MALLOC_SECTION_BEGIN 21 | namespace tcmalloc { 22 | namespace tcmalloc_internal { 23 | 24 | // This only provides correct answer for TCMalloc-allocated memory, 25 | // and may give a false positive for non-allocated block. 26 | extern "C" bool TCMalloc_Internal_PossiblyCold(const void* ptr) { 27 | return GetMemoryTag(ptr) == MemoryTag::kCold; 28 | } 29 | 30 | } // namespace tcmalloc_internal 31 | } // namespace tcmalloc 32 | GOOGLE_MALLOC_SECTION_END 33 | -------------------------------------------------------------------------------- /tcmalloc/copts.bzl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TCMalloc Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """This package provides default compiler warning flags for the OSS release""" 16 | 17 | TCMALLOC_LLVM_FLAGS = [ 18 | # Ensure TCMalloc itself builds without errors, even if its dependencies 19 | # aren't necessarily -Werror clean. 20 | "-Werror", 21 | "-Wno-deprecated-declarations", 22 | "-Wno-deprecated-volatile", 23 | "-Wno-implicit-int-float-conversion", 24 | "-Wno-sign-compare", 25 | "-Wno-uninitialized", 26 | "-Wno-unused-function", 27 | "-Wno-unused-variable", 28 | ] 29 | 30 | TCMALLOC_GCC_FLAGS = [ 31 | # Ensure TCMalloc itself builds without errors, even if its dependencies 32 | # aren't necessarily -Werror clean. 33 | "-Werror", 34 | "-Wno-array-bounds", 35 | "-Wno-attribute-alias", 36 | "-Wno-deprecated-declarations", 37 | "-Wno-sign-compare", 38 | "-Wno-stringop-overflow", 39 | "-Wno-uninitialized", 40 | "-Wno-unused-function", 41 | # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66425 42 | "-Wno-unused-result", 43 | "-Wno-unused-variable", 44 | ] 45 | 46 | TCMALLOC_DEFAULT_COPTS = select({ 47 | "//tcmalloc:llvm": TCMALLOC_LLVM_FLAGS, 48 | "//conditions:default": TCMALLOC_GCC_FLAGS, 49 | }) 50 | -------------------------------------------------------------------------------- /tcmalloc/cpu_cache.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/cpu_cache.h" 16 | 17 | #include 18 | 19 | #include 20 | #include 21 | 22 | #include "absl/base/attributes.h" 23 | #include "tcmalloc/experiment.h" 24 | #include "tcmalloc/experiment_config.h" 25 | #include "tcmalloc/internal/config.h" 26 | #include "tcmalloc/internal/environment.h" 27 | #include "tcmalloc/internal/logging.h" 28 | #include "tcmalloc/internal/percpu.h" 29 | #include "tcmalloc/internal_malloc_extension.h" 30 | #include "tcmalloc/parameters.h" 31 | #include "tcmalloc/static_vars.h" 32 | 33 | GOOGLE_MALLOC_SECTION_BEGIN 34 | namespace tcmalloc { 35 | namespace tcmalloc_internal { 36 | 37 | static void ActivatePerCpuCaches() { 38 | if (tcmalloc::tcmalloc_internal::tc_globals.CpuCacheActive()) { 39 | // Already active. 40 | return; 41 | } 42 | 43 | if (Parameters::per_cpu_caches() && subtle::percpu::IsFast()) { 44 | tc_globals.InitIfNecessary(); 45 | tc_globals.cpu_cache().Activate(); 46 | tc_globals.ActivateCpuCache(); 47 | // no need for this thread cache anymore, I guess. 48 | ThreadCache::BecomeIdle(); 49 | // If there's a problem with this code, let's notice it right away: 50 | ::operator delete(::operator new(1)); 51 | } 52 | } 53 | 54 | class PerCPUInitializer { 55 | public: 56 | PerCPUInitializer() { 57 | ActivatePerCpuCaches(); 58 | } 59 | }; 60 | static PerCPUInitializer module_enter_exit; 61 | 62 | } // namespace tcmalloc_internal 63 | } // namespace tcmalloc 64 | GOOGLE_MALLOC_SECTION_END 65 | 66 | extern "C" void TCMalloc_Internal_ForceCpuCacheActivation() { 67 | tcmalloc::tcmalloc_internal::ActivatePerCpuCaches(); 68 | } 69 | 70 | extern "C" bool MallocExtension_Internal_GetPerCpuCachesActive() { 71 | return tcmalloc::tcmalloc_internal::tc_globals.CpuCacheActive(); 72 | } 73 | 74 | extern "C" int32_t MallocExtension_Internal_GetMaxPerCpuCacheSize() { 75 | return tcmalloc::tcmalloc_internal::Parameters::max_per_cpu_cache_size(); 76 | } 77 | 78 | extern "C" void MallocExtension_Internal_SetMaxPerCpuCacheSize(int32_t value) { 79 | tcmalloc::tcmalloc_internal::Parameters::set_max_per_cpu_cache_size(value); 80 | } 81 | -------------------------------------------------------------------------------- /tcmalloc/deallocation_profiler.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_DEALLOCATION_PROFILER_H_ 16 | #define TCMALLOC_DEALLOCATION_PROFILER_H_ 17 | 18 | #include 19 | 20 | #include "absl/base/const_init.h" 21 | #include "absl/base/internal/spinlock.h" 22 | #include "absl/time/time.h" 23 | #include "tcmalloc/internal/config.h" 24 | #include "tcmalloc/internal/logging.h" 25 | #include "tcmalloc/malloc_extension.h" 26 | 27 | GOOGLE_MALLOC_SECTION_BEGIN 28 | namespace tcmalloc { 29 | namespace deallocationz { 30 | 31 | class DeallocationProfiler; 32 | 33 | class DeallocationProfilerList { 34 | public: 35 | constexpr DeallocationProfilerList() = default; 36 | 37 | void ReportMalloc(const tcmalloc_internal::StackTrace& stack_trace); 38 | void ReportFree(tcmalloc_internal::AllocHandle handle); 39 | void Add(DeallocationProfiler* profiler); 40 | void Remove(DeallocationProfiler* profiler); 41 | 42 | private: 43 | DeallocationProfiler* first_ = nullptr; 44 | absl::base_internal::SpinLock profilers_lock_{ 45 | absl::kConstInit, absl::base_internal::SCHEDULE_KERNEL_ONLY}; 46 | }; 47 | 48 | class DeallocationSample final 49 | : public tcmalloc_internal::AllocationProfilingTokenBase { 50 | public: 51 | explicit DeallocationSample(DeallocationProfilerList* list); 52 | // We define the dtor to ensure it is placed in the desired text section. 53 | ~DeallocationSample() override; 54 | 55 | tcmalloc::Profile Stop() && override; 56 | 57 | private: 58 | std::unique_ptr profiler_; 59 | }; 60 | 61 | namespace internal { 62 | absl::Duration LifetimeNsToBucketedDuration(double lifetime_ns); 63 | } // namespace internal 64 | } // namespace deallocationz 65 | } // namespace tcmalloc 66 | GOOGLE_MALLOC_SECTION_END 67 | 68 | #endif // TCMALLOC_DEALLOCATION_PROFILER_H_ 69 | -------------------------------------------------------------------------------- /tcmalloc/error_reporting.h: -------------------------------------------------------------------------------- 1 | // Copyright 2025 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_ERROR_REPORTING_H_ 16 | #define TCMALLOC_ERROR_REPORTING_H_ 17 | 18 | #include 19 | 20 | #include "tcmalloc/internal/config.h" 21 | #include "tcmalloc/static_vars.h" 22 | 23 | GOOGLE_MALLOC_SECTION_BEGIN 24 | namespace tcmalloc::tcmalloc_internal { 25 | 26 | [[noreturn]] ABSL_ATTRIBUTE_NOINLINE void ReportMismatchedDelete( 27 | Static& state, const SampledAllocation& alloc, size_t size, 28 | size_t requested_size, std::optional allocated_size); 29 | 30 | [[noreturn]] ABSL_ATTRIBUTE_NOINLINE void ReportMismatchedDelete( 31 | Static& state, void* ptr, size_t size, size_t minimum_size, 32 | size_t maximum_size); 33 | 34 | [[noreturn]] 35 | ABSL_ATTRIBUTE_NOINLINE void ReportMismatchedSizeClass(Static& state, 36 | void* object, 37 | int page_size_class, 38 | int object_size_class); 39 | 40 | [[noreturn]] 41 | ABSL_ATTRIBUTE_NOINLINE void ReportDoubleFree(Static& state, void* ptr); 42 | 43 | [[noreturn]] 44 | ABSL_ATTRIBUTE_NOINLINE void ReportCorruptedFree( 45 | Static& state, std::align_val_t expected_alignment, void* ptr); 46 | 47 | } // namespace tcmalloc::tcmalloc_internal 48 | GOOGLE_MALLOC_SECTION_END 49 | 50 | #endif // TCMALLOC_ERROR_REPORTING_H_ 51 | -------------------------------------------------------------------------------- /tcmalloc/experiment.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_EXPERIMENT_H_ 16 | #define TCMALLOC_EXPERIMENT_H_ 17 | 18 | #include 19 | 20 | #include 21 | 22 | #include "absl/functional/function_ref.h" 23 | #include "absl/strings/string_view.h" 24 | #include "absl/types/optional.h" 25 | #include "tcmalloc/experiment_config.h" 26 | #include "tcmalloc/internal/config.h" 27 | 28 | // TCMalloc Experiment Controller 29 | // 30 | // This consumes environment variables to decide whether to activate experiments 31 | // to control TCMalloc behavior. It avoids memory allocations when making 32 | // experiment decisions to allow experiments to be used in critical TCMalloc 33 | // initialization paths. 34 | // 35 | // If an experiment is causing difficulty, all experiments can be disabled by 36 | // setting the environment variable: 37 | // BORG_DISABLE_EXPERIMENTS=all *or* 38 | // BORG_DISABLE_EXPERIMENTS=BAD_EXPERIMENT_LABEL 39 | 40 | GOOGLE_MALLOC_SECTION_BEGIN 41 | namespace tcmalloc { 42 | namespace tcmalloc_internal { 43 | 44 | constexpr size_t kNumExperiments = 45 | static_cast(Experiment::kMaxExperimentID); 46 | 47 | // SelectExperiments parses the experiments enumerated by active and disabled 48 | // and updates buffer[experiment_id] accordingly. 49 | // 50 | // buffer must be sized for kMaxExperimentID entries. 51 | // 52 | // This is exposed for testing purposes only. 53 | const bool* SelectExperiments(bool* buffer, absl::string_view test_target, 54 | absl::string_view active, 55 | absl::string_view disabled, 56 | bool unset); 57 | 58 | } // namespace tcmalloc_internal 59 | 60 | bool IsExperimentActive(Experiment exp); 61 | 62 | std::optional FindExperimentByName(absl::string_view name); 63 | 64 | void WalkExperiments( 65 | absl::FunctionRef callback); 66 | 67 | } // namespace tcmalloc 68 | GOOGLE_MALLOC_SECTION_END 69 | 70 | #endif // TCMALLOC_EXPERIMENT_H_ 71 | -------------------------------------------------------------------------------- /tcmalloc/experiment_config.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_EXPERIMENT_CONFIG_H_ 16 | #define TCMALLOC_EXPERIMENT_CONFIG_H_ 17 | 18 | #include "absl/strings/string_view.h" 19 | 20 | // Autogenerated by experiments_proto_test --experiments_generate_config=true 21 | namespace tcmalloc { 22 | 23 | enum class Experiment : int { 24 | // clang-format off 25 | // go/keep-sorted start 26 | TCMALLOC_MIN_HOT_ACCESS_HINT_ABLATION, // TODO(b/376902157): Complete experiment. 27 | TCMALLOC_USERMODE_HUGEPAGE_COLLAPSE, // TODO(b/287498389): Complete experiment. 28 | TEST_ONLY_L3_AWARE, // TODO(b/239977380): Complete experiment. 29 | TEST_ONLY_TCMALLOC_COARSE_LFR_TRACKERS, // TODO(b/368054442): Complete experiment. 30 | TEST_ONLY_TCMALLOC_HUGE_CACHE_RELEASE_30S, // TODO(b/319872040): Complete experiment. 31 | TEST_ONLY_TCMALLOC_POW2_SIZECLASS, 32 | TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE, 33 | // go/keep-sorted end 34 | kMaxExperimentID, 35 | // clang-format on 36 | }; 37 | 38 | struct ExperimentConfig { 39 | Experiment id; 40 | absl::string_view name; 41 | }; 42 | 43 | // clang-format off 44 | inline constexpr ExperimentConfig experiments[] = { 45 | // go/keep-sorted start 46 | {Experiment::TCMALLOC_MIN_HOT_ACCESS_HINT_ABLATION, "TCMALLOC_MIN_HOT_ACCESS_HINT_ABLATION"}, 47 | {Experiment::TCMALLOC_USERMODE_HUGEPAGE_COLLAPSE, "TCMALLOC_USERMODE_HUGEPAGE_COLLAPSE"}, 48 | {Experiment::TEST_ONLY_L3_AWARE, "TEST_ONLY_L3_AWARE"}, 49 | {Experiment::TEST_ONLY_TCMALLOC_COARSE_LFR_TRACKERS, "TEST_ONLY_TCMALLOC_COARSE_LFR_TRACKERS"}, 50 | {Experiment::TEST_ONLY_TCMALLOC_HUGE_CACHE_RELEASE_30S, "TEST_ONLY_TCMALLOC_HUGE_CACHE_RELEASE_30S"}, 51 | {Experiment::TEST_ONLY_TCMALLOC_POW2_SIZECLASS, "TEST_ONLY_TCMALLOC_POW2_SIZECLASS"}, 52 | {Experiment::TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE, "TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE"}, 53 | // go/keep-sorted end 54 | }; 55 | // clang-format on 56 | 57 | } // namespace tcmalloc 58 | 59 | #endif // TCMALLOC_EXPERIMENT_CONFIG_H_ 60 | -------------------------------------------------------------------------------- /tcmalloc/experiment_config_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/experiment_config.h" 16 | 17 | #include "gtest/gtest.h" 18 | 19 | namespace tcmalloc { 20 | namespace { 21 | 22 | // Verify IDs are non-negative and strictly less than kMaxExperimentID. 23 | TEST(ExperimentConfigTest, ValidateIDs) { 24 | for (const auto& exp : experiments) { 25 | ASSERT_LE(0, static_cast(exp.id)); 26 | ASSERT_LT(exp.id, Experiment::kMaxExperimentID); 27 | } 28 | } 29 | 30 | } // namespace 31 | } // namespace tcmalloc 32 | -------------------------------------------------------------------------------- /tcmalloc/experiment_fuzz.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "gtest/gtest.h" 19 | #include "fuzztest/fuzztest.h" 20 | #include "absl/strings/string_view.h" 21 | #include "tcmalloc/experiment.h" 22 | 23 | namespace tcmalloc::tcmalloc_internal { 24 | namespace { 25 | 26 | void FuzzSelectExperiments(absl::string_view test_target, 27 | absl::string_view active, absl::string_view disabled, 28 | bool unset) { 29 | if (unset && !test_target.empty() && (!active.empty() || !disabled.empty())) { 30 | return; 31 | } 32 | 33 | bool buffer[tcmalloc::tcmalloc_internal::kNumExperiments]; 34 | 35 | SelectExperiments(buffer, test_target, active, disabled, 36 | unset); 37 | } 38 | 39 | FUZZ_TEST(ExperimentTest, FuzzSelectExperiments); 40 | 41 | TEST(ExperimentTest, FuzzSelectExperiments_b395212979) { 42 | FuzzSelectExperiments( 43 | "t_fuenchmark&" 44 | "vvvvvvvvvvvvvvvvvvVvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv", 45 | "", "", 46 | true); 47 | } 48 | 49 | } // namespace 50 | } // namespace tcmalloc::tcmalloc_internal 51 | -------------------------------------------------------------------------------- /tcmalloc/guarded_allocations.h: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_GUARDED_ALLOCATIONS_H_ 16 | #define TCMALLOC_GUARDED_ALLOCATIONS_H_ 17 | 18 | #include 19 | 20 | #include "tcmalloc/internal/config.h" 21 | #include "tcmalloc/internal/logging.h" 22 | #include "tcmalloc/malloc_extension.h" 23 | 24 | GOOGLE_MALLOC_SECTION_BEGIN 25 | namespace tcmalloc { 26 | namespace tcmalloc_internal { 27 | 28 | struct GuardedAllocationsStackTrace { 29 | void* stack[kMaxStackDepth]; 30 | size_t depth = 0; 31 | pid_t thread_id = 0; 32 | }; 33 | 34 | enum class WriteFlag : int { Unknown, Read, Write }; 35 | 36 | enum class GuardedAllocationsErrorType { 37 | kUseAfterFree, 38 | kUseAfterFreeRead, 39 | kUseAfterFreeWrite, 40 | kBufferUnderflow, 41 | kBufferUnderflowRead, 42 | kBufferUnderflowWrite, 43 | kBufferOverflow, 44 | kBufferOverflowRead, 45 | kBufferOverflowWrite, 46 | kDoubleFree, 47 | kBufferOverflowOnDealloc, 48 | kUnknown, 49 | }; 50 | 51 | struct GuardedAllocWithStatus { 52 | void* alloc = nullptr; 53 | Profile::Sample::GuardedStatus status; 54 | }; 55 | 56 | } // namespace tcmalloc_internal 57 | } // namespace tcmalloc 58 | GOOGLE_MALLOC_SECTION_END 59 | 60 | #endif // TCMALLOC_GUARDED_ALLOCATIONS_H_ 61 | -------------------------------------------------------------------------------- /tcmalloc/huge_address_map_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/huge_address_map.h" 16 | 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | #include "gmock/gmock.h" 23 | #include "gtest/gtest.h" 24 | #include "tcmalloc/mock_metadata_allocator.h" 25 | 26 | namespace tcmalloc { 27 | namespace tcmalloc_internal { 28 | namespace { 29 | 30 | class HugeAddressMapTest : public ::testing::Test { 31 | protected: 32 | HugeAddressMapTest() : map_(malloc_metadata_) {} 33 | 34 | std::vector Contents() { 35 | std::vector ret; 36 | auto node = map_.first(); 37 | while (node) { 38 | ret.push_back(node->range()); 39 | node = node->next(); 40 | } 41 | 42 | return ret; 43 | } 44 | 45 | HugePage hp(size_t i) { return {i}; } 46 | HugeLength hl(size_t i) { return NHugePages(i); } 47 | 48 | HugeAddressMap map_; 49 | 50 | private: 51 | FakeMetadataAllocator malloc_metadata_; 52 | }; 53 | 54 | // This test verifies that HugeAddressMap merges properly. 55 | TEST_F(HugeAddressMapTest, Merging) { 56 | const HugeRange r1 = HugeRange::Make(hp(0), hl(1)); 57 | const HugeRange r2 = HugeRange::Make(hp(1), hl(1)); 58 | const HugeRange r3 = HugeRange::Make(hp(2), hl(1)); 59 | const HugeRange all = Join(r1, Join(r2, r3)); 60 | map_.Insert(r1); 61 | map_.Check(); 62 | EXPECT_THAT(Contents(), testing::ElementsAre(r1)); 63 | map_.Insert(r3); 64 | map_.Check(); 65 | EXPECT_THAT(Contents(), testing::ElementsAre(r1, r3)); 66 | map_.Insert(r2); 67 | map_.Check(); 68 | EXPECT_THAT(Contents(), testing::ElementsAre(all)); 69 | } 70 | 71 | } // namespace 72 | } // namespace tcmalloc_internal 73 | } // namespace tcmalloc 74 | -------------------------------------------------------------------------------- /tcmalloc/internal/affinity.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_AFFINITY_H_ 16 | #define TCMALLOC_INTERNAL_AFFINITY_H_ 17 | 18 | #include 19 | 20 | #include 21 | 22 | #include "absl/types/span.h" 23 | #include "tcmalloc/internal/config.h" 24 | #include "tcmalloc/internal/cpu_utils.h" 25 | 26 | GOOGLE_MALLOC_SECTION_BEGIN 27 | namespace tcmalloc { 28 | namespace tcmalloc_internal { 29 | 30 | // Affinity helpers. 31 | 32 | // Returns a vector of the which cpus the currently allowed thread is allowed to 33 | // run on. There are no guarantees that this will not change before, after, or 34 | // even during, the call to AllowedCpus(). 35 | std::vector AllowedCpus(); 36 | 37 | // Enacts a scoped affinity mask on the constructing thread. Attempts to 38 | // restore the original affinity mask on destruction. 39 | // 40 | // REQUIRES: For test-use only. Do not use this in production code. 41 | class ScopedAffinityMask { 42 | public: 43 | // When racing with an external restriction that has a zero-intersection with 44 | // "allowed_cpus" we will construct, but immediately register as "Tampered()", 45 | // without actual changes to affinity. 46 | explicit ScopedAffinityMask(absl::Span allowed_cpus); 47 | explicit ScopedAffinityMask(int allowed_cpu); 48 | 49 | // Restores original affinity iff our scoped affinity has not been externally 50 | // modified (i.e. Tampered()). Otherwise, the updated affinity is preserved. 51 | ~ScopedAffinityMask(); 52 | 53 | // Returns true if the affinity mask no longer matches what was set at point 54 | // of construction. 55 | // 56 | // Note: This is instantaneous and not fool-proof. It's possible for an 57 | // external affinity modification to subsequently align with our originally 58 | // specified "allowed_cpus". In this case Tampered() will return false when 59 | // time may have been spent executing previously on non-specified cpus. 60 | bool Tampered(); 61 | 62 | private: 63 | CpuSet original_cpus_, specified_cpus_; 64 | }; 65 | 66 | } // namespace tcmalloc_internal 67 | } // namespace tcmalloc 68 | GOOGLE_MALLOC_SECTION_END 69 | 70 | #endif // TCMALLOC_INTERNAL_AFFINITY_H_ 71 | -------------------------------------------------------------------------------- /tcmalloc/internal/allocation_guard.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/internal/allocation_guard.h" 16 | 17 | #include "absl/base/attributes.h" 18 | #include "tcmalloc/internal/config.h" 19 | 20 | GOOGLE_MALLOC_SECTION_BEGIN 21 | namespace tcmalloc::tcmalloc_internal { 22 | 23 | } // namespace tcmalloc::tcmalloc_internal 24 | GOOGLE_MALLOC_SECTION_END 25 | -------------------------------------------------------------------------------- /tcmalloc/internal/allocation_guard.h: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_ALLOCATION_GUARD_H_ 16 | #define TCMALLOC_INTERNAL_ALLOCATION_GUARD_H_ 17 | 18 | #include "absl/base/attributes.h" 19 | #include "absl/base/internal/spinlock.h" 20 | #include "absl/base/thread_annotations.h" 21 | #include "tcmalloc/internal/config.h" 22 | 23 | GOOGLE_MALLOC_SECTION_BEGIN 24 | namespace tcmalloc::tcmalloc_internal { 25 | 26 | // TODO(b/143069684): actually ensure no allocations in debug mode here. 27 | struct AllocationGuard { 28 | AllocationGuard() {} 29 | }; 30 | 31 | // A SpinLockHolder that also enforces no allocations while the lock is held in 32 | // debug mode. 33 | class ABSL_SCOPED_LOCKABLE AllocationGuardSpinLockHolder { 34 | public: 35 | explicit AllocationGuardSpinLockHolder(absl::base_internal::SpinLock* l) 36 | ABSL_EXCLUSIVE_LOCK_FUNCTION(l) 37 | : lock_holder_(l) { 38 | #ifndef NDEBUG 39 | if (l->IsCooperative()) { 40 | abort(); 41 | } 42 | #endif // NDEBUG 43 | } 44 | 45 | inline ~AllocationGuardSpinLockHolder() ABSL_UNLOCK_FUNCTION() = default; 46 | 47 | private: 48 | absl::base_internal::SpinLockHolder lock_holder_; 49 | // In debug mode, enforces no allocations. 50 | AllocationGuard enforce_no_alloc_; 51 | }; 52 | 53 | } // namespace tcmalloc::tcmalloc_internal 54 | GOOGLE_MALLOC_SECTION_END 55 | 56 | #endif // TCMALLOC_INTERNAL_ALLOCATION_GUARD_H_ 57 | -------------------------------------------------------------------------------- /tcmalloc/internal/allocation_guard_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/internal/allocation_guard.h" 16 | 17 | #include 18 | 19 | #include "gtest/gtest.h" 20 | #include "absl/base/const_init.h" 21 | #include "absl/base/internal/spinlock.h" 22 | 23 | namespace tcmalloc::tcmalloc_internal { 24 | namespace { 25 | 26 | TEST(AllocationGuard, Noncooperative) { 27 | absl::base_internal::SpinLock lock(absl::kConstInit, 28 | absl::base_internal::SCHEDULE_KERNEL_ONLY); 29 | AllocationGuardSpinLockHolder h(&lock); 30 | } 31 | 32 | TEST(AllocationGuard, CooperativeDeathTest) { 33 | absl::base_internal::SpinLock lock; 34 | 35 | EXPECT_DEBUG_DEATH( 36 | { AllocationGuardSpinLockHolder h(&lock); }, "SIGABRT received"); 37 | } 38 | 39 | } // namespace 40 | } // namespace tcmalloc::tcmalloc_internal 41 | -------------------------------------------------------------------------------- /tcmalloc/internal/atomic_danger.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // The routines exported by this module are subtle and dangerous. 16 | 17 | #ifndef TCMALLOC_INTERNAL_ATOMIC_DANGER_H_ 18 | #define TCMALLOC_INTERNAL_ATOMIC_DANGER_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include "tcmalloc/internal/config.h" 25 | 26 | GOOGLE_MALLOC_SECTION_BEGIN 27 | namespace tcmalloc { 28 | namespace tcmalloc_internal { 29 | namespace atomic_danger { 30 | 31 | // Casts the address of a std::atomic to the address of an IntType. 32 | // 33 | // This is almost certainly not the function you are looking for! It is 34 | // undefined behavior, as the object under a std::atomic isn't 35 | // fundamentally an int. This function is intended for passing the address of an 36 | // atomic integer to syscalls or for assembly interpretation. 37 | // 38 | // Callers should be migrated if C++ standardizes a better way to do this: 39 | // * http://wg21.link/n4013 (Atomic operations on non-atomic data) 40 | // * http://wg21.link/p0019 (Atomic Ref, merged into C++20) 41 | // * http://wg21.link/p1478 (Byte-wise atomic memcpy) 42 | template 43 | IntType* CastToIntegral(std::atomic* atomic_for_syscall) { 44 | static_assert(std::is_integral::value, 45 | "CastToIntegral must be instantiated with an integral type."); 46 | #if __cpp_lib_atomic_is_always_lock_free >= 201603 47 | static_assert(std::atomic::is_always_lock_free, 48 | "CastToIntegral must be instantiated with a lock-free type."); 49 | #else 50 | static_assert(__atomic_always_lock_free(sizeof(IntType), 51 | nullptr /* typical alignment */), 52 | "CastToIntegral must be instantiated with a lock-free type."); 53 | #endif 54 | return reinterpret_cast(atomic_for_syscall); 55 | } 56 | } // namespace atomic_danger 57 | } // namespace tcmalloc_internal 58 | } // namespace tcmalloc 59 | GOOGLE_MALLOC_SECTION_END 60 | 61 | #endif // TCMALLOC_INTERNAL_ATOMIC_DANGER_H_ 62 | -------------------------------------------------------------------------------- /tcmalloc/internal/cache_topology.h: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_CACHE_TOPOLOGY_H_ 16 | #define TCMALLOC_INTERNAL_CACHE_TOPOLOGY_H_ 17 | 18 | #include 19 | 20 | #include "absl/base/attributes.h" 21 | #include "absl/strings/string_view.h" 22 | #include "tcmalloc/internal/config.h" 23 | #include "tcmalloc/internal/cpu_utils.h" 24 | #include "tcmalloc/internal/logging.h" 25 | 26 | GOOGLE_MALLOC_SECTION_BEGIN 27 | namespace tcmalloc { 28 | namespace tcmalloc_internal { 29 | 30 | class CacheTopology { 31 | public: 32 | static CacheTopology& Instance() { 33 | ABSL_CONST_INIT static CacheTopology instance; 34 | return instance; 35 | } 36 | 37 | constexpr CacheTopology() = default; 38 | 39 | void Init(); 40 | 41 | unsigned l3_count() const { return l3_count_; } 42 | 43 | unsigned GetL3FromCpuId(int cpu) const { 44 | TC_ASSERT_GE(cpu, 0); 45 | TC_ASSERT_LT(cpu, cpu_count_); 46 | unsigned l3 = l3_cache_index_[cpu]; 47 | TC_ASSERT_LT(l3, l3_count_); 48 | return l3; 49 | } 50 | 51 | private: 52 | unsigned cpu_count_ = 0; 53 | unsigned l3_count_ = 0; 54 | uint8_t l3_cache_index_[kMaxCpus] = {}; 55 | }; 56 | 57 | } // namespace tcmalloc_internal 58 | } // namespace tcmalloc 59 | GOOGLE_MALLOC_SECTION_END 60 | 61 | #endif // TCMALLOC_INTERNAL_CACHE_TOPOLOGY_H_ 62 | -------------------------------------------------------------------------------- /tcmalloc/internal/cache_topology_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/internal/cache_topology.h" 16 | 17 | #include "gtest/gtest.h" 18 | #include "tcmalloc/internal/sysinfo.h" 19 | 20 | namespace tcmalloc::tcmalloc_internal { 21 | namespace { 22 | 23 | TEST(CacheTopology, ComputesSomethingReasonable) { 24 | // This test verifies that each L3 cache serves the same number of CPU. This 25 | // is not a strict requirement for the correct operation of this code, but a 26 | // sign of sanity. 27 | CacheTopology topology; 28 | topology.Init(); 29 | EXPECT_EQ(NumCPUs() % topology.l3_count(), 0); 30 | ASSERT_GT(topology.l3_count(), 0); 31 | static const int kMaxNodes = 256 / 8; 32 | int count_per_node[kMaxNodes] = {0}; 33 | for (int i = 0, n = NumCPUs(); i < n; ++i) { 34 | count_per_node[topology.GetL3FromCpuId(i)]++; 35 | } 36 | for (int i = 0; i < topology.l3_count(); ++i) { 37 | EXPECT_EQ(count_per_node[i], NumCPUs() / topology.l3_count()); 38 | } 39 | } 40 | 41 | } // namespace 42 | } // namespace tcmalloc::tcmalloc_internal 43 | -------------------------------------------------------------------------------- /tcmalloc/internal/clock.h: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_CLOCK_H_ 16 | #define TCMALLOC_INTERNAL_CLOCK_H_ 17 | 18 | #include 19 | 20 | #include "tcmalloc/internal/config.h" 21 | 22 | GOOGLE_MALLOC_SECTION_BEGIN 23 | namespace tcmalloc { 24 | namespace tcmalloc_internal { 25 | 26 | // Represents an abstract clock. The now and freq functions are analogous to 27 | // CycleClock::Now and CycleClock::Frequency, which will be the most commonly 28 | // used implementations. Tests can use this interface to mock out the clock. 29 | struct Clock { 30 | // Returns the current time in ticks (relative to an arbitrary time base). 31 | int64_t (*now)(); 32 | 33 | // Returns the number of ticks per second. 34 | double (*freq)(); 35 | }; 36 | 37 | } // namespace tcmalloc_internal 38 | } // namespace tcmalloc 39 | GOOGLE_MALLOC_SECTION_END 40 | 41 | #endif // TCMALLOC_INTERNAL_CLOCK_H_ 42 | -------------------------------------------------------------------------------- /tcmalloc/internal/declarations.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // These declarations are for internal use, allowing us to have access to 16 | // allocation functions whose declarations are not provided by the standard 17 | // library. 18 | #ifndef TCMALLOC_INTERNAL_DECLARATIONS_H_ 19 | #define TCMALLOC_INTERNAL_DECLARATIONS_H_ 20 | 21 | #if !defined(__cpp_sized_deallocation) || !defined(__cpp_aligned_new) 22 | #include 23 | #include 24 | #endif // !defined(__cpp_sized_deallocation) || !defined(__cpp_aligned_new) 25 | 26 | #if !defined(__cpp_sized_deallocation) 27 | 28 | void operator delete(void*, std::size_t) noexcept; 29 | void operator delete[](void*, std::size_t) noexcept; 30 | 31 | #endif // !defined(__cpp_sized_deallocation) 32 | 33 | #if !defined(__cpp_aligned_new) 34 | 35 | namespace std { 36 | enum class align_val_t : size_t; 37 | } // namespace std 38 | 39 | void* operator new(std::size_t, std::align_val_t); 40 | void* operator new(std::size_t, std::align_val_t, 41 | const std::nothrow_t&) noexcept; 42 | void* operator new[](std::size_t, std::align_val_t); 43 | void* operator new[](std::size_t, std::align_val_t, 44 | const std::nothrow_t&) noexcept; 45 | 46 | void operator delete(void*, std::align_val_t) noexcept; 47 | void operator delete[](void*, std::align_val_t) noexcept; 48 | 49 | #endif // !defined(__cpp_aligned_new) 50 | 51 | #if !defined(__cpp_sized_deallocation) || !defined(__cpp_aligned_new) 52 | 53 | void operator delete(void*, std::size_t, std::align_val_t) noexcept; 54 | void operator delete[](void*, std::size_t, std::align_val_t) noexcept; 55 | 56 | #endif // !defined(__cpp_sized_deallocation) || !defined(__cpp_aligned_new) 57 | 58 | #endif // TCMALLOC_INTERNAL_DECLARATIONS_H_ 59 | -------------------------------------------------------------------------------- /tcmalloc/internal/environment.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include "tcmalloc/internal/environment.h" 15 | 16 | #include 17 | 18 | #include "absl/base/attributes.h" 19 | #include "tcmalloc/internal/config.h" 20 | 21 | GOOGLE_MALLOC_SECTION_BEGIN 22 | namespace tcmalloc { 23 | namespace tcmalloc_internal { 24 | 25 | #ifdef __linux__ 26 | // POSIX provides the **environ array which contains environment variables in a 27 | // linear array, terminated by a NULL string. This array is only perturbed when 28 | // the environment is changed (which is inherently unsafe) so it's safe to 29 | // return a const pointer into it. 30 | // e.g. { "SHELL=/bin/bash", "MY_ENV_VAR=1", "" } 31 | extern "C" char** environ; 32 | ABSL_ATTRIBUTE_WEAK const char* thread_safe_getenv(const char* env_var) { 33 | int var_len = strlen(env_var); 34 | 35 | char** envv = environ; 36 | if (!envv) { 37 | return nullptr; 38 | } 39 | 40 | for (; *envv != nullptr; envv++) 41 | if (strncmp(*envv, env_var, var_len) == 0 && (*envv)[var_len] == '=') 42 | return *envv + var_len + 1; // skip over the '=' 43 | 44 | return nullptr; 45 | } 46 | #else 47 | const char* thread_safe_getenv(const char* env_var) { return nullptr; } 48 | #endif 49 | 50 | } // namespace tcmalloc_internal 51 | } // namespace tcmalloc 52 | GOOGLE_MALLOC_SECTION_END 53 | -------------------------------------------------------------------------------- /tcmalloc/internal/environment.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_ENVIRONMENT_H_ 16 | #define TCMALLOC_INTERNAL_ENVIRONMENT_H_ 17 | 18 | #include "tcmalloc/internal/config.h" 19 | 20 | GOOGLE_MALLOC_SECTION_BEGIN 21 | namespace tcmalloc { 22 | namespace tcmalloc_internal { 23 | 24 | // WARNING ******************************************************************** 25 | // getenv(2) can only be safely used in the absence of calls which perturb the 26 | // environment (e.g. putenv/setenv/clearenv). The use of such calls is 27 | // strictly thread-hostile since these calls do *NOT* synchronize and there is 28 | // *NO* thread-safe way in which the POSIX **environ array may be queried about 29 | // modification. 30 | // **************************************************************************** 31 | // The default getenv(2) is not guaranteed to be thread-safe as there are no 32 | // semantics specifying the implementation of the result buffer. The result 33 | // from thread_safe_getenv() may be safely queried in a multi-threaded context. 34 | // If you have explicit synchronization with changes environment variables then 35 | // any copies of the returned pointer must be invalidated across modification. 36 | const char* thread_safe_getenv(const char* env_var); 37 | 38 | } // namespace tcmalloc_internal 39 | } // namespace tcmalloc 40 | GOOGLE_MALLOC_SECTION_END 41 | 42 | #endif // TCMALLOC_INTERNAL_ENVIRONMENT_H_ 43 | -------------------------------------------------------------------------------- /tcmalloc/internal/environment_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/internal/environment.h" 16 | 17 | #include 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | 22 | namespace tcmalloc { 23 | namespace tcmalloc_internal { 24 | namespace { 25 | 26 | TEST(EnvironmentTest, thread_safe_getenv) { 27 | // Should never be defined at test start 28 | const char *result, *undefined_env_var = "UTIL_TEST_UNDEFINED_ENV_VAR"; 29 | 30 | // Check that we handle an undefined variable and then set it 31 | ASSERT_TRUE(getenv(undefined_env_var) == nullptr); 32 | ASSERT_TRUE(thread_safe_getenv(undefined_env_var) == nullptr); 33 | ASSERT_EQ(setenv(undefined_env_var, "1234567890", 0), 0); 34 | ASSERT_TRUE(getenv(undefined_env_var) != nullptr); 35 | 36 | // Make sure we can find the new variable 37 | result = thread_safe_getenv(undefined_env_var); 38 | ASSERT_TRUE(result != nullptr); 39 | // ... and that it matches what was set 40 | EXPECT_EQ(strcmp(result, getenv(undefined_env_var)), 0); 41 | } 42 | 43 | } // namespace 44 | } // namespace tcmalloc_internal 45 | } // namespace tcmalloc 46 | -------------------------------------------------------------------------------- /tcmalloc/internal/explicitly_constructed.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_EXPLICITLY_CONSTRUCTED_H_ 16 | #define TCMALLOC_INTERNAL_EXPLICITLY_CONSTRUCTED_H_ 17 | 18 | #include 19 | 20 | #include 21 | 22 | #include "tcmalloc/internal/config.h" 23 | 24 | GOOGLE_MALLOC_SECTION_BEGIN 25 | namespace tcmalloc { 26 | namespace tcmalloc_internal { 27 | 28 | // Wraps a variable whose constructor is explicitly called. It is particularly 29 | // useful for a global variable, without its constructor and destructor run on 30 | // start and end of the program lifetime. This circumvents the initial 31 | // construction order fiasco, while keeping the address of the empty string a 32 | // compile time constant. 33 | // 34 | // Pay special attention to the initialization state of the object. 35 | // 1. The object is "uninitialized" to begin with. 36 | // 2. Call Construct() only if the object is uninitialized. After the call, the 37 | // object becomes "initialized". 38 | // 3. Call get_mutable() only if the object is initialized. 39 | template 40 | class ExplicitlyConstructed { 41 | public: 42 | template 43 | void Construct(Args&&... args) { 44 | new (&union_) T(std::forward(args)...); 45 | } 46 | 47 | T& get_mutable() { return reinterpret_cast(union_); } 48 | 49 | private: 50 | union AlignedUnion { 51 | constexpr AlignedUnion() = default; 52 | alignas(T) char space[sizeof(T)]; 53 | int64_t align_to_int64; 54 | void* align_to_ptr; 55 | } union_; 56 | }; 57 | 58 | } // namespace tcmalloc_internal 59 | } // namespace tcmalloc 60 | GOOGLE_MALLOC_SECTION_END 61 | 62 | #endif // TCMALLOC_INTERNAL_EXPLICITLY_CONSTRUCTED_H_ 63 | -------------------------------------------------------------------------------- /tcmalloc/internal/exponential_biased.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_EXPONENTIAL_BIASED_H_ 16 | #define TCMALLOC_INTERNAL_EXPONENTIAL_BIASED_H_ 17 | 18 | #include 19 | #include 20 | 21 | namespace tcmalloc { 22 | namespace tcmalloc_internal { 23 | 24 | class ExponentialBiased { 25 | public: 26 | static uint64_t NextRandom(uint64_t rnd); 27 | static uint32_t GetRandom(uint64_t rnd); 28 | }; 29 | 30 | // Returns the next prng value. 31 | // pRNG is: aX+b mod c with a = 0x5DEECE66D, b = 0xB, c = 1<<48 32 | // This is the lrand64 generator. 33 | inline uint64_t ExponentialBiased::NextRandom(uint64_t rnd) { 34 | const uint64_t prng_mult = UINT64_C(0x5DEECE66D); 35 | const uint64_t prng_add = 0xB; 36 | const uint64_t prng_mod_power = 48; 37 | const uint64_t prng_mod_mask = 38 | ~((~static_cast(0)) << prng_mod_power); 39 | return (prng_mult * rnd + prng_add) & prng_mod_mask; 40 | } 41 | 42 | // Extracts higher-quality random bits. 43 | // The raw value returned from NextRandom has poor randomness low bits 44 | // and is not directly suitable for things like 'if (rnd % 2)'. 45 | inline uint32_t ExponentialBiased::GetRandom(uint64_t rnd) { return rnd >> 16; } 46 | 47 | // Convenience wrapper to initialize a seed and return a sequence of 48 | // pseudo-random values. Thread-safety: thread safe. 49 | class Random { 50 | public: 51 | constexpr explicit Random(uint64_t seed) : state_(seed) {} 52 | 53 | // Return the next pseudo-random value. 54 | uint32_t Next(); 55 | 56 | // Reset internal state with provided seed. 57 | void Reset(uint64_t seed); 58 | 59 | private: 60 | std::atomic state_; 61 | }; 62 | 63 | inline uint32_t Random::Next() { 64 | uint64_t r = state_.load(std::memory_order_relaxed); 65 | r = ExponentialBiased::NextRandom(r); 66 | state_.store(r, std::memory_order_relaxed); 67 | return ExponentialBiased::GetRandom(r); 68 | } 69 | 70 | inline void Random::Reset(uint64_t seed) { 71 | state_.store(seed, std::memory_order_relaxed); 72 | } 73 | 74 | } // namespace tcmalloc_internal 75 | } // namespace tcmalloc 76 | 77 | #endif // TCMALLOC_INTERNAL_EXPONENTIAL_BIASED_H_ 78 | -------------------------------------------------------------------------------- /tcmalloc/internal/fake_profile.h: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_FAKE_PROFILE_H_ 16 | #define TCMALLOC_INTERNAL_FAKE_PROFILE_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "absl/functional/function_ref.h" 23 | #include "absl/time/time.h" 24 | #include "tcmalloc/malloc_extension.h" 25 | 26 | namespace tcmalloc { 27 | namespace tcmalloc_internal { 28 | 29 | class FakeProfile final : public ProfileBase { 30 | public: 31 | void SetSamples(std::vector samples) { 32 | samples_ = std::move(samples); 33 | } 34 | 35 | // For each sample in the profile, Iterate invokes the callback f on the 36 | // sample. 37 | void Iterate( 38 | absl::FunctionRef f) const override { 39 | for (const auto& sample : samples_) { 40 | f(sample); 41 | } 42 | } 43 | 44 | // The type of profile (live objects, allocated, etc.). 45 | ProfileType Type() const override { return type_; } 46 | void SetType(ProfileType type) { type_ = type; } 47 | 48 | // The duration of the profile 49 | absl::Duration Duration() const override { return duration_; } 50 | void SetDuration(absl::Duration duration) { duration_ = duration; } 51 | 52 | std::optional StartTime() const override { return start_time_; } 53 | void SetStartTime(std::optional t) { start_time_ = t; } 54 | 55 | private: 56 | std::vector samples_; 57 | ProfileType type_; 58 | absl::Duration duration_; 59 | std::optional start_time_; 60 | }; 61 | 62 | } // namespace tcmalloc_internal 63 | } // namespace tcmalloc 64 | 65 | #endif // TCMALLOC_INTERNAL_FAKE_PROFILE_H_ 66 | -------------------------------------------------------------------------------- /tcmalloc/internal/hook_list.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2025 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/internal/hook_list.h" 16 | 17 | #include "absl/base/attributes.h" 18 | #include "absl/base/const_init.h" 19 | #include "absl/base/internal/spinlock.h" 20 | #include "tcmalloc/internal/config.h" 21 | 22 | GOOGLE_MALLOC_SECTION_BEGIN 23 | namespace tcmalloc::tcmalloc_internal { 24 | 25 | ABSL_CONST_INIT absl::base_internal::SpinLock HookListBase::hooklist_spinlock_{ 26 | absl::kConstInit, absl::base_internal::SCHEDULE_KERNEL_ONLY}; 27 | 28 | } // namespace tcmalloc::tcmalloc_internal 29 | GOOGLE_MALLOC_SECTION_END 30 | -------------------------------------------------------------------------------- /tcmalloc/internal/linux_syscall_support.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_LINUX_SYSCALL_SUPPORT_H_ 16 | #define TCMALLOC_INTERNAL_LINUX_SYSCALL_SUPPORT_H_ 17 | 18 | #ifdef __linux__ 19 | #ifdef __has_include 20 | #if __has_include() 21 | #include 22 | #endif // __has_include() 23 | #endif // __has_include 24 | #endif // __linux__ 25 | 26 | /* include/uapi/linux/rseq.h */ 27 | 28 | struct kernel_rseq { 29 | unsigned cpu_id_start; 30 | unsigned cpu_id; 31 | unsigned long long rseq_cs; 32 | unsigned flags; 33 | unsigned padding[2]; 34 | // This is a prototype extension to the rseq() syscall. Since a process may 35 | // run on only a few cores at a time, we can use a dense set of "v(irtual) 36 | // cpus." This can reduce cache requirements, as we only need N caches for 37 | // the cores we actually run on simultaneously, rather than a cache for every 38 | // physical core. 39 | union { 40 | struct { 41 | short numa_node_id; 42 | short vcpu_id; 43 | }; 44 | int vcpu_flat; 45 | }; 46 | } __attribute__((aligned(4 * sizeof(unsigned long long)))); 47 | 48 | static_assert(sizeof(kernel_rseq) == (4 * sizeof(unsigned long long)), 49 | "Unexpected size for rseq structure"); 50 | 51 | struct kernel_rseq_cs { 52 | // This is aligned, per upstream RSEQ specification. 53 | } __attribute__((aligned(4 * sizeof(unsigned long long)))); 54 | 55 | static_assert(sizeof(kernel_rseq_cs) == (4 * sizeof(unsigned long long)), 56 | "Unexpected size for rseq_cs structure"); 57 | 58 | #if !defined(__NR_rseq) 59 | #if defined(__x86_64__) 60 | #define __NR_rseq 334 61 | #elif defined(__aarch64__) 62 | #define __NR_rseq 293 63 | #elif defined(__PPC__) 64 | #define __NR_rseq 387 65 | #endif 66 | #endif 67 | 68 | #ifndef KPF_ZERO_PAGE 69 | #define KPF_ZERO_PAGE 24 70 | #endif 71 | 72 | #endif // TCMALLOC_INTERNAL_LINUX_SYSCALL_SUPPORT_H_ 73 | -------------------------------------------------------------------------------- /tcmalloc/internal/logging_test_helper.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This is a trivial program. When run with a virtual address size rlimit, 16 | // TCMalloc should crash cleanly, rather than hang. 17 | 18 | int main() { return 0; } 19 | -------------------------------------------------------------------------------- /tcmalloc/internal/memory_stats.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_MEMORY_STATS_H_ 16 | #define TCMALLOC_INTERNAL_MEMORY_STATS_H_ 17 | 18 | #include 19 | 20 | #include "tcmalloc/internal/config.h" 21 | 22 | GOOGLE_MALLOC_SECTION_BEGIN 23 | namespace tcmalloc { 24 | namespace tcmalloc_internal { 25 | 26 | struct MemoryStats { 27 | int64_t vss; 28 | int64_t rss; 29 | int64_t shared; 30 | int64_t code; 31 | int64_t data; 32 | }; 33 | 34 | // Memory stats of a process 35 | bool GetMemoryStats(MemoryStats* stats); 36 | 37 | } // namespace tcmalloc_internal 38 | } // namespace tcmalloc 39 | GOOGLE_MALLOC_SECTION_END 40 | 41 | #endif // TCMALLOC_INTERNAL_MEMORY_STATS_H_ 42 | -------------------------------------------------------------------------------- /tcmalloc/internal/memory_stats_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/internal/memory_stats.h" 16 | 17 | #include 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | 22 | namespace tcmalloc { 23 | namespace tcmalloc_internal { 24 | namespace { 25 | 26 | TEST(Stats, ValidRanges) { 27 | MemoryStats stats; 28 | #if defined(__linux__) 29 | ASSERT_TRUE(GetMemoryStats(&stats)); 30 | #else 31 | ASSERT_FALSE(GetMemoryStats(&stats)); 32 | return; 33 | #endif 34 | 35 | EXPECT_GT(stats.vss, 0); 36 | EXPECT_GT(stats.rss, 0); 37 | EXPECT_GT(stats.shared, 0); 38 | EXPECT_GT(stats.code, 0); 39 | EXPECT_GT(stats.data, 0); 40 | } 41 | 42 | } // namespace 43 | } // namespace tcmalloc_internal 44 | } // namespace tcmalloc 45 | -------------------------------------------------------------------------------- /tcmalloc/internal/memory_tag.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/internal/memory_tag.h" 16 | 17 | #include "absl/strings/string_view.h" 18 | #include "tcmalloc/internal/config.h" 19 | #include "tcmalloc/internal/optimization.h" 20 | 21 | GOOGLE_MALLOC_SECTION_BEGIN 22 | namespace tcmalloc::tcmalloc_internal { 23 | 24 | absl::string_view MemoryTagToLabel(MemoryTag tag) { 25 | switch (tag) { 26 | case MemoryTag::kNormal: 27 | return "NORMAL"; 28 | case MemoryTag::kNormalP1: 29 | return "NORMAL_P1"; 30 | case MemoryTag::kSampled: 31 | return "SAMPLED"; 32 | case MemoryTag::kSelSan: 33 | return "SELSAN"; 34 | case MemoryTag::kCold: 35 | return "COLD"; 36 | case MemoryTag::kMetadata: 37 | return "METADATA"; 38 | } 39 | 40 | ASSUME(false); 41 | } 42 | 43 | } // namespace tcmalloc::tcmalloc_internal 44 | GOOGLE_MALLOC_SECTION_END 45 | -------------------------------------------------------------------------------- /tcmalloc/internal/mincore.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_MINCORE_H_ 16 | #define TCMALLOC_INTERNAL_MINCORE_H_ 17 | 18 | #include 19 | 20 | #include "tcmalloc/internal/config.h" 21 | 22 | GOOGLE_MALLOC_SECTION_BEGIN 23 | namespace tcmalloc { 24 | namespace tcmalloc_internal { 25 | 26 | // Class to wrap mincore so that we can replace it for testing. 27 | class MInCoreInterface { 28 | public: 29 | MInCoreInterface() {} 30 | virtual ~MInCoreInterface() {} 31 | virtual int mincore(void* addr, size_t length, unsigned char* result) = 0; 32 | 33 | private: 34 | MInCoreInterface(const MInCoreInterface&) = delete; 35 | MInCoreInterface& operator=(const MInCoreInterface&) = delete; 36 | }; 37 | 38 | // The MInCore class through the function residence(addr, size) provides 39 | // a convenient way to report the residence of an arbitrary memory region. 40 | // This is a wrapper for the ::mincore() function. The ::mincore() function has 41 | // the constraint of requiring the base address to be page aligned. 42 | class MInCore { 43 | public: 44 | MInCore() {} 45 | // For a region of memory return the number of bytes that are 46 | // actually resident in memory. Note that the address and size 47 | // do not need to be a multiple of the system page size. 48 | static size_t residence(void* addr, size_t size); 49 | 50 | private: 51 | // Separate out the implementation to make the code easier to test. 52 | static size_t residence_impl(void* addr, size_t size, 53 | MInCoreInterface* mincore); 54 | 55 | // Size of the array used to gather results from mincore(). 56 | static constexpr int kArrayLength = 4096; 57 | // Friends required for testing 58 | friend class MInCoreTest; 59 | }; 60 | 61 | } // namespace tcmalloc_internal 62 | } // namespace tcmalloc 63 | GOOGLE_MALLOC_SECTION_END 64 | 65 | #endif // TCMALLOC_INTERNAL_MINCORE_H_ 66 | -------------------------------------------------------------------------------- /tcmalloc/internal/mincore_benchmark.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "benchmark/benchmark.h" 24 | #include "tcmalloc/internal/config.h" 25 | #include "tcmalloc/internal/logging.h" 26 | #include "tcmalloc/internal/page_size.h" 27 | 28 | GOOGLE_MALLOC_SECTION_BEGIN 29 | namespace tcmalloc { 30 | namespace { 31 | 32 | // Benchmark performance of mincore. We use an array on the stack to gather 33 | // mincore data. The larger the array the more we amortise the cost of calling 34 | // mincore, but the more stack space the array takes up. 35 | void BM_mincore(benchmark::State& state) { 36 | const int size = state.range(0); 37 | 38 | // If we want to place the array on the stack then the maximum frame size is 39 | // 16KiB. So there is no point in benchmarking sizes larger than this. 40 | const int kMaxArraySize = 16 * 1024; 41 | TC_CHECK_LE(size, kMaxArraySize); 42 | auto resident = std::make_unique(kMaxArraySize); 43 | 44 | const size_t kPageSize = tcmalloc_internal::GetPageSize(); 45 | // We want to scan the same amount of memory in all cases 46 | const size_t regionSize = 1 * 1024 * 1024 * 1024; 47 | for (auto s : state) { 48 | uintptr_t memory = 0; 49 | while (memory < regionSize) { 50 | // Call mincore for the next section 51 | int length = std::min(size * kPageSize, (regionSize - memory)); 52 | ::mincore(reinterpret_cast(memory), length, resident.get()); 53 | memory += length * kPageSize; 54 | } 55 | } 56 | } 57 | BENCHMARK(BM_mincore)->Range(1, 16 * 1024); 58 | 59 | } // namespace 60 | } // namespace tcmalloc 61 | GOOGLE_MALLOC_SECTION_END 62 | -------------------------------------------------------------------------------- /tcmalloc/internal/mock_span.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_MOCK_SPAN_H_ 16 | #define TCMALLOC_INTERNAL_MOCK_SPAN_H_ 17 | 18 | #include "tcmalloc/internal/linked_list.h" 19 | 20 | namespace tcmalloc { 21 | namespace tcmalloc_internal { 22 | 23 | class MockSpan; 24 | typedef TList MockSpanList; 25 | 26 | class MockSpan : public MockSpanList::Elem { 27 | public: 28 | MockSpan() {} 29 | 30 | static MockSpan* New(int idx = 0) { 31 | MockSpan* ret = new MockSpan(); 32 | ret->index_ = idx; 33 | return ret; 34 | } 35 | 36 | int index_; 37 | }; 38 | 39 | } // namespace tcmalloc_internal 40 | } // namespace tcmalloc 41 | 42 | #endif // TCMALLOC_INTERNAL_MOCK_SPAN_H_ 43 | -------------------------------------------------------------------------------- /tcmalloc/internal/optimization.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_OPTIMIZATION_H_ 16 | #define TCMALLOC_INTERNAL_OPTIMIZATION_H_ 17 | 18 | #include "absl/base/attributes.h" 19 | #include "tcmalloc/internal/config.h" 20 | #include "tcmalloc/internal/logging.h" 21 | 22 | GOOGLE_MALLOC_SECTION_BEGIN 23 | namespace tcmalloc { 24 | namespace tcmalloc_internal { 25 | 26 | // Our wrapper for __builtin_assume, allowing us to check the assumption on 27 | // debug builds. 28 | #ifndef NDEBUG 29 | #ifdef __clang__ 30 | #define ASSUME(cond) TC_CHECK(cond), __builtin_assume(cond) 31 | #else 32 | #define ASSUME(cond) \ 33 | TC_CHECK(cond), (!(cond) ? __builtin_unreachable() : (void)0) 34 | #endif 35 | #else 36 | #ifdef __clang__ 37 | #define ASSUME(cond) __builtin_assume(cond) 38 | #else 39 | #define ASSUME(cond) (!(cond) ? __builtin_unreachable() : (void)0) 40 | #endif 41 | #endif 42 | 43 | // Annotations for functions that are not affected by nor affect observable 44 | // state of the program. 45 | #if ABSL_HAVE_ATTRIBUTE(const) 46 | #define TCMALLOC_ATTRIBUTE_CONST __attribute__((const)) 47 | #else 48 | #define TCMALLOC_ATTRIBUTE_CONST 49 | #endif 50 | 51 | // Can be applied to a return statement to tell the compiler to generate 52 | // a tail call. 53 | #if ABSL_HAVE_CPP_ATTRIBUTE(clang::musttail) 54 | #define TCMALLOC_MUSTTAIL [[clang::musttail]] 55 | #else 56 | #define TCMALLOC_MUSTTAIL 57 | #endif 58 | 59 | inline void* AssumeNotNull(void* p) { 60 | ASSUME(p != nullptr); 61 | return p; 62 | } 63 | 64 | } // namespace tcmalloc_internal 65 | } // namespace tcmalloc 66 | GOOGLE_MALLOC_SECTION_END 67 | 68 | #endif // TCMALLOC_INTERNAL_OPTIMIZATION_H_ 69 | -------------------------------------------------------------------------------- /tcmalloc/internal/overflow.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_OVERFLOW_H_ 16 | #define TCMALLOC_INTERNAL_OVERFLOW_H_ 17 | 18 | #include 19 | 20 | #include "absl/base/config.h" 21 | #include "tcmalloc/internal/config.h" 22 | 23 | GOOGLE_MALLOC_SECTION_BEGIN 24 | namespace tcmalloc { 25 | namespace tcmalloc_internal { 26 | 27 | inline bool MultiplyOverflow(size_t a, size_t b, size_t* out) { 28 | #if ABSL_HAVE_BUILTIN(__builtin_mul_overflow) 29 | return __builtin_mul_overflow(a, b, out); 30 | #else 31 | *out = a * b; 32 | return b != 0 && *out / b != a; 33 | #endif 34 | } 35 | 36 | } // namespace tcmalloc_internal 37 | } // namespace tcmalloc 38 | GOOGLE_MALLOC_SECTION_END 39 | 40 | #endif // TCMALLOC_INTERNAL_OVERFLOW_H_ 41 | -------------------------------------------------------------------------------- /tcmalloc/internal/page_size.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/internal/page_size.h" 16 | 17 | #include 18 | 19 | #include 20 | 21 | #include "absl/base/attributes.h" 22 | #include "absl/base/call_once.h" 23 | #include "tcmalloc/internal/config.h" 24 | 25 | GOOGLE_MALLOC_SECTION_BEGIN 26 | namespace tcmalloc { 27 | namespace tcmalloc_internal { 28 | 29 | size_t GetPageSize() { 30 | ABSL_CONST_INIT static size_t page_size; 31 | ABSL_CONST_INIT static absl::once_flag flag; 32 | 33 | absl::base_internal::LowLevelCallOnce(&flag, [&]() { 34 | #if defined(__wasm__) || defined(__asmjs__) 35 | page_size = static_cast(getpagesize()); 36 | #else 37 | page_size = static_cast(sysconf(_SC_PAGESIZE)); 38 | #endif 39 | }); 40 | return page_size; 41 | } 42 | 43 | } // namespace tcmalloc_internal 44 | } // namespace tcmalloc 45 | GOOGLE_MALLOC_SECTION_END 46 | -------------------------------------------------------------------------------- /tcmalloc/internal/page_size.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_PAGE_SIZE_H_ 16 | #define TCMALLOC_INTERNAL_PAGE_SIZE_H_ 17 | 18 | #include 19 | 20 | #include "absl/base/attributes.h" 21 | #include "tcmalloc/internal/config.h" 22 | 23 | GOOGLE_MALLOC_SECTION_BEGIN 24 | namespace tcmalloc { 25 | namespace tcmalloc_internal { 26 | 27 | ABSL_ATTRIBUTE_PURE_FUNCTION size_t GetPageSize(); 28 | 29 | } // namespace tcmalloc_internal 30 | } // namespace tcmalloc 31 | GOOGLE_MALLOC_SECTION_END 32 | 33 | #endif // TCMALLOC_INTERNAL_PAGE_SIZE_H_ 34 | -------------------------------------------------------------------------------- /tcmalloc/internal/percpu_early_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include "gtest/gtest.h" 18 | #include "absl/base/attributes.h" 19 | #include "tcmalloc/internal/percpu.h" 20 | 21 | namespace tcmalloc::tcmalloc_internal::subtle::percpu { 22 | namespace { 23 | 24 | ABSL_CONST_INIT std::optional success; 25 | 26 | TEST(PerCpu, IsRegistered) { 27 | // Verify preinit ran. Its success should be identical to running it after 28 | // main has started. 29 | EXPECT_TRUE(success.has_value()); 30 | EXPECT_EQ(success, IsFast()); 31 | } 32 | 33 | void register_rseq() { success = IsFast(); } 34 | 35 | __attribute__((section(".preinit_array"), 36 | used)) void (*__local_install_factory_preinit)() = register_rseq; 37 | 38 | } // namespace 39 | } // namespace tcmalloc::tcmalloc_internal::subtle::percpu 40 | -------------------------------------------------------------------------------- /tcmalloc/internal/percpu_rseq_asm.S: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Single file to include target specific implementations for percpu. 16 | 17 | #include "tcmalloc/internal/percpu.h" 18 | 19 | #ifdef __GCC_HAVE_DWARF2_CFI_ASM 20 | #define CFI(...) __VA_ARGS__ 21 | #else 22 | #define CFI(...) 23 | #endif 24 | 25 | #if TCMALLOC_PERCPU_RSEQ_SUPPORTED_PLATFORM 26 | #if defined(__x86_64__) 27 | #include "tcmalloc/internal/percpu_rseq_x86_64.S" 28 | #elif defined(__aarch64__) 29 | #include "tcmalloc/internal/percpu_rseq_aarch64.S" 30 | #else 31 | #error "RSEQ support expected, but not found." 32 | #endif 33 | 34 | // See the comment about data layout in percpu.h for details. 35 | .type tcmalloc_sampler, @object 36 | .type tcmalloc_cached_vcpu, @object 37 | .type tcmalloc_slabs, @object 38 | .type __rseq_abi, @object 39 | .section .tdata, "awT", @progbits 40 | .globl tcmalloc_sampler 41 | .globl tcmalloc_cached_vcpu 42 | .globl tcmalloc_slabs 43 | .globl __rseq_abi 44 | .p2align 6 45 | .zero 64 + 32 - TCMALLOC_SAMPLER_SIZE - 8 46 | tcmalloc_sampler: 47 | .zero TCMALLOC_SAMPLER_SIZE 48 | tcmalloc_cached_vcpu: 49 | .long 0xffffffff // cpu_id (kCpuIdUninitialized) 50 | tcmalloc_slabs: 51 | .long 0 52 | __rseq_abi: 53 | .long 0 // cpu_id_start 54 | .long 0xffffffff // cpu_id (kCpuIdUninitialized) 55 | .quad 0 // rseq_cs 56 | .long 0 // flags 57 | .quad 0 // padding 58 | .short 0xffff // numa_node_id (kCpuIdUninitialized) 59 | .short 0xffff // vcpu_id (kCpuIdUninitialized) 60 | .size __rseq_abi, 32 61 | .size tcmalloc_sampler, TCMALLOC_SAMPLER_SIZE 62 | .size tcmalloc_slabs, 8 63 | 64 | #endif // TCMALLOC_PERCPU_RSEQ_SUPPORTED_PLATFORM 65 | 66 | // We do not need an executable stack. Put this outside the 67 | // architecture-specific region above in order to suppress "missing 68 | // .note.GNU-stack section implies executable stack" errors. 69 | // 70 | // Cf. http://en.chys.info/2010/12/note-gnu-stack/ 71 | #if defined(__arm__) 72 | .section .note.GNU-stack, "", %progbits 73 | #else 74 | .section .note.GNU-stack, "", @progbits 75 | #endif // __arm__ 76 | 77 | -------------------------------------------------------------------------------- /tcmalloc/internal/percpu_rseq_unsupported.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Provides skeleton RSEQ functions which raise a hard error in the case of 16 | // being erroneously called on an unsupported platform. 17 | 18 | #include "tcmalloc/internal/percpu.h" 19 | 20 | #if !TCMALLOC_PERCPU_RSEQ_SUPPORTED_PLATFORM 21 | 22 | GOOGLE_MALLOC_SECTION_BEGIN 23 | namespace tcmalloc { 24 | namespace tcmalloc_internal { 25 | namespace subtle { 26 | namespace percpu { 27 | 28 | static void Unsupported() { 29 | TC_BUG("RSEQ function called on unsupported platform."); 30 | } 31 | 32 | int TcmallocSlab_Internal_PerCpuCmpxchg64(int target_cpu, intptr_t* p, 33 | intptr_t old_val, intptr_t new_val, 34 | size_t virtual_cpu_id_offset) { 35 | Unsupported(); 36 | return -1; 37 | } 38 | 39 | size_t TcmallocSlab_Internal_PushBatch(size_t size_class, void** batch, 40 | size_t len, uintptr_t slabs_and_shift, 41 | size_t virtual_cpu_id_offset) { 42 | Unsupported(); 43 | return 0; 44 | } 45 | 46 | size_t TcmallocSlab_Internal_PopBatch(size_t size_class, void** batch, 47 | size_t len, uintptr_t slabs_and_shift, 48 | size_t virtual_cpu_id_offset) { 49 | Unsupported(); 50 | return 0; 51 | } 52 | 53 | int PerCpuReadCycleCounter(int64_t* cycles) { 54 | Unsupported(); 55 | return -1; 56 | } 57 | 58 | } // namespace percpu 59 | } // namespace subtle 60 | } // namespace tcmalloc_internal 61 | } // namespace tcmalloc 62 | GOOGLE_MALLOC_SECTION_END 63 | 64 | #endif // !TCMALLOC_PERCPU_RSEQ_SUPPORTED_PLATFORM 65 | -------------------------------------------------------------------------------- /tcmalloc/internal/percpu_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/internal/percpu.h" 16 | 17 | #include 18 | 19 | #include 20 | #include 21 | 22 | #include "gtest/gtest.h" 23 | #include "absl/base/attributes.h" 24 | #include "absl/log/absl_check.h" 25 | #include "absl/time/time.h" 26 | #include "tcmalloc/internal/logging.h" 27 | #include "tcmalloc/testing/testutil.h" 28 | 29 | namespace tcmalloc::tcmalloc_internal::subtle::percpu { 30 | namespace { 31 | 32 | ABSL_CONST_INIT std::atomic alarms{0}; 33 | 34 | void sa_alrm(int sig) { 35 | alarms.fetch_add(1, std::memory_order_relaxed); 36 | TC_CHECK(IsFast()); 37 | } 38 | 39 | TEST(PerCpu, SignalHandling) { 40 | if (!IsFast()) { 41 | GTEST_SKIP() << "per-CPU unavailable"; 42 | } 43 | 44 | struct sigaction sig; 45 | memset(&sig, 0, sizeof(sig)); // sa_flags == 0 => SA_RESTART not set 46 | sig.sa_handler = sa_alrm; 47 | ABSL_CHECK_EQ(sigaction(SIGALRM, &sig, nullptr), 48 | 0); // install signal handler 49 | 50 | constexpr absl::Duration interval = absl::Microseconds(1); 51 | struct timeval timeval = absl::ToTimeval(interval); 52 | 53 | struct itimerval signal_interval; 54 | signal_interval.it_value = timeval; 55 | signal_interval.it_interval = timeval; 56 | 57 | setitimer(ITIMER_REAL, &signal_interval, nullptr); 58 | 59 | for (int i = 0; i < 100000; ++i) { 60 | UnregisterRseq(); 61 | TC_CHECK(IsFast()); 62 | } 63 | 64 | timeval = absl::ToTimeval(absl::ZeroDuration()); 65 | signal_interval.it_value = timeval; 66 | signal_interval.it_interval = timeval; 67 | 68 | setitimer(ITIMER_REAL, &signal_interval, nullptr); 69 | 70 | EXPECT_GT(alarms.load(std::memory_order_relaxed), 0); 71 | } 72 | 73 | } // namespace 74 | } // namespace tcmalloc::tcmalloc_internal::subtle::percpu 75 | -------------------------------------------------------------------------------- /tcmalloc/internal/prefetch_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/internal/prefetch.h" 16 | 17 | #include "gtest/gtest.h" 18 | 19 | namespace tcmalloc { 20 | namespace tcmalloc_internal { 21 | namespace { 22 | 23 | int number = 42; 24 | 25 | TEST(Prefetch, TemporalLocalityNone) { 26 | PrefetchNta(&number); 27 | EXPECT_EQ(number, 42); 28 | } 29 | 30 | TEST(Prefetch, TemporalLocalityLow) { 31 | PrefetchT2(&number); 32 | EXPECT_EQ(number, 42); 33 | } 34 | 35 | TEST(Prefetch, TemporalLocalityMedium) { 36 | PrefetchT1(&number); 37 | EXPECT_EQ(number, 42); 38 | } 39 | 40 | TEST(Prefetch, TemporalLocalityHigh) { 41 | PrefetchT0(&number); 42 | EXPECT_EQ(number, 42); 43 | } 44 | 45 | TEST(Prefetch, PrefetchForWrite) { 46 | PrefetchW(&number); 47 | EXPECT_EQ(number, 42); 48 | } 49 | 50 | TEST(Prefetch, WriteTemporalLocalityNone) { 51 | PrefetchWNta(&number); 52 | EXPECT_EQ(number, 42); 53 | } 54 | 55 | TEST(Prefetch, WriteTemporalLocalityLow) { 56 | PrefetchWT2(&number); 57 | EXPECT_EQ(number, 42); 58 | } 59 | 60 | TEST(Prefetch, WriteTemporalLocalityMedium) { 61 | PrefetchWT1(&number); 62 | EXPECT_EQ(number, 42); 63 | } 64 | 65 | TEST(Prefetch, WriteTemporalLocalityHigh) { 66 | PrefetchWT0(&number); 67 | EXPECT_EQ(number, 42); 68 | } 69 | 70 | } // namespace 71 | } // namespace tcmalloc_internal 72 | } // namespace tcmalloc 73 | -------------------------------------------------------------------------------- /tcmalloc/internal/proc_maps.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_PROC_MAPS_H_ 16 | #define TCMALLOC_INTERNAL_PROC_MAPS_H_ 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | #include "tcmalloc/internal/config.h" 24 | 25 | GOOGLE_MALLOC_SECTION_BEGIN 26 | namespace tcmalloc { 27 | namespace tcmalloc_internal { 28 | 29 | // A ProcMapsIterator abstracts access to /proc/maps for a given process. 30 | class ProcMapsIterator { 31 | public: 32 | struct Buffer { 33 | static constexpr size_t kBufSize = PATH_MAX + 1024; 34 | char buf[kBufSize]; 35 | }; 36 | 37 | // Create an iterator with specified storage (for use in signal handler). 38 | // 39 | // pid can be 0 for "self". 40 | ProcMapsIterator(pid_t pid, Buffer* buffer); 41 | 42 | // Returns true if the iterator successfully initialized; 43 | bool Valid() const; 44 | 45 | bool NextExt(uint64_t* start, uint64_t* end, char** flags, uint64_t* offset, 46 | int64_t* inode, char** filename, dev_t* dev); 47 | 48 | ~ProcMapsIterator(); 49 | 50 | private: 51 | char* ibuf_; // input buffer 52 | char* stext_; // start of text 53 | char* etext_; // end of text 54 | char* nextline_; // start of next line 55 | char* ebuf_; // end of buffer (1 char for a nul) 56 | int fd_; // filehandle on /proc/*/maps 57 | pid_t pid_; 58 | char flags_[10]; 59 | }; 60 | 61 | } // namespace tcmalloc_internal 62 | } // namespace tcmalloc 63 | GOOGLE_MALLOC_SECTION_END 64 | 65 | #endif // TCMALLOC_INTERNAL_PROC_MAPS_H_ 66 | -------------------------------------------------------------------------------- /tcmalloc/internal/profile_builder_fuzz.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/internal/profile_builder.h" 16 | 17 | #if defined(__linux__) 18 | #include 19 | #include 20 | #endif // defined(__linux__) 21 | 22 | #include 23 | 24 | #include "fuzztest/fuzztest.h" 25 | 26 | namespace tcmalloc::tcmalloc_internal { 27 | namespace { 28 | 29 | void ParseBuildID(const std::string& s) { 30 | const char* data = s.data(); 31 | size_t size = s.size(); 32 | #if defined(__linux__) 33 | ElfW(Phdr) note; 34 | note.p_type = PT_NOTE; 35 | note.p_vaddr = reinterpret_cast(nullptr); 36 | note.p_filesz = size; 37 | note.p_memsz = size; 38 | note.p_align = 4; 39 | 40 | dl_phdr_info info = {}; 41 | info.dlpi_name = "test"; 42 | info.dlpi_addr = reinterpret_cast(data); 43 | info.dlpi_phdr = ¬e; 44 | info.dlpi_phnum = 1; 45 | 46 | GetBuildId(&info); 47 | #endif // defined(__linux__) 48 | } 49 | 50 | FUZZ_TEST(ProfileBuilderTest, ParseBuildID) 51 | ; 52 | 53 | } // namespace 54 | } // namespace tcmalloc::tcmalloc_internal 55 | -------------------------------------------------------------------------------- /tcmalloc/internal/profile_builder_no_tcmalloc_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "gmock/gmock.h" 16 | #include "gtest/gtest.h" 17 | #include "absl/status/status.h" 18 | #include "absl/status/status_matchers.h" 19 | #include "tcmalloc/internal/profile_builder.h" 20 | #include "tcmalloc/malloc_extension.h" 21 | 22 | namespace tcmalloc { 23 | namespace tcmalloc_internal { 24 | namespace { 25 | 26 | using ::absl_testing::StatusIs; 27 | using ::testing::HasSubstr; 28 | 29 | TEST(ProfileBuilderNoTCMallocTest, StatusErrorTest) { 30 | auto profile_st = 31 | MakeProfileProto(MallocExtension::SnapshotCurrent(ProfileType::kHeap)); 32 | #if defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ 33 | defined(ABSL_HAVE_LEAK_SANITIZER) || \ 34 | defined(ABSL_HAVE_MEMORY_SANITIZER) || defined(ABSL_HAVE_THREAD_SANITIZER) 35 | EXPECT_THAT(profile_st, 36 | StatusIs(absl::StatusCode::kUnimplemented, 37 | HasSubstr("Program was built with sanitizers enabled"))); 38 | #else 39 | EXPECT_THAT(profile_st, StatusIs(absl::StatusCode::kInvalidArgument, 40 | HasSubstr("Empty heap profile"))); 41 | #endif 42 | } 43 | 44 | } // namespace 45 | } // namespace tcmalloc_internal 46 | } // namespace tcmalloc 47 | -------------------------------------------------------------------------------- /tcmalloc/internal/sampled_allocation.h: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_INTERNAL_SAMPLED_ALLOCATION_H_ 16 | #define TCMALLOC_INTERNAL_SAMPLED_ALLOCATION_H_ 17 | 18 | #include 19 | 20 | #include "tcmalloc/internal/logging.h" 21 | #include "tcmalloc/internal/sampled_allocation_recorder.h" 22 | 23 | GOOGLE_MALLOC_SECTION_BEGIN 24 | namespace tcmalloc { 25 | namespace tcmalloc_internal { 26 | 27 | // Stores information about the sampled allocation. 28 | struct SampledAllocation : public tcmalloc_internal::Sample { 29 | // We use this constructor to initialize `graveyard_`, which is used to 30 | // maintain the freelist of SampledAllocations. When we revive objects from 31 | // the freelist, we use `PrepareForSampling()` to update the state of the 32 | // object. 33 | constexpr SampledAllocation() = default; 34 | 35 | // When no object is available on the freelist, we allocate for a new 36 | // SampledAllocation object and invoke this constructor with 37 | // `PrepareForSampling()`. 38 | explicit SampledAllocation(StackTrace&& stack_trace) { 39 | PrepareForSampling(std::move(stack_trace)); 40 | } 41 | 42 | SampledAllocation(const SampledAllocation&) = delete; 43 | SampledAllocation& operator=(const SampledAllocation&) = delete; 44 | 45 | SampledAllocation(SampledAllocation&&) = delete; 46 | SampledAllocation& operator=(SampledAllocation&&) = delete; 47 | 48 | // Prepares the state of the object. It is invoked when either a new sampled 49 | // allocation is constructed or when an object is revived from the freelist. 50 | void PrepareForSampling(StackTrace&& stack_trace) 51 | ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock) { 52 | sampled_stack = std::move(stack_trace); 53 | } 54 | 55 | // The stack trace of the sampled allocation. 56 | StackTrace sampled_stack = {}; 57 | }; 58 | 59 | } // namespace tcmalloc_internal 60 | } // namespace tcmalloc 61 | GOOGLE_MALLOC_SECTION_END 62 | 63 | #endif // TCMALLOC_INTERNAL_SAMPLED_ALLOCATION_H_ 64 | -------------------------------------------------------------------------------- /tcmalloc/internal/sysinfo_fuzz.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "fuzztest/fuzztest.h" 21 | #include "tcmalloc/internal/cpu_utils.h" 22 | #include "tcmalloc/internal/sysinfo.h" 23 | 24 | namespace tcmalloc::tcmalloc_internal { 25 | namespace { 26 | 27 | void ParseInput(const std::string& s) { 28 | const char* data = s.data(); 29 | size_t size = s.size(); 30 | 31 | std::optional r = 32 | ParseCpulist([&](char* buf, size_t count) -> ssize_t { 33 | size_t to_read = std::min(size, count); 34 | if (to_read > 0) { 35 | memcpy(buf, data, to_read); 36 | data += to_read; 37 | size -= to_read; 38 | } 39 | return to_read; 40 | }); 41 | (void)r; 42 | } 43 | 44 | FUZZ_TEST(SysinfoTest, ParseInput); 45 | 46 | } // namespace 47 | } // namespace tcmalloc::tcmalloc_internal 48 | -------------------------------------------------------------------------------- /tcmalloc/internal/testdata/b180635896.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/tcmalloc/internal/testdata/b180635896.so -------------------------------------------------------------------------------- /tcmalloc/internal/testdata/gnu-property.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/tcmalloc/internal/testdata/gnu-property.so -------------------------------------------------------------------------------- /tcmalloc/internal/testdata/profile_builder_fuzz/clusterfuzz-testcase-minimized-profile_builder_fuzz-5534221534363648: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/tcmalloc/internal/testdata/profile_builder_fuzz/clusterfuzz-testcase-minimized-profile_builder_fuzz-5534221534363648 -------------------------------------------------------------------------------- /tcmalloc/internal/testdata/profile_builder_fuzz/clusterfuzz-testcase-minimized-profile_builder_fuzz-5647243657216000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/tcmalloc/internal/testdata/profile_builder_fuzz/clusterfuzz-testcase-minimized-profile_builder_fuzz-5647243657216000 -------------------------------------------------------------------------------- /tcmalloc/internal/testdata/profile_builder_fuzz/clusterfuzz-testcase-minimized-profile_builder_fuzz-5915530833559552: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/tcmalloc/internal/testdata/profile_builder_fuzz/clusterfuzz-testcase-minimized-profile_builder_fuzz-5915530833559552 -------------------------------------------------------------------------------- /tcmalloc/internal/testdata/profile_builder_fuzz/clusterfuzz-testcase-minimized-profile_builder_fuzz-6685031907328000.fuzz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/tcmalloc/internal/testdata/profile_builder_fuzz/clusterfuzz-testcase-minimized-profile_builder_fuzz-6685031907328000.fuzz -------------------------------------------------------------------------------- /tcmalloc/internal/testdata/profile_builder_fuzz/crash-adc83b19e793491b1c6ea0fd8b46cd9f32e592fc: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tcmalloc/internal_malloc_tracing_extension.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Extra extensions exported by some malloc implementations. These 16 | // extensions are accessed through a virtual base class so an 17 | // application can link against a malloc that does not implement these 18 | // extensions, and it will get default versions that do nothing. 19 | 20 | #ifndef TCMALLOC_INTERNAL_MALLOC_TRACING_EXTENSION_H_ 21 | #define TCMALLOC_INTERNAL_MALLOC_TRACING_EXTENSION_H_ 22 | 23 | #include "absl/base/attributes.h" 24 | #include "absl/status/statusor.h" 25 | #include "tcmalloc/malloc_tracing_extension.h" 26 | 27 | #if ABSL_HAVE_ATTRIBUTE_WEAK && !defined(__APPLE__) && !defined(__EMSCRIPTEN__) 28 | 29 | ABSL_ATTRIBUTE_WEAK 30 | absl::StatusOr 31 | MallocTracingExtension_Internal_GetAllocatedAddressRanges(); 32 | 33 | #endif 34 | 35 | #endif // TCMALLOC_INTERNAL_MALLOC_TRACING_EXTENSION_H_ 36 | -------------------------------------------------------------------------------- /tcmalloc/malloc_extension_fuzz.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "fuzztest/fuzztest.h" 23 | #include "tcmalloc/malloc_extension.h" 24 | 25 | namespace tcmalloc { 26 | namespace { 27 | 28 | void FuzzGetProperty(const std::string& property) { 29 | std::optional val = MallocExtension::GetNumericProperty(property); 30 | if (!val.has_value()) { 31 | // Rather than inspect the result of MallocExtension::GetProperties, we 32 | // defer to the test in 33 | // //tcmalloc/testing/malloc_extension_test.cc to ensure that 34 | // every key in GetProperties has a value returned by GetNumericProperty. 35 | return; 36 | } 37 | 38 | std::map properties = 39 | MallocExtension::GetProperties(); 40 | if (properties.find(property) == properties.end()) { 41 | __builtin_trap(); 42 | } 43 | } 44 | 45 | FUZZ_TEST(MallocExtensionTest, FuzzGetProperty) 46 | ; 47 | 48 | } // namespace 49 | } // namespace tcmalloc 50 | -------------------------------------------------------------------------------- /tcmalloc/malloc_hook_invoke.h: -------------------------------------------------------------------------------- 1 | // Copyright 2025 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_MALLOC_HOOK_INVOKE_H_ 16 | #define TCMALLOC_MALLOC_HOOK_INVOKE_H_ 17 | 18 | #include "tcmalloc/internal/config.h" 19 | #include "tcmalloc/internal/hook_list.h" 20 | #include "tcmalloc/malloc_hook.h" 21 | 22 | GOOGLE_MALLOC_SECTION_BEGIN 23 | namespace tcmalloc { 24 | namespace tcmalloc_internal { 25 | 26 | extern HookList sampled_new_hooks_; 27 | extern HookList sampled_delete_hooks_; 28 | 29 | } // namespace tcmalloc_internal 30 | 31 | inline void MallocHook::InvokeSampledNewHook( 32 | const SampledAlloc& sampled_alloc) { 33 | if (!tcmalloc_internal::sampled_new_hooks_.empty()) { 34 | InvokeSampledNewHookSlow(sampled_alloc); 35 | } 36 | } 37 | 38 | inline void MallocHook::InvokeSampledDeleteHook( 39 | const 40 | SampledAlloc& sampled_alloc) { 41 | if (!tcmalloc_internal::sampled_delete_hooks_.empty()) { 42 | InvokeSampledDeleteHookSlow(sampled_alloc); 43 | } 44 | } 45 | 46 | } // namespace tcmalloc 47 | GOOGLE_MALLOC_SECTION_END 48 | 49 | #endif // TCMALLOC_MALLOC_HOOK_INVOKE_H_ 50 | -------------------------------------------------------------------------------- /tcmalloc/malloc_tracing_extension.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Extra extensions exported by some malloc implementations. These 16 | // extensions are accessed through a virtual base class so an 17 | // application can link against a malloc that does not implement these 18 | // extensions, and it will get default versions that do nothing. 19 | 20 | #include "tcmalloc/malloc_tracing_extension.h" 21 | 22 | #include "absl/base/attributes.h" 23 | #include "absl/status/status.h" 24 | #include "absl/status/statusor.h" 25 | #include "tcmalloc/internal_malloc_tracing_extension.h" 26 | 27 | namespace tcmalloc { 28 | namespace malloc_tracing_extension { 29 | 30 | absl::StatusOr GetAllocatedAddressRanges() { 31 | #if ABSL_HAVE_ATTRIBUTE_WEAK && !defined(__APPLE__) && !defined(__EMSCRIPTEN__) 32 | if (&MallocTracingExtension_Internal_GetAllocatedAddressRanges != nullptr) { 33 | return MallocTracingExtension_Internal_GetAllocatedAddressRanges(); 34 | } 35 | #endif 36 | return absl::UnimplementedError( 37 | "malloc_tracing_extension routines not exported by the current malloc."); 38 | } 39 | 40 | } // namespace malloc_tracing_extension 41 | } // namespace tcmalloc 42 | -------------------------------------------------------------------------------- /tcmalloc/malloc_tracing_extension.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Extra extensions exported by some malloc implementations. These 16 | // extensions are accessed through a virtual base class so an 17 | // application can link against a malloc that does not implement these 18 | // extensions, and it will get default versions that do nothing. 19 | 20 | #ifndef TCMALLOC_MALLOC_TRACING_EXTENSION_H_ 21 | #define TCMALLOC_MALLOC_TRACING_EXTENSION_H_ 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #include "absl/status/statusor.h" 28 | 29 | namespace tcmalloc { 30 | namespace malloc_tracing_extension { 31 | 32 | // Type used by GetAllocatedAddressRanges. Contains details of address ranges 33 | // that have a corresponding Span in TCMalloc. 34 | struct AllocatedAddressRanges { 35 | struct SpanDetails { 36 | uintptr_t start_addr; 37 | size_t size; 38 | // For Spans with objects that fit into some size-class, object_size is 39 | // actually the size-class bytes, not the exact object size bytes. 40 | // This is zero for non-size-class objects that are objects larger than 41 | // kMaxSize. 42 | size_t object_size; 43 | }; 44 | // Note that any subset of size-class-sized objects may be currently 45 | // allocated from each Span. 46 | std::vector spans; 47 | }; 48 | // Returns the address ranges currently allocated by TCMalloc. 49 | absl::StatusOr GetAllocatedAddressRanges(); 50 | 51 | } // namespace malloc_tracing_extension 52 | } // namespace tcmalloc 53 | 54 | #endif // TCMALLOC_MALLOC_TRACING_EXTENSION_H_ 55 | -------------------------------------------------------------------------------- /tcmalloc/metadata_allocator.h: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_METADATA_ALLOCATOR_H_ 16 | #define TCMALLOC_METADATA_ALLOCATOR_H_ 17 | 18 | #include 19 | 20 | #include "absl/base/attributes.h" 21 | #include "tcmalloc/internal/config.h" 22 | 23 | GOOGLE_MALLOC_SECTION_BEGIN 24 | namespace tcmalloc::tcmalloc_internal { 25 | 26 | class MetadataAllocator { 27 | public: 28 | MetadataAllocator() = default; 29 | virtual ~MetadataAllocator() = default; 30 | 31 | MetadataAllocator(const MetadataAllocator&) = delete; 32 | MetadataAllocator(MetadataAllocator&&) = delete; 33 | MetadataAllocator& operator=(const MetadataAllocator&) = delete; 34 | MetadataAllocator& operator=(MetadataAllocator&&) = delete; 35 | 36 | // Allocates bytes suitable for metadata. 37 | [[nodiscard]] virtual void* operator()(size_t bytes) = 0; 38 | }; 39 | 40 | } // namespace tcmalloc::tcmalloc_internal 41 | GOOGLE_MALLOC_SECTION_END 42 | 43 | #endif // TCMALLOC_METADATA_ALLOCATOR_H_ 44 | -------------------------------------------------------------------------------- /tcmalloc/mock_central_freelist.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/mock_central_freelist.h" 16 | 17 | #include "gtest/gtest.h" 18 | #include "absl/base/internal/spinlock.h" 19 | #include "absl/types/span.h" 20 | #include "tcmalloc/internal/logging.h" 21 | 22 | namespace tcmalloc { 23 | namespace tcmalloc_internal { 24 | 25 | void RealCentralFreeListForTesting::AllocateBatch(absl::Span batch) { 26 | int total = 0; 27 | 28 | while (total < batch.size()) { 29 | const int to_remove = batch.size() - total; 30 | const int removed = RemoveRange(batch.subspan(total)); 31 | ASSERT_GT(removed, 0); 32 | ASSERT_LE(removed, to_remove); 33 | total += removed; 34 | } 35 | } 36 | 37 | void RealCentralFreeListForTesting::FreeBatch(absl::Span batch) { 38 | InsertRange(batch); 39 | } 40 | 41 | void MinimalFakeCentralFreeList::AllocateBatch(absl::Span batch) { 42 | for (void*& v : batch) v = &v; 43 | } 44 | 45 | void MinimalFakeCentralFreeList::FreeBatch(absl::Span batch) { 46 | for (void* x : batch) TC_CHECK_NE(x, nullptr); 47 | } 48 | 49 | void MinimalFakeCentralFreeList::InsertRange(absl::Span batch) { 50 | absl::base_internal::SpinLockHolder h(&lock_); 51 | FreeBatch(batch); 52 | } 53 | 54 | int MinimalFakeCentralFreeList::RemoveRange(absl::Span batch) { 55 | absl::base_internal::SpinLockHolder h(&lock_); 56 | AllocateBatch(batch); 57 | return batch.size(); 58 | } 59 | 60 | void FakeCentralFreeList::AllocateBatch(absl::Span batch) { 61 | for (int i = 0; i < batch.size(); ++i) { 62 | batch[i] = ::operator new(4); 63 | } 64 | } 65 | 66 | void FakeCentralFreeList::FreeBatch(absl::Span batch) { 67 | for (void* x : batch) { 68 | ::operator delete(x); 69 | } 70 | } 71 | 72 | void FakeCentralFreeList::InsertRange(absl::Span batch) { 73 | FreeBatch(batch); 74 | } 75 | 76 | int FakeCentralFreeList::RemoveRange(absl::Span batch) { 77 | AllocateBatch(batch); 78 | return batch.size(); 79 | } 80 | 81 | } // namespace tcmalloc_internal 82 | } // namespace tcmalloc 83 | -------------------------------------------------------------------------------- /tcmalloc/mock_huge_page_static_forwarder.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | -------------------------------------------------------------------------------- /tcmalloc/mock_metadata_allocator.h: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_MOCK_METADATA_ALLOCATOR_H_ 16 | #define TCMALLOC_MOCK_METADATA_ALLOCATOR_H_ 17 | 18 | #include 19 | #include 20 | 21 | #include "absl/base/attributes.h" 22 | #include "tcmalloc/metadata_allocator.h" 23 | 24 | namespace tcmalloc::tcmalloc_internal { 25 | 26 | class FakeMetadataAllocator final : public MetadataAllocator { 27 | public: 28 | ~FakeMetadataAllocator() override { 29 | for (void* p : metadata_allocs_) { 30 | free(p); 31 | } 32 | } 33 | 34 | [[nodiscard]] void* operator()(size_t size) override { 35 | void* ptr = malloc(size); 36 | metadata_allocs_.push_back(ptr); 37 | return ptr; 38 | } 39 | 40 | private: 41 | std::vector metadata_allocs_; 42 | }; 43 | 44 | } // namespace tcmalloc::tcmalloc_internal 45 | 46 | #endif // TCMALLOC_MOCK_METADATA_ALLOCATOR_H_ 47 | -------------------------------------------------------------------------------- /tcmalloc/mock_transfer_cache.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/mock_transfer_cache.h" 16 | 17 | #include "absl/base/attributes.h" 18 | 19 | namespace tcmalloc { 20 | namespace tcmalloc_internal { 21 | 22 | ABSL_CONST_INIT bool 23 | ArenaBasedFakeTransferCacheManager::partial_legacy_transfer_cache_(false); 24 | ABSL_CONST_INIT bool FakeShardedTransferCacheManager::enable_generic_cache_( 25 | false); 26 | ABSL_CONST_INIT bool 27 | FakeShardedTransferCacheManager::enable_cache_for_large_classes_only_( 28 | false); 29 | } // namespace tcmalloc_internal 30 | } // namespace tcmalloc 31 | -------------------------------------------------------------------------------- /tcmalloc/mock_virtual_allocator.h: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_MOCK_VIRTUAL_ALLOCATOR_H_ 16 | #define TCMALLOC_MOCK_VIRTUAL_ALLOCATOR_H_ 17 | 18 | #include 19 | #include 20 | 21 | #include "absl/base/attributes.h" 22 | #include "tcmalloc/huge_allocator.h" 23 | #include "tcmalloc/huge_pages.h" 24 | #include "tcmalloc/internal/config.h" 25 | #include "tcmalloc/internal/logging.h" 26 | #include "tcmalloc/system-alloc.h" 27 | 28 | namespace tcmalloc::tcmalloc_internal { 29 | 30 | class FakeVirtualAllocator final : public VirtualAllocator { 31 | public: 32 | [[nodiscard]] AddressRange operator()(size_t bytes, size_t align) override; 33 | 34 | static constexpr size_t kMaxBacking = 1024 * 1024; 35 | // This isn't super good form but we'll never have more than one HAT 36 | // extant at once. 37 | std::vector backing_; 38 | 39 | bool should_overallocate_ = false; 40 | HugeLength huge_pages_requested_; 41 | HugeLength huge_pages_received_; 42 | }; 43 | 44 | // Use a tiny fraction of actual size so we can test aggressively. 45 | inline AddressRange FakeVirtualAllocator::operator()(size_t bytes, 46 | size_t align) { 47 | TC_CHECK_EQ(bytes % kHugePageSize, 0); 48 | TC_CHECK_EQ(align % kHugePageSize, 0); 49 | HugeLength req = HLFromBytes(bytes); 50 | huge_pages_requested_ += req; 51 | // Test the case where our sys allocator provides too much. 52 | if (should_overallocate_) ++req; 53 | huge_pages_received_ += req; 54 | // we'll actually provide hidden backing, one word per hugepage. 55 | bytes = req / NHugePages(1); 56 | align /= kHugePageSize; 57 | size_t index = backing_.size(); 58 | if (index % align != 0) { 59 | index += (align - (index & align)); 60 | } 61 | if (index + bytes > kMaxBacking) return {nullptr, 0}; 62 | backing_.resize(index + bytes); 63 | void* ptr = reinterpret_cast(index * kHugePageSize); 64 | return {ptr, req.in_bytes()}; 65 | } 66 | 67 | } // namespace tcmalloc::tcmalloc_internal 68 | 69 | #endif // TCMALLOC_MOCK_VIRTUAL_ALLOCATOR_H_ 70 | -------------------------------------------------------------------------------- /tcmalloc/new_extension.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_NEW_EXTENSION_H_ 16 | #define TCMALLOC_NEW_EXTENSION_H_ 17 | 18 | #include 19 | #include 20 | 21 | #include "tcmalloc/malloc_extension.h" 22 | 23 | #endif // TCMALLOC_NEW_EXTENSION_H_ 24 | -------------------------------------------------------------------------------- /tcmalloc/page_allocator_interface.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/page_allocator_interface.h" 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "tcmalloc/internal/config.h" 23 | #include "tcmalloc/internal/logging.h" 24 | #include "tcmalloc/internal/memory_tag.h" 25 | 26 | GOOGLE_MALLOC_SECTION_BEGIN 27 | namespace tcmalloc { 28 | namespace tcmalloc_internal { 29 | 30 | PageAllocatorInterface::PageAllocatorInterface(const char* label, MemoryTag tag) 31 | : info_(label), tag_(tag) {} 32 | 33 | } // namespace tcmalloc_internal 34 | } // namespace tcmalloc 35 | GOOGLE_MALLOC_SECTION_END 36 | -------------------------------------------------------------------------------- /tcmalloc/pages_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/pages.h" 16 | 17 | #include "benchmark/benchmark.h" 18 | #include "gmock/gmock.h" 19 | #include "gtest/gtest.h" 20 | 21 | namespace tcmalloc::tcmalloc_internal { 22 | namespace { 23 | 24 | TEST(LengthDeathTest, UnderflowChecks) { 25 | const Length kBig = Length(5); 26 | const Length kSmall = Length(1); 27 | 28 | Length tmp = kSmall; 29 | EXPECT_DEBUG_DEATH(tmp -= kBig, "n_.*n_"); 30 | EXPECT_DEBUG_DEATH(benchmark::DoNotOptimize(kSmall - kBig), "n_.*n_"); 31 | } 32 | 33 | } // namespace 34 | } // namespace tcmalloc::tcmalloc_internal 35 | -------------------------------------------------------------------------------- /tcmalloc/profile_marshaler.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/profile_marshaler.h" 16 | 17 | #include 18 | 19 | #include "google/protobuf/io/gzip_stream.h" 20 | #include "google/protobuf/io/zero_copy_stream_impl.h" 21 | #include "tcmalloc/internal/profile_builder.h" 22 | 23 | namespace tcmalloc { 24 | 25 | // Marshal converts a Profile instance into a gzip-encoded, serialized 26 | // representation suitable for viewing with PProf 27 | // (https://github.com/google/pprof). 28 | absl::StatusOr Marshal(const tcmalloc::Profile& profile) { 29 | auto converted_or = tcmalloc_internal::MakeProfileProto(profile); 30 | if (!converted_or.ok()) { 31 | return converted_or.status(); 32 | } 33 | 34 | std::string output; 35 | google::protobuf::io::StringOutputStream stream(&output); 36 | google::protobuf::io::GzipOutputStream gzip_stream(&stream); 37 | if (!(*converted_or)->SerializeToZeroCopyStream(&gzip_stream)) { 38 | return absl::InternalError("Failed to serialize to gzip stream"); 39 | } 40 | if (!gzip_stream.Close()) { 41 | return absl::InternalError("Failed to serialize to gzip stream"); 42 | } 43 | return output; 44 | } 45 | 46 | } // namespace tcmalloc 47 | -------------------------------------------------------------------------------- /tcmalloc/profile_marshaler.h: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_PROFILE_MARSHALER_H_ 16 | #define TCMALLOC_PROFILE_MARSHALER_H_ 17 | 18 | #include 19 | 20 | #include "absl/status/statusor.h" 21 | #include "tcmalloc/malloc_extension.h" 22 | 23 | namespace tcmalloc { 24 | 25 | // Marshal converts a Profile instance into a gzip-encoded, serialized 26 | // representation suitable for viewing with PProf 27 | // (https://github.com/google/pprof). 28 | [[nodiscard]] absl::StatusOr Marshal( 29 | const tcmalloc::Profile& profile); 30 | 31 | } // namespace tcmalloc 32 | 33 | #endif // TCMALLOC_PROFILE_MARSHALER_H_ 34 | -------------------------------------------------------------------------------- /tcmalloc/segv_handler.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_SEGV_HANDLER_H_ 16 | #define TCMALLOC_SEGV_HANDLER_H_ 17 | 18 | #include 19 | 20 | #include "tcmalloc/guarded_allocations.h" 21 | #include "tcmalloc/internal/config.h" 22 | 23 | GOOGLE_MALLOC_SECTION_BEGIN 24 | namespace tcmalloc { 25 | namespace tcmalloc_internal { 26 | 27 | GuardedAllocationsErrorType RefineErrorTypeBasedOnContext( 28 | const void* context, GuardedAllocationsErrorType error); 29 | 30 | void SegvHandler(int signo, siginfo_t* info, void* context); 31 | 32 | } // namespace tcmalloc_internal 33 | } // namespace tcmalloc 34 | GOOGLE_MALLOC_SECTION_END 35 | 36 | #endif // TCMALLOC_SEGV_HANDLER_H_ 37 | -------------------------------------------------------------------------------- /tcmalloc/selsan/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The TCMalloc Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("//tcmalloc:copts.bzl", "TCMALLOC_DEFAULT_COPTS") 16 | 17 | package(default_visibility = ["//visibility:private"]) 18 | 19 | licenses(["notice"]) 20 | 21 | cc_library( 22 | name = "selsan", 23 | srcs = ["selsan.cc"], 24 | hdrs = ["selsan.h"], 25 | copts = TCMALLOC_DEFAULT_COPTS, 26 | features = [ 27 | "-use_header_modules", # to avoid flags mismatch errors (b/327555681) 28 | "-thin_lto", # workaround thin lto crash (b/330654590) 29 | "-cfi", # Requires thin_lto 30 | "-cfi_diag", # Requires thin_lto 31 | "-cfi_dbg", # Requires thin_lto 32 | ], 33 | linkstatic = 1, 34 | visibility = ["//tcmalloc:__subpackages__"], 35 | deps = [ 36 | "//tcmalloc/internal:config", 37 | "//tcmalloc/internal:exponential_biased", 38 | "//tcmalloc/internal:logging", 39 | "@com_google_absl//absl/base:core_headers", 40 | ], 41 | alwayslink = 1, 42 | ) 43 | 44 | cc_test( 45 | name = "shadow_test", 46 | srcs = ["shadow_test.cc"], 47 | tags = ["nosan"], 48 | deps = [ 49 | ":selsan", 50 | "@com_google_googletest//:gtest_main", 51 | ], 52 | ) 53 | 54 | cc_test( 55 | name = "report_test", 56 | srcs = ["report_test.cc"], 57 | deps = [ 58 | ":selsan", 59 | "@com_google_absl//absl/base:core_headers", 60 | "@com_google_absl//absl/strings:str_format", 61 | "@com_google_googletest//:gtest_main", 62 | ], 63 | ) 64 | -------------------------------------------------------------------------------- /tcmalloc/selsan/README.md: -------------------------------------------------------------------------------- 1 | # SelectiveSanitizer (SelSan) 2 | 3 | `SelSan` is implementation of 4 | [HWASAN](https://clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html) 5 | runtime in `TCMalloc`. `SelSan` allows to use `HWASAN` runtime checking for 6 | production applications that use/rely on `TCMalloc`. 7 | 8 | This is currently work-in-progress. 9 | 10 | `SelSan` requires either Arm TBI (top byte ignore), or Intel LAM (linear address 11 | masking), or AMD UAI (upper address ignore) CPU features. `SelSan` can also be 12 | used with Arm MTE (memory tagging extension). 13 | 14 | Reference compiler flags required for `SelSan` are: 15 | 16 | ``` 17 | -DTCMALLOC_INTERNAL_SELSAN=1 18 | -fsanitize=hwaddress 19 | -mllvm -hwasan-globals=0 20 | -mllvm -hwasan-with-tls=0 21 | -mllvm -hwasan-instrument-mem-intrinsics=0 22 | -mllvm -hwasan-memory-access-callback-prefix=__selsan_ 23 | -mllvm -hwasan-mapping-offset=0 24 | -lselsan 25 | ``` 26 | -------------------------------------------------------------------------------- /tcmalloc/size_class_info.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Specification of Size classes 16 | #ifndef TCMALLOC_SIZE_CLASS_INFO_H_ 17 | #define TCMALLOC_SIZE_CLASS_INFO_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "absl/types/span.h" 23 | #include "tcmalloc/internal/config.h" 24 | 25 | GOOGLE_MALLOC_SECTION_BEGIN 26 | namespace tcmalloc { 27 | namespace tcmalloc_internal { 28 | 29 | // Precomputed size class parameters. 30 | struct SizeClassInfo { 31 | // Max size storable in that class 32 | uint32_t size; 33 | 34 | // Number of pages to allocate at a time 35 | uint8_t pages; 36 | 37 | // Number of objects to move between a per-thread list and a central list in 38 | // one shot. We want this to be not too small so we can amortize the lock 39 | // overhead for accessing the central list. Making it too big may temporarily 40 | // cause unnecessary memory wastage in the per-thread free list until the 41 | // scavenger cleans up the list. 42 | uint8_t num_to_move; 43 | }; 44 | 45 | struct SizeClassAssumptions { 46 | bool has_expanded_classes; // kHasExpandedClasses 47 | size_t span_size; // sizeof(Span) 48 | size_t sampling_interval; // kDefaultProfileSamplingInterval 49 | size_t large_size; // SizeMap::kLargeSize 50 | size_t large_size_alignment; // SizeMap::kLargeSizeAlignment 51 | }; 52 | 53 | struct SizeClasses { 54 | absl::Span classes; 55 | SizeClassAssumptions assumptions; 56 | }; 57 | 58 | } // namespace tcmalloc_internal 59 | } // namespace tcmalloc 60 | GOOGLE_MALLOC_SECTION_END 61 | 62 | #endif // TCMALLOC_SIZE_CLASS_INFO_H_ 63 | -------------------------------------------------------------------------------- /tcmalloc/sizemap_fuzz.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "fuzztest/fuzztest.h" 19 | #include "absl/log/check.h" 20 | #include "absl/types/span.h" 21 | #include "tcmalloc/common.h" 22 | #include "tcmalloc/size_class_info.h" 23 | #include "tcmalloc/sizemap.h" 24 | #include "tcmalloc/tcmalloc_policy.h" 25 | 26 | namespace tcmalloc::tcmalloc_internal { 27 | namespace { 28 | 29 | void FuzzSizeMap(const std::string& s) { 30 | if (s.empty()) { 31 | return; 32 | } 33 | 34 | // TODO(b/271282540): Strongly type input to fuzzer. 35 | const char* data = s.data(); 36 | const size_t size = s.size(); 37 | 38 | SizeMap m; 39 | const SizeClassInfo* info = reinterpret_cast(data); 40 | if (!m.Init(absl::MakeSpan(info, size / sizeof(*info)))) { 41 | return; 42 | } 43 | 44 | // Validate that every size on [0, kMaxSize] maps to a size class that is 45 | // neither too big nor too small. 46 | int last_size_class = -1; 47 | for (size_t size = 0; size <= kMaxSize; size++) { 48 | const int size_class = m.SizeClass(CppPolicy(), size); 49 | CHECK_GT(size_class, 0) << size; 50 | CHECK_LT(size_class, kNumClasses) << size; 51 | 52 | const size_t s = m.class_to_size(size_class); 53 | CHECK_LE(size, s); 54 | CHECK_NE(s, 0) << size; 55 | 56 | CHECK_LE(last_size_class, size_class); 57 | last_size_class = size_class; 58 | } 59 | } 60 | 61 | FUZZ_TEST(SizeMapTest, FuzzSizeMap) 62 | ; 63 | 64 | } // namespace 65 | } // namespace tcmalloc::tcmalloc_internal 66 | -------------------------------------------------------------------------------- /tcmalloc/span_stats.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_SPAN_STATS_H_ 16 | #define TCMALLOC_SPAN_STATS_H_ 17 | 18 | #include 19 | 20 | #include "absl/base/optimization.h" 21 | #include "tcmalloc/internal/config.h" 22 | 23 | GOOGLE_MALLOC_SECTION_BEGIN 24 | namespace tcmalloc { 25 | namespace tcmalloc_internal { 26 | 27 | struct SpanStats { 28 | size_t num_spans_requested = 0; 29 | size_t num_spans_returned = 0; 30 | size_t obj_capacity = 0; // cap of number of objs that could be live anywhere 31 | 32 | size_t num_live_spans() { 33 | if (num_spans_requested < num_spans_returned) { 34 | return 0; 35 | } 36 | return num_spans_requested - num_spans_returned; 37 | } 38 | 39 | // Probability that a span will be returned 40 | double prob_returned() { 41 | if (ABSL_PREDICT_FALSE(num_spans_requested == 0)) return 0.0; 42 | return static_cast(num_spans_returned) / num_spans_requested; 43 | } 44 | }; 45 | 46 | } // namespace tcmalloc_internal 47 | } // namespace tcmalloc 48 | GOOGLE_MALLOC_SECTION_END 49 | 50 | #endif // TCMALLOC_SPAN_STATS_H_ 51 | -------------------------------------------------------------------------------- /tcmalloc/testdata/central_freelist_fuzz/clusterfuzz-testcase-central_freelist_fuzz-6338860943802368: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/tcmalloc/testdata/central_freelist_fuzz/clusterfuzz-testcase-central_freelist_fuzz-6338860943802368 -------------------------------------------------------------------------------- /tcmalloc/testdata/central_freelist_fuzz/clusterfuzz-testcase-minimized-central_freelist_fuzz-5328515345809408: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/tcmalloc/testdata/central_freelist_fuzz/clusterfuzz-testcase-minimized-central_freelist_fuzz-5328515345809408 -------------------------------------------------------------------------------- /tcmalloc/testdata/clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-4796454007799808: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/tcmalloc/testdata/clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-4796454007799808 -------------------------------------------------------------------------------- /tcmalloc/testdata/huge_page_aware_allocator_fuzz/clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-4650158169391104: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/tcmalloc/5181028bb5a889ca0d0da941e715906b7ef45029/tcmalloc/testdata/huge_page_aware_allocator_fuzz/clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-4650158169391104 -------------------------------------------------------------------------------- /tcmalloc/testdata/huge_page_aware_allocator_fuzz/clusterfuzz-testcase-minimized-huge_page_aware_allocator_fuzz-5216394376773632: -------------------------------------------------------------------------------- 1 | 2 | 16 | #include 17 | 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | #include "absl/time/clock.h" 22 | #include "absl/time/time.h" 23 | #include "tcmalloc/malloc_extension.h" 24 | #include "tcmalloc/testing/test_allocator_harness.h" 25 | #include "tcmalloc/testing/testutil.h" 26 | #include "tcmalloc/testing/thread_manager.h" 27 | 28 | namespace tcmalloc { 29 | namespace { 30 | 31 | TEST(BackgroundTest, Defaults) { 32 | EXPECT_TRUE(MallocExtension::GetBackgroundProcessActionsEnabled()); 33 | EXPECT_EQ(MallocExtension::GetBackgroundProcessSleepInterval(), 34 | absl::Seconds(1)); 35 | } 36 | 37 | TEST(BackgroundTest, Stress) { 38 | // Process background actions by setting a custom sleep interval. 39 | struct ProcessActions { 40 | static void Go() { 41 | constexpr absl::Duration kSleepTime = absl::Milliseconds(10); 42 | ScopedBackgroundProcessSleepInterval sleep_time(kSleepTime); 43 | MallocExtension::ProcessBackgroundActions(); 44 | } 45 | }; 46 | 47 | // Make sure that background acions are indeed enabled. 48 | EXPECT_TRUE(MallocExtension::GetBackgroundProcessActionsEnabled()); 49 | 50 | std::thread background(ProcessActions::Go); 51 | 52 | constexpr int kThreads = 10; 53 | ThreadManager mgr; 54 | AllocatorHarness harness(kThreads); 55 | 56 | mgr.Start(kThreads, [&](int thread_id) { harness.Run(thread_id); }); 57 | 58 | absl::SleepFor(absl::Seconds(5)); 59 | 60 | mgr.Stop(); 61 | 62 | ScopedBackgroundProcessActionsEnabled background_process_enabled( 63 | /*value=*/false); 64 | background.join(); 65 | } 66 | 67 | } // namespace 68 | } // namespace tcmalloc 69 | 70 | int main(int argc, char** argv) { 71 | testing::InitGoogleTest(&argc, argv); 72 | return RUN_ALL_TESTS(); 73 | } 74 | -------------------------------------------------------------------------------- /tcmalloc/testing/benchmark_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "benchmark/benchmark.h" 16 | 17 | int main(int argc, char* argv[]) { 18 | benchmark::Initialize(&argc, argv); 19 | benchmark::RunSpecifiedBenchmarks(); 20 | return 0; 21 | } 22 | -------------------------------------------------------------------------------- /tcmalloc/testing/current_allocated_bytes_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // This tests the accounting done by tcmalloc. When we allocate and 16 | // free a small buffer, the number of bytes used by the application 17 | // before the alloc+free should match the number of bytes used after. 18 | // However, the internal data structures used by tcmalloc will be 19 | // quite different -- new spans will have been allocated, etc. This 20 | // is, thus, a simple test that we account properly for the internal 21 | // data structures, so that we report the actual application-used 22 | // bytes properly. 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | 30 | #include "absl/strings/string_view.h" 31 | #include "tcmalloc/internal/logging.h" 32 | #include "tcmalloc/malloc_extension.h" 33 | 34 | int main() { 35 | constexpr absl::string_view kCurrent = "generic.current_allocated_bytes"; 36 | size_t before_bytes = 37 | *tcmalloc::MallocExtension::GetNumericProperty(kCurrent); 38 | 39 | free(malloc(200)); 40 | 41 | size_t after_bytes = *tcmalloc::MallocExtension::GetNumericProperty(kCurrent); 42 | TC_CHECK_EQ(before_bytes, after_bytes); 43 | 44 | // Do a lot of different allocs in a lot of different size classes, 45 | // then free them all, to make sure that the logic is correct. 46 | void* ptrs[1000]; // how many pointers to allocate in one run 47 | for (int size = 1; size < 1000000; 48 | size = std::max(size + 1, size * 2 - 100)) { 49 | for (int cycles = 0; cycles < 2; ++cycles) { 50 | for (int repeat = 0; repeat < sizeof(ptrs) / sizeof(*ptrs); ++repeat) { 51 | ptrs[repeat] = malloc(size); 52 | } 53 | for (int repeat = 0; repeat < sizeof(ptrs) / sizeof(*ptrs); ++repeat) { 54 | free(ptrs[repeat]); 55 | } 56 | } 57 | } 58 | 59 | after_bytes = *tcmalloc::MallocExtension::GetNumericProperty(kCurrent); 60 | TC_CHECK_EQ(before_bytes, after_bytes); 61 | 62 | printf("PASS\n"); 63 | return 0; 64 | } 65 | -------------------------------------------------------------------------------- /tcmalloc/testing/disable_numa_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "gtest/gtest.h" 16 | #include "tcmalloc/internal/numa.h" 17 | #include "tcmalloc/static_vars.h" 18 | 19 | namespace { 20 | 21 | TEST(Numa, Inactive) { 22 | EXPECT_FALSE( 23 | tcmalloc::tcmalloc_internal::tc_globals.numa_topology().numa_aware()); 24 | } 25 | 26 | } // namespace 27 | -------------------------------------------------------------------------------- /tcmalloc/testing/fast_path.insecure.golden: -------------------------------------------------------------------------------- 1 | func call div jcc jmp load mul pop push rmw rsp store 2 | aligned_alloc - - 7 - 6 - 2 1 1 1 2 3 | calloc - - 6 - 6 1 2 3 1 2 2 4 | delete - - 4 - 5 - - - - - 3 5 | delete(size) - - 4 - 4 - - - - - 3 6 | delete(size,align) - - 6 - 5 - - - - - 3 7 | malloc - - 5 - 6 - - - 1 - 2 8 | new - - 4 - 5 - - - 1 - 2 9 | new(align) - - 6 - 6 - - - 1 - 2 10 | new(align)->size - - 6 - 7 - - - 1 - 2 11 | new(align,cold)->size - - 6 - 8 - - - 1 - 2 12 | new(align,nothrow) - - 6 - 6 - - - 1 - 2 13 | new(cold)->size - - 4 - 7 - - - 1 - 2 14 | new(nothrow) - - 4 - 5 - - - 1 - 2 15 | new->size - - 4 - 6 - - - 1 - 2 16 | posix_memalign - - 8 - 6 - 3 3 1 2 3 17 | -------------------------------------------------------------------------------- /tcmalloc/testing/fast_path.opt.golden: -------------------------------------------------------------------------------- 1 | func call div jcc jmp load mul pop push rmw rsp store 2 | aligned_alloc - - 7 - 6 - 2 1 1 1 2 3 | calloc - - 6 - 6 1 2 3 1 2 2 4 | delete - - 4 - 5 - - - - - 3 5 | delete(size) - - 4 - 4 - - - - - 3 6 | delete(size,align) - - 6 - 5 - - - - - 3 7 | malloc - - 5 - 6 - - - 1 - 2 8 | new - - 4 - 5 - - - 1 - 2 9 | new(align) - - 6 - 6 - - - 1 - 2 10 | new(align)->size - - 6 - 7 - - - 1 - 2 11 | new(align,cold)->size - - 6 - 8 - 1 1 1 - 2 12 | new(align,nothrow) - - 6 - 6 - - - 1 - 2 13 | new(cold)->size - - 4 - 7 - - - 1 - 2 14 | new(nothrow) - - 4 - 5 - - - 1 - 2 15 | new->size - - 4 - 6 - - - 1 - 2 16 | posix_memalign - - 8 - 6 - 3 3 1 2 3 17 | -------------------------------------------------------------------------------- /tcmalloc/testing/fast_path.release+insecure.golden: -------------------------------------------------------------------------------- 1 | func call div jcc jmp load mul pop push rmw rsp store 2 | aligned_alloc - - 7 - 6 - 2 1 1 1 2 3 | calloc - - 6 - 6 1 2 3 1 2 2 4 | delete - - 4 - 5 - - - - - 3 5 | delete(size) - - 4 - 4 - - - - - 3 6 | delete(size,align) - - 6 - 5 - - - - - 3 7 | malloc - - 5 - 6 - - - 1 - 2 8 | new - - 4 - 5 - - - 1 - 2 9 | new(align) - - 6 - 6 - - - 1 - 2 10 | new(align)->size - - 6 - 7 - - - 1 - 2 11 | new(align,cold)->size - - 6 - 8 - - - 1 - 2 12 | new(align,nothrow) - - 6 - 6 - - - 1 - 2 13 | new(cold)->size - - 4 - 7 - - - 1 - 2 14 | new(nothrow) - - 4 - 5 - - - 1 - 2 15 | new->size - - 4 - 6 - - - 1 - 2 16 | posix_memalign - - 8 - 6 - 3 3 1 2 3 17 | -------------------------------------------------------------------------------- /tcmalloc/testing/fast_path.release.golden: -------------------------------------------------------------------------------- 1 | func call div jcc jmp load mul pop push rmw rsp store 2 | aligned_alloc - - 7 - 6 - 2 1 1 1 2 3 | calloc - - 6 - 6 1 2 3 1 2 2 4 | delete - - 4 - 5 - - - - - 3 5 | delete(size) - - 4 - 4 - - - - - 3 6 | delete(size,align) - - 6 - 5 - - - - - 3 7 | malloc - - 5 - 6 - - - 1 - 2 8 | new - - 4 - 5 - - - 1 - 2 9 | new(align) - - 6 - 6 - - - 1 - 2 10 | new(align)->size - - 6 - 7 - - - 1 - 2 11 | new(align,cold)->size - - 6 - 8 - - - 1 - 2 12 | new(align,nothrow) - - 6 - 6 - - - 1 - 2 13 | new(cold)->size - - 4 - 7 - - - 1 - 2 14 | new(nothrow) - - 4 - 5 - - - 1 - 2 15 | new->size - - 4 - 6 - - - 1 - 2 16 | posix_memalign - - 8 - 6 - 3 3 1 2 3 17 | -------------------------------------------------------------------------------- /tcmalloc/testing/fast_path.unstable.release.golden: -------------------------------------------------------------------------------- 1 | func call div jcc jmp load mul pop push rmw rsp store 2 | aligned_alloc - - 7 - 6 - 2 1 1 1 2 3 | calloc - - 6 - 6 1 2 3 1 2 2 4 | delete - - 4 - 5 - - - - - 3 5 | delete(size) - - 4 - 4 - - - - - 3 6 | delete(size,align) - - 6 - 5 - - - - - 3 7 | malloc - - 5 - 6 - - - 1 - 2 8 | new - - 4 - 5 - - - 1 - 2 9 | new(align) - - 6 - 6 - - - 1 - 2 10 | new(align)->size - - 6 - 7 - - - 1 - 2 11 | new(align,cold)->size - - 6 - 8 - - - 1 - 2 12 | new(align,nothrow) - - 6 - 6 - - - 1 - 2 13 | new(cold)->size - - 5 - 7 - - - 1 - 2 14 | new(nothrow) - - 4 - 5 - - - 1 - 2 15 | new->size - - 4 - 6 - - - 1 - 2 16 | posix_memalign - - 8 - 6 - 3 3 1 2 3 17 | -------------------------------------------------------------------------------- /tcmalloc/testing/frag_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Test speed of handling fragmented heap 16 | 17 | #include 18 | 19 | #include 20 | 21 | #include "gtest/gtest.h" 22 | #include "tcmalloc/common.h" 23 | #include "tcmalloc/malloc_extension.h" 24 | 25 | namespace tcmalloc { 26 | namespace { 27 | 28 | TEST(Fragmentation, Slack) { 29 | // Make kAllocSize one page larger than the maximum small object size. 30 | static const int kAllocSize = 31 | tcmalloc_internal::kMaxSize + tcmalloc_internal::kPageSize; 32 | // Allocate 400MB in total. 33 | static const int kTotalAlloc = 400 << 20; 34 | static const int kAllocIterations = kTotalAlloc / kAllocSize; 35 | 36 | // Allocate lots of objects 37 | std::vector saved(kAllocIterations); 38 | for (int i = 0; i < kAllocIterations; i++) { 39 | saved[i] = ::operator new(kAllocSize); 40 | } 41 | 42 | // Check the current "slack". 43 | size_t slack_before = 44 | *MallocExtension::GetNumericProperty("tcmalloc.slack_bytes"); 45 | 46 | // Free alternating ones to fragment heap 47 | size_t free_bytes = 0; 48 | for (int i = 0; i < saved.size(); i += 2) { 49 | ::operator delete(saved[i]); 50 | free_bytes += kAllocSize; 51 | } 52 | 53 | // Check that slack delta is within 10% of expected. 54 | size_t slack_after = 55 | *MallocExtension::GetNumericProperty("tcmalloc.slack_bytes"); 56 | ASSERT_GE(slack_after, slack_before); 57 | size_t slack = slack_after - slack_before; 58 | 59 | EXPECT_GT(double(slack), 0.9 * free_bytes); 60 | EXPECT_LT(double(slack), 1.1 * free_bytes); 61 | 62 | // Free remaining allocations. 63 | for (int i = 1; i < saved.size(); i += 2) { 64 | ::operator delete(saved[i]); 65 | } 66 | } 67 | 68 | } // namespace 69 | } // namespace tcmalloc 70 | -------------------------------------------------------------------------------- /tcmalloc/testing/hello_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "absl/strings/str_cat.h" 23 | #include "absl/strings/str_format.h" 24 | #include "tcmalloc/malloc_extension.h" 25 | 26 | int main(int argc, char** argv) { 27 | std::string msg = absl::StrCat("hello ", argc < 2 ? "world" : argv[1], "!"); 28 | 29 | std::optional heap_size = 30 | tcmalloc::MallocExtension::GetNumericProperty( 31 | "generic.current_allocated_bytes"); 32 | if (heap_size.has_value()) { 33 | std::cout << "Current heap size = " << *heap_size << " bytes" << '\n'; 34 | } 35 | 36 | std::cout << msg << '\n'; 37 | 38 | // Allocate memory, printing the pointer to deter an optimizing compiler from 39 | // eliding the allocation. 40 | constexpr size_t kSize = 1024 * 1024 * 1024; 41 | std::unique_ptr ptr(new char[kSize]); 42 | 43 | std::cout << absl::StreamFormat("new'd %d bytes at %p\n", kSize, ptr.get()); 44 | 45 | heap_size = tcmalloc::MallocExtension::GetNumericProperty( 46 | "generic.current_allocated_bytes"); 47 | if (heap_size.has_value()) { 48 | std::cout << "Current heap size = " << *heap_size << " bytes" << '\n'; 49 | } 50 | 51 | void* ptr2 = malloc(kSize); 52 | std::cout << absl::StreamFormat("malloc'd %d bytes at %p\n", kSize, ptr2); 53 | 54 | heap_size = tcmalloc::MallocExtension::GetNumericProperty( 55 | "generic.current_allocated_bytes"); 56 | if (heap_size.has_value()) { 57 | std::cout << "Current heap size = " << *heap_size << " bytes" << '\n'; 58 | } 59 | 60 | free(ptr2); 61 | } 62 | -------------------------------------------------------------------------------- /tcmalloc/testing/large_alloc_size_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | 21 | #include "gtest/gtest.h" 22 | #include "tcmalloc/common.h" 23 | #include "tcmalloc/internal/config.h" 24 | #include "tcmalloc/malloc_extension.h" 25 | 26 | namespace tcmalloc { 27 | namespace { 28 | 29 | // Regression test for b/31102171. 30 | // Ensure that when we allocate lots of kHugePageSize + epsilon blocks, 31 | // tcmalloc does not double memory consumption. 32 | TEST(LargeAllocSizeTest, Basic) { 33 | typedef std::map PropertyMap; 34 | PropertyMap map = MallocExtension::GetProperties(); 35 | const size_t start_mem = map["generic.physical_memory_used"].value; 36 | const size_t kTotalToAllocate = 1024 << 20; 37 | const size_t kAllocSize = 38 | tcmalloc_internal::kHugePageSize + tcmalloc_internal::kPageSize; 39 | const size_t kBlocks = kTotalToAllocate / kAllocSize; 40 | void* volatile blocks[kBlocks]; 41 | for (size_t i = 0; i < kBlocks; ++i) { 42 | blocks[i] = malloc(kAllocSize); 43 | } 44 | map = MallocExtension::GetProperties(); 45 | const size_t end_mem = map["generic.physical_memory_used"].value; 46 | for (size_t i = 0; i < kBlocks; ++i) { 47 | free(blocks[i]); 48 | } 49 | EXPECT_LE(end_mem - start_mem, kTotalToAllocate * 1.3) 50 | << "start: " << (start_mem >> 20) << "MB -> " 51 | << "end: " << (end_mem >> 20) << "MB " 52 | << "(+" << ((end_mem - start_mem) >> 20) << "MB)"; 53 | } 54 | 55 | } // namespace 56 | } // namespace tcmalloc 57 | -------------------------------------------------------------------------------- /tcmalloc/testing/malloc_extension_system_malloc_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // These tests assume TCMalloc is not linked in, and therefore the features 16 | // exposed by MallocExtension should be no-ops, but otherwise safe to call. 17 | 18 | #include 19 | #include 20 | 21 | #include "gtest/gtest.h" 22 | #include "absl/random/random.h" 23 | #include "tcmalloc/malloc_extension.h" 24 | 25 | namespace tcmalloc { 26 | namespace { 27 | 28 | TEST(MallocExtension, SnapshotCurrentIsEmpty) { 29 | // Allocate memory to use the allocator. 30 | absl::BitGen gen; 31 | int bytes_remaining = 1 << 24; 32 | std::vector ptrs; 33 | 34 | while (bytes_remaining > 0) { 35 | int size = absl::LogUniform(gen, 0, 1 << 20); 36 | ptrs.push_back(::operator new(size)); 37 | bytes_remaining -= size; 38 | } 39 | 40 | // All of the profiles should be empty. 41 | ProfileType types[] = { 42 | ProfileType::kHeap, 43 | ProfileType::kFragmentation, 44 | ProfileType::kPeakHeap, 45 | ProfileType::kAllocations, 46 | }; 47 | 48 | for (auto t : types) { 49 | SCOPED_TRACE(static_cast(t)); 50 | 51 | Profile p = MallocExtension::SnapshotCurrent(t); 52 | int samples = 0; 53 | p.Iterate([&](const Profile::Sample&) { samples++; }); 54 | 55 | EXPECT_EQ(samples, 0); 56 | } 57 | 58 | for (void* ptr : ptrs) { 59 | ::operator delete(ptr); 60 | } 61 | } 62 | 63 | TEST(MallocExtension, AllocationProfile) { 64 | auto token = MallocExtension::StartAllocationProfiling(); 65 | 66 | // Allocate memory to use the allocator. 67 | absl::BitGen gen; 68 | int bytes_remaining = 1 << 24; 69 | std::vector ptrs; 70 | 71 | while (bytes_remaining > 0) { 72 | int size = absl::LogUniform(gen, 0, 1 << 20); 73 | ptrs.push_back(::operator new(size)); 74 | bytes_remaining -= size; 75 | } 76 | 77 | // Finish profiling and verify the profile is empty. 78 | Profile p = std::move(token).Stop(); 79 | int samples = 0; 80 | p.Iterate([&](const Profile::Sample&) { samples++; }); 81 | 82 | EXPECT_EQ(samples, 0); 83 | 84 | for (void* ptr : ptrs) { 85 | ::operator delete(ptr); 86 | } 87 | } 88 | 89 | } // namespace 90 | } // namespace tcmalloc 91 | -------------------------------------------------------------------------------- /tcmalloc/testing/parallel_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | #include "absl/log/absl_check.h" 22 | #include "absl/random/random.h" 23 | #include "absl/time/clock.h" 24 | #include "absl/time/time.h" 25 | #include "tcmalloc/malloc_extension.h" 26 | 27 | extern "C" { 28 | 29 | void* TCMallocInternalNew(size_t); 30 | void TCMallocInternalDelete(void*); 31 | void TCMallocInternalDeleteSized(void*, size_t); 32 | 33 | } // extern "C" 34 | 35 | namespace tcmalloc { 36 | namespace { 37 | 38 | struct Allocator { 39 | Allocator(std::atomic& stop, bool do_sized_delete) 40 | : stop(stop), do_sized_delete(do_sized_delete) {} 41 | 42 | void operator()() { 43 | const int kNumAllocations = 65536; 44 | std::vector v; 45 | v.reserve(kNumAllocations); 46 | 47 | absl::BitGen rng; 48 | 49 | while (!stop.load(std::memory_order_acquire)) { 50 | const size_t size = 1u << absl::LogUniform(rng, 1, 12); 51 | for (int i = 0; i < kNumAllocations; ++i) { 52 | v.push_back(TCMallocInternalNew(size)); 53 | } 54 | 55 | for (void* ptr : v) { 56 | ABSL_CHECK_GE(*MallocExtension::GetAllocatedSize(ptr), size); 57 | } 58 | 59 | for (void* ptr : v) { 60 | if (do_sized_delete) { 61 | TCMallocInternalDeleteSized(ptr, size); 62 | } else { 63 | TCMallocInternalDelete(ptr); 64 | } 65 | } 66 | v.clear(); 67 | } 68 | } 69 | 70 | std::atomic& stop; 71 | bool do_sized_delete; 72 | }; 73 | 74 | TEST(ParallelTest, Stable) { 75 | std::atomic stop{false}; 76 | Allocator a1(stop, /*do_sized_delete=*/true), 77 | a2(stop, /*do_sized_delete=*/true), a3(stop, /*do_sized_delete=*/false); 78 | 79 | std::thread t1(a1), t2(a2), t3(a3); 80 | 81 | absl::SleepFor(absl::Seconds(1)); 82 | 83 | stop.store(true, std::memory_order_release); 84 | t1.join(); 85 | t2.join(); 86 | t3.join(); 87 | } 88 | 89 | } // namespace 90 | } // namespace tcmalloc 91 | -------------------------------------------------------------------------------- /tcmalloc/testing/thread_ctor_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Test for crashes in tcmalloc during shared library initialization 16 | // http://b/3485510. 17 | 18 | #include 19 | 20 | extern pthread_t* Func(); 21 | 22 | int main() { Func(); } 23 | -------------------------------------------------------------------------------- /tcmalloc/testing/thread_ctor_test_lib.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Test for crashes in tcmalloc during shared library initialization 16 | // http://b/3485510. 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | namespace { 25 | 26 | struct Foo { 27 | int x; 28 | Foo() : x(42) {} 29 | }; 30 | 31 | static void* fn(void*) { 32 | while (true) { 33 | std::vector v; 34 | v.reserve(1000); 35 | for (int i = 0; i < 1000; ++i) { 36 | v.push_back(new Foo); 37 | } 38 | for (int i = 0; i < 1000; ++i) { 39 | assert(v[i]->x == 42); 40 | delete v[i]; 41 | } 42 | } 43 | return nullptr; 44 | } 45 | 46 | #ifndef NTHR 47 | #define NTHR 10 48 | #endif 49 | 50 | static pthread_t Init() { 51 | pthread_t tid[NTHR]; 52 | for (uintptr_t i = 0; i < NTHR; ++i) { 53 | pthread_create(&tid[i], nullptr, fn, (void*)i); 54 | } 55 | return tid[0]; 56 | } 57 | 58 | pthread_t ignored_init_result = Init(); 59 | 60 | } // namespace 61 | 62 | // This is used to pull in this object from archive 63 | // (when built with --dynamic_mode=off). 64 | pthread_t* Func() { return &ignored_init_result; } 65 | -------------------------------------------------------------------------------- /tcmalloc/testing/thread_manager.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_TESTING_THREAD_MANAGER_H_ 16 | #define TCMALLOC_TESTING_THREAD_MANAGER_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "gtest/gtest.h" 24 | #include "absl/synchronization/blocking_counter.h" 25 | 26 | namespace tcmalloc { 27 | 28 | class ThreadManager { 29 | public: 30 | ThreadManager() : shutdown_(false) {} 31 | ~ThreadManager() { 32 | EXPECT_TRUE(shutdown_.load()) << "ThreadManager not stopped"; 33 | } 34 | 35 | // Invokes `func` repeatedly on each of `n` threads until `Stop` is called. 36 | void Start(int n, const std::function& func) { 37 | absl::BlockingCounter started(n); 38 | for (int i = 0; i < n; ++i) { 39 | threads_.emplace_back( 40 | [this, func, &started](int thread_id) { 41 | started.DecrementCount(); 42 | while (!shutdown_.load()) { 43 | func(thread_id); 44 | } 45 | }, 46 | i); 47 | } 48 | started.Wait(); 49 | } 50 | 51 | void Stop() { 52 | shutdown_.store(true); 53 | for (auto& t : threads_) t.join(); 54 | } 55 | 56 | private: 57 | std::atomic shutdown_; 58 | std::vector threads_; 59 | }; 60 | 61 | } // namespace tcmalloc 62 | 63 | #endif // TCMALLOC_TESTING_THREAD_MANAGER_H_ 64 | -------------------------------------------------------------------------------- /tcmalloc/testing/want_disable_huge_region_more_often_test_helper.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "absl/strings/match.h" 19 | #include "absl/strings/string_view.h" 20 | #include "tcmalloc/malloc_extension.h" 21 | 22 | int main(int argc, char** argv) { 23 | std::string input = tcmalloc::MallocExtension::GetStats(); 24 | 25 | if (absl::StrContains(input, "PARAMETER use_huge_region_more_often 1")) { 26 | printf("Active"); 27 | } else { 28 | printf("Inactive"); 29 | } 30 | 31 | return 0; 32 | } 33 | -------------------------------------------------------------------------------- /tcmalloc/testing/want_disable_wider_slabs_test_helper.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | 16 | #include 17 | #include 18 | 19 | #include "absl/strings/match.h" 20 | #include "absl/strings/string_view.h" 21 | #include "tcmalloc/malloc_extension.h" 22 | 23 | int main(int argc, char** argv) { 24 | std::string input = tcmalloc::MallocExtension::GetStats(); 25 | 26 | if (absl::StrContains(input, "PARAMETER tcmalloc_use_wider_slabs 1")) { 27 | printf("Active"); 28 | } else { 29 | printf("Inactive"); 30 | } 31 | 32 | return 0; 33 | } 34 | -------------------------------------------------------------------------------- /tcmalloc/testing/want_hpaa_test_helper.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "absl/log/check.h" 19 | #include "absl/strings/match.h" 20 | #include "absl/strings/numbers.h" 21 | #include "absl/strings/str_split.h" 22 | #include "absl/strings/string_view.h" 23 | #include "absl/strings/strip.h" 24 | #include "tcmalloc/malloc_extension.h" 25 | 26 | int main(int argc, char** argv) { 27 | // Why examine mallocz data, rather than just call decide_want_hpaa? We want 28 | // as close of an end-to-end validation as we can get. 29 | std::string input = tcmalloc::MallocExtension::GetStats(); 30 | bool hpaa = false; 31 | int subrelease = -1; 32 | for (absl::string_view line : absl::StrSplit(input, '\n')) { 33 | if (absl::StrContains(line, "Begin SAMPLED page allocator")) { 34 | // Stop when we reach the end of the main page allocator. We don't 35 | // want to look at the sampled or cold allocator parameters for this 36 | // test. 37 | break; 38 | } 39 | 40 | constexpr absl::string_view kHPAAToken = "HugePageAware"; 41 | hpaa |= absl::StrContains(line, kHPAAToken); 42 | 43 | if (absl::ConsumePrefix(&line, "PARAMETER hpaa_subrelease ")) { 44 | CHECK(absl::SimpleAtoi(line, &subrelease)) << "Could not parse: " << line; 45 | } 46 | } 47 | 48 | printf(hpaa ? "HPAA" : "NoHPAA"); 49 | if (hpaa) { 50 | CHECK_NE(subrelease, -1) << "subrelease parameter not found in mallocz"; 51 | if (subrelease) { 52 | printf("|subrelease"); 53 | } 54 | } else { 55 | CHECK_EQ(subrelease, -1) 56 | << "found unexpected subrelease parameter for non-HPAA allocator"; 57 | } 58 | 59 | return 0; 60 | } 61 | -------------------------------------------------------------------------------- /tcmalloc/testing/want_pow2below64_size_classes_helper.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include "absl/types/span.h" 18 | #include "tcmalloc/common.h" 19 | #include "tcmalloc/internal/logging.h" 20 | #include "tcmalloc/sizemap.h" 21 | #include "tcmalloc/static_vars.h" 22 | 23 | int main(int argc, char** argv) { 24 | const size_t kExpectedClasses[] = {0, 8, 16, 32, 64, 72, 80, 88}; 25 | 26 | absl::Span classes = absl::MakeSpan(kExpectedClasses); 27 | 28 | TC_CHECK_LE(classes.size(), tcmalloc::tcmalloc_internal::kNumClasses); 29 | for (int c = 0; c < classes.size(); ++c) { 30 | if (tcmalloc::tcmalloc_internal::Static::sizemap().class_to_size(c) != 31 | classes[c]) { 32 | printf("Other"); 33 | return 0; 34 | } 35 | } 36 | printf("Pow2Below64"); 37 | 38 | return 0; 39 | } 40 | -------------------------------------------------------------------------------- /tcmalloc/transfer_cache.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "tcmalloc/transfer_cache.h" 16 | 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | #include "absl/base/attributes.h" 23 | #include "absl/types/span.h" 24 | #include "tcmalloc/internal/config.h" 25 | #include "tcmalloc/static_vars.h" 26 | 27 | GOOGLE_MALLOC_SECTION_BEGIN 28 | namespace tcmalloc { 29 | namespace tcmalloc_internal { 30 | 31 | #ifndef TCMALLOC_INTERNAL_SMALL_BUT_SLOW 32 | 33 | size_t StaticForwarder::class_to_size(int size_class) { 34 | return tc_globals.sizemap().class_to_size(size_class); 35 | } 36 | size_t StaticForwarder::num_objects_to_move(int size_class) { 37 | return tc_globals.sizemap().num_objects_to_move(size_class); 38 | } 39 | void *StaticForwarder::Alloc(size_t size, std::align_val_t alignment) { 40 | return tc_globals.arena().Alloc(size, alignment); 41 | } 42 | 43 | ABSL_CONST_INIT bool ShardedStaticForwarder::use_generic_cache_(false); 44 | ABSL_CONST_INIT bool 45 | ShardedStaticForwarder::enable_cache_for_large_classes_only_(false); 46 | 47 | void BackingTransferCache::InsertRange(absl::Span batch) const { 48 | tc_globals.transfer_cache().InsertRange(size_class_, batch); 49 | } 50 | 51 | [[nodiscard]] int BackingTransferCache::RemoveRange( 52 | const absl::Span batch) const { 53 | return tc_globals.transfer_cache().RemoveRange(size_class_, batch); 54 | } 55 | 56 | #endif 57 | 58 | } // namespace tcmalloc_internal 59 | } // namespace tcmalloc 60 | GOOGLE_MALLOC_SECTION_END 61 | -------------------------------------------------------------------------------- /tcmalloc/transfer_cache_stats.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef TCMALLOC_TRANSFER_CACHE_STATS_H_ 16 | #define TCMALLOC_TRANSFER_CACHE_STATS_H_ 17 | 18 | #include 19 | 20 | namespace tcmalloc { 21 | namespace tcmalloc_internal { 22 | 23 | struct TransferCacheStats { 24 | size_t insert_hits; 25 | size_t insert_misses; 26 | size_t insert_object_misses; 27 | size_t remove_hits; 28 | size_t remove_misses; 29 | size_t remove_object_misses; 30 | size_t used; 31 | size_t capacity; 32 | size_t max_capacity; 33 | }; 34 | 35 | } // namespace tcmalloc_internal 36 | } // namespace tcmalloc 37 | 38 | #endif // TCMALLOC_TRANSFER_CACHE_STATS_H_ 39 | -------------------------------------------------------------------------------- /tcmalloc/want_disable_dynamic_slabs.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "absl/base/attributes.h" 16 | 17 | namespace tcmalloc { 18 | namespace tcmalloc_internal { 19 | 20 | // This - if linked into a binary - allows dynamic slabs to be disabled. 21 | ABSL_ATTRIBUTE_UNUSED int default_want_disable_dynamic_slabs() { return 1; } 22 | 23 | } // namespace tcmalloc_internal 24 | } // namespace tcmalloc 25 | -------------------------------------------------------------------------------- /tcmalloc/want_disable_huge_region_more_often.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "absl/base/attributes.h" 16 | 17 | namespace tcmalloc { 18 | namespace tcmalloc_internal { 19 | 20 | // This - if linked into a binary - allows huge-region-more-often feature to be 21 | // disabled. 22 | extern "C" ABSL_ATTRIBUTE_UNUSED bool 23 | default_want_disable_huge_region_more_often() { 24 | return true; 25 | } 26 | 27 | } // namespace tcmalloc_internal 28 | } // namespace tcmalloc 29 | -------------------------------------------------------------------------------- /tcmalloc/want_hpaa.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "absl/base/attributes.h" 16 | #include "tcmalloc/internal/config.h" 17 | 18 | GOOGLE_MALLOC_SECTION_BEGIN 19 | namespace tcmalloc { 20 | namespace tcmalloc_internal { 21 | 22 | // This -if linked into a binary - overrides page_allocator.cc and forces HPAA 23 | // subrelease off. 24 | ABSL_ATTRIBUTE_UNUSED int default_subrelease() { return -1; } 25 | 26 | } // namespace tcmalloc_internal 27 | } // namespace tcmalloc 28 | GOOGLE_MALLOC_SECTION_END 29 | -------------------------------------------------------------------------------- /tcmalloc/want_legacy_size_classes.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "absl/base/attributes.h" 16 | 17 | namespace tcmalloc { 18 | namespace tcmalloc_internal { 19 | 20 | // This - if linked into a binary - overrides common.cc and forces old size 21 | // classes. 22 | ABSL_ATTRIBUTE_UNUSED int default_want_legacy_size_classes() { return 1; } 23 | 24 | } // namespace tcmalloc_internal 25 | } // namespace tcmalloc 26 | -------------------------------------------------------------------------------- /tcmalloc/want_legacy_size_classes_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include "gtest/gtest.h" 18 | #include "absl/types/span.h" 19 | #include "tcmalloc/common.h" 20 | #include "tcmalloc/sizemap.h" 21 | #include "tcmalloc/static_vars.h" 22 | 23 | namespace tcmalloc { 24 | namespace tcmalloc_internal { 25 | namespace { 26 | 27 | TEST(SizeClassesTest, SmallClasses) { 28 | // This test needs to validate against the actual SizeMap TCMalloc will use. 29 | tc_globals.InitIfNecessary(); 30 | 31 | if (__STDCPP_DEFAULT_NEW_ALIGNMENT__ > 8) 32 | GTEST_SKIP() << "Unexpected default new alignment."; 33 | 34 | const size_t kExpectedClasses[] = {0, 8, 16, 24, 32, 40, 48, 56, 64}; 35 | 36 | absl::Span classes = absl::MakeSpan(kExpectedClasses); 37 | 38 | ASSERT_LE(classes.size(), kNumClasses); 39 | for (int c = 0; c < classes.size(); ++c) { 40 | EXPECT_EQ(tc_globals.sizemap().class_to_size(c), classes[c]) << c; 41 | } 42 | } 43 | 44 | } // namespace 45 | } // namespace tcmalloc_internal 46 | } // namespace tcmalloc 47 | -------------------------------------------------------------------------------- /tcmalloc/want_numa_aware.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The TCMalloc Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "absl/base/attributes.h" 16 | #include "tcmalloc/internal/config.h" 17 | 18 | GOOGLE_MALLOC_SECTION_BEGIN 19 | namespace tcmalloc { 20 | namespace tcmalloc_internal { 21 | 22 | // When linked into a binary this overrides the weak implementation in numa.cc 23 | // and causes TCMalloc to enable NUMA awareness by default. 24 | ABSL_ATTRIBUTE_UNUSED bool default_want_numa_aware() { return true; } 25 | 26 | } // namespace tcmalloc_internal 27 | } // namespace tcmalloc 28 | GOOGLE_MALLOC_SECTION_END 29 | --------------------------------------------------------------------------------