From be21842361fd89d63660c52a256d034364969ac5 Mon Sep 17 00:00:00 2001 From: Vaibhav Gogte Date: Wed, 22 Jul 2026 13:41:22 -0700 Subject: [PATCH] Release as much memory as possible from cold heap filler. PiperOrigin-RevId: 952305994 --- tcmalloc/huge_page_aware_allocator.h | 11 +++++- tcmalloc/huge_page_aware_allocator_fuzz.cc | 22 +++++++++-- tcmalloc/huge_page_aware_allocator_test.cc | 43 ++++++++++++++++++++++ tcmalloc/internal/parameter_accessors.h | 2 + tcmalloc/mock_huge_page_static_forwarder.h | 5 +++ tcmalloc/parameters.cc | 9 +++++ tcmalloc/parameters.h | 10 +++++ 7 files changed, 97 insertions(+), 5 deletions(-) diff --git a/tcmalloc/huge_page_aware_allocator.h b/tcmalloc/huge_page_aware_allocator.h index 025a8d18e..9df38183a 100644 --- a/tcmalloc/huge_page_aware_allocator.h +++ b/tcmalloc/huge_page_aware_allocator.h @@ -86,6 +86,10 @@ class StaticForwarder { return Parameters::huge_region_adaptive_release(); } + static bool release_max_cold_pages() { + return Parameters::release_max_cold_pages(); + } + // Arena state. static Arena& arena(); @@ -1034,9 +1038,12 @@ inline Length HugePageAwareAllocator::ReleaseAtLeastNPages( // THP coverage. It is however very useful to have the ability to turn this on // for testing. if (hpaa_subrelease()) { - if (released < num_pages) { + const bool release_max_cold = + tag_ == MemoryTag::kCold && forwarder_.release_max_cold_pages(); + if (released < num_pages || release_max_cold) { + Length desired = release_max_cold ? Length::max() : num_pages - released; released += filler_.ReleasePages( - num_pages - released, + desired, SkipSubreleaseIntervals{ .short_interval = forwarder_.filler_skip_subrelease_short_interval(), diff --git a/tcmalloc/huge_page_aware_allocator_fuzz.cc b/tcmalloc/huge_page_aware_allocator_fuzz.cc index 72fce282e..940decb6f 100644 --- a/tcmalloc/huge_page_aware_allocator_fuzz.cc +++ b/tcmalloc/huge_page_aware_allocator_fuzz.cc @@ -281,6 +281,15 @@ struct SetEnableUnfilteredCollapse { } }; +struct SetReleaseMaxColdPages { + bool value; + + template + friend void AbslStringify(Sink& sink, const SetReleaseMaxColdPages& s) { + absl::Format(&sink, "SetReleaseMaxColdPages{.value=%v}", s.value); + } +}; + struct Instruction; template @@ -295,7 +304,7 @@ using ParamOp = std::variant< SetFillerSkipSubreleaseLongInterval, SetReleasePartialAllocPages, SetHpaaSubrelease, SetReleaseSucceeds, SetHugeRegionDemandBasedRelease, SetHugeRegionAdaptiveRelease, SetBackAllocations, SetBackSizeThresholdBytes, - ReentrantSubprogram, SetEnableUnfilteredCollapse>; + ReentrantSubprogram, SetEnableUnfilteredCollapse, SetReleaseMaxColdPages>; template void AbslStringify(Sink& sink, const ParamOp& p) { @@ -588,6 +597,9 @@ void FuzzHPAA(FuzzHugePageAwareAllocatorOptions fuzz_options, } else if constexpr (std::is_same_v< P, SetEnableUnfilteredCollapse>) { forwarder.set_enable_unfiltered_collapse(param_arg.value); + } else if constexpr (std::is_same_v< + P, SetReleaseMaxColdPages>) { + forwarder.set_release_max_cold_pages(param_arg.value); } }, arg.op); @@ -726,7 +738,9 @@ fuzztest::Domain GetChangeParamDomain(int depth) { .WithSize(0)), fuzztest::Map( [](SetEnableUnfilteredCollapse s) { return ChangeParam{s}; }, - fuzztest::Arbitrary())); + fuzztest::Arbitrary()), + fuzztest::Map([](SetReleaseMaxColdPages s) { return ChangeParam{s}; }, + fuzztest::Arbitrary())); } else { return fuzztest::OneOf( fuzztest::Map([](ResetSubreleaseIntervals r) { return ChangeParam{r}; }, @@ -767,7 +781,9 @@ fuzztest::Domain GetChangeParamDomain(int depth) { fuzztest::VectorOf(GetInstructionDomain(depth - 1))), fuzztest::Map( [](SetEnableUnfilteredCollapse s) { return ChangeParam{s}; }, - fuzztest::Arbitrary())); + fuzztest::Arbitrary()), + fuzztest::Map([](SetReleaseMaxColdPages s) { return ChangeParam{s}; }, + fuzztest::Arbitrary())); } } diff --git a/tcmalloc/huge_page_aware_allocator_test.cc b/tcmalloc/huge_page_aware_allocator_test.cc index e20984473..af6ac8e12 100644 --- a/tcmalloc/huge_page_aware_allocator_test.cc +++ b/tcmalloc/huge_page_aware_allocator_test.cc @@ -2091,6 +2091,49 @@ TEST_F(GetReleaseStatsTest, b339535705) { })); } +TEST(HugePageAwareAllocatorTest, ReleaseMaxColdPages) { + constexpr SpanAllocInfo kAllocInfo = { + .objects_per_span = 1, + .density = AccessDensityPrediction::kSparse, + }; + constexpr Length kAllocPages = kPagesPerHugePage / 2; + + for (bool release_max_cold_pages : {false, true}) { + FakeHugePageAwareAllocator cold_allocator({.tag = MemoryTag::kCold}); + cold_allocator.forwarder().set_filler_skip_subrelease_short_interval( + absl::ZeroDuration()); + cold_allocator.forwarder().set_filler_skip_subrelease_long_interval( + absl::ZeroDuration()); + cold_allocator.forwarder().set_release_max_cold_pages( + release_max_cold_pages); + + Span* s1 = cold_allocator.New(kAllocPages, kAllocInfo); + Span* s2 = cold_allocator.New(kAllocPages, kAllocInfo); + Span* s3 = cold_allocator.New(kAllocPages, kAllocInfo); + Span* s4 = cold_allocator.New(kAllocPages, kAllocInfo); + + SpanDeleter deleter(&cold_allocator); + deleter(s1); + deleter(s3); + + Length released; + { + PageHeapSpinLockHolder l; + released = cold_allocator.ReleaseAtLeastNPages( + kAllocPages, PageReleaseReason::kReleaseMemoryToSystem); + } + + if (release_max_cold_pages) { + EXPECT_EQ(released, 2 * kAllocPages); + } else { + EXPECT_EQ(released, kAllocPages); + } + + deleter(s2); + deleter(s4); + } +} + } // namespace } // namespace tcmalloc_internal } // namespace tcmalloc diff --git a/tcmalloc/internal/parameter_accessors.h b/tcmalloc/internal/parameter_accessors.h index 9f474226c..8d93888cd 100644 --- a/tcmalloc/internal/parameter_accessors.h +++ b/tcmalloc/internal/parameter_accessors.h @@ -113,6 +113,8 @@ ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetHugeRegionAdaptiveReleaseEnabled(); ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetHugeRegionAdaptiveReleaseEnabled( bool v); +ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetReleaseMaxColdPages(); +ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetReleaseMaxColdPages(bool v); ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_GetSizeClasses( std::vector* absl_nonnull diff --git a/tcmalloc/mock_huge_page_static_forwarder.h b/tcmalloc/mock_huge_page_static_forwarder.h index 78d5e42cc..15e995cf7 100644 --- a/tcmalloc/mock_huge_page_static_forwarder.h +++ b/tcmalloc/mock_huge_page_static_forwarder.h @@ -91,6 +91,10 @@ class FakeStaticForwarder { void set_huge_region_adaptive_release(bool value) { huge_region_adaptive_release_ = value; } + bool release_max_cold_pages() const { return release_max_cold_pages_; } + void set_release_max_cold_pages(bool value) { + release_max_cold_pages_ = value; + } bool BackAllocations() const { return back_allocations_; } void SetBackAllocations(bool value) { back_allocations_ = value; } @@ -228,6 +232,7 @@ class FakeStaticForwarder { int error_number_ = 0; bool huge_region_demand_based_release_ = false; bool huge_region_adaptive_release_ = false; + bool release_max_cold_pages_ = false; bool back_allocations_ = false; int32_t back_size_threshold_bytes_ = kPageSize; diff --git a/tcmalloc/parameters.cc b/tcmalloc/parameters.cc index 6328f5b8d..2b1ab13bd 100644 --- a/tcmalloc/parameters.cc +++ b/tcmalloc/parameters.cc @@ -250,6 +250,7 @@ ABSL_CONST_INIT std::atomic Parameters::back_size_threshold_bytes_( kPageSize); ABSL_CONST_INIT std::atomic Parameters::enable_unfiltered_collapse_( false); +ABSL_CONST_INIT std::atomic Parameters::release_max_cold_pages_(false); static std::atomic& huge_region_adaptive_release_enabled() { ABSL_CONST_INIT static absl::once_flag flag; ABSL_CONST_INIT static std::atomic v{false}; @@ -664,6 +665,14 @@ void TCMalloc_Internal_SetHugeRegionAdaptiveReleaseEnabled(bool v) { v, std::memory_order_relaxed); } +bool TCMalloc_Internal_GetReleaseMaxColdPages() { + return Parameters::release_max_cold_pages(); +} + +void TCMalloc_Internal_SetReleaseMaxColdPages(bool v) { + Parameters::release_max_cold_pages_.store(v, std::memory_order_relaxed); +} + } // extern "C" GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/parameters.h b/tcmalloc/parameters.h index 378531591..65918df6f 100644 --- a/tcmalloc/parameters.h +++ b/tcmalloc/parameters.h @@ -131,6 +131,14 @@ class Parameters { static bool huge_region_adaptive_release(); + static bool release_max_cold_pages() { + return release_max_cold_pages_.load(std::memory_order_relaxed); + } + + static void set_release_max_cold_pages(bool value) { + TCMalloc_Internal_SetReleaseMaxColdPages(value); + } + static void set_per_cpu_caches(bool value) { #if !defined(TCMALLOC_DEPRECATED_PERTHREAD) if (!value) { @@ -221,6 +229,7 @@ class Parameters { friend void ::TCMalloc_Internal_SetBackSizeThresholdBytes(int32_t v); friend void ::TCMalloc_Internal_SetEnableUnfilteredCollapse(bool v); friend void ::TCMalloc_Internal_SetHugeRegionAdaptiveReleaseEnabled(bool v); + friend void ::TCMalloc_Internal_SetReleaseMaxColdPages(bool v); static std::atomic guarded_sampling_interval_; static std::atomic max_per_cpu_cache_size_; @@ -240,6 +249,7 @@ class Parameters { static std::atomic back_size_threshold_bytes_; static std::atomic enable_unfiltered_collapse_; + static std::atomic release_max_cold_pages_; }; } // namespace tcmalloc_internal