Skip to content

Release as much memory as possible from cold heap filler. #413

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Jul 22, 2026
Merged

Release as much memory as possible from cold heap filler. #413

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions tcmalloc/huge_page_aware_allocator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ class StaticForwarder {
return Parameters::huge_region_adaptive_release();
}

static bool release_max_cold_pages() {
return Parameters::release_max_cold_pages();
}

// Arena state.
static Arena& arena();

Expand Down Expand Up @@ -1034,9 +1038,12 @@ inline Length HugePageAwareAllocator<Forwarder>::ReleaseAtLeastNPages(
// THP coverage. It is however very useful to have the ability to turn this on
// for testing.
if (hpaa_subrelease()) {
if (released < num_pages) {
const bool release_max_cold =
tag_ == MemoryTag::kCold && forwarder_.release_max_cold_pages();
if (released < num_pages || release_max_cold) {
Length desired = release_max_cold ? Length::max() : num_pages - released;
released += filler_.ReleasePages(
num_pages - released,
desired,
SkipSubreleaseIntervals{
.short_interval =
forwarder_.filler_skip_subrelease_short_interval(),
Expand Down
22 changes: 19 additions & 3 deletions tcmalloc/huge_page_aware_allocator_fuzz.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,15 @@ struct SetEnableUnfilteredCollapse {
}
};

struct SetReleaseMaxColdPages {
bool value;

template <typename Sink>
friend void AbslStringify(Sink& sink, const SetReleaseMaxColdPages& s) {
absl::Format(&sink, "SetReleaseMaxColdPages{.value=%v}", s.value);
}
};

struct Instruction;

template <typename Sink>
Expand All @@ -295,7 +304,7 @@ using ParamOp = std::variant<
SetFillerSkipSubreleaseLongInterval, SetReleasePartialAllocPages,
SetHpaaSubrelease, SetReleaseSucceeds, SetHugeRegionDemandBasedRelease,
SetHugeRegionAdaptiveRelease, SetBackAllocations, SetBackSizeThresholdBytes,
ReentrantSubprogram, SetEnableUnfilteredCollapse>;
ReentrantSubprogram, SetEnableUnfilteredCollapse, SetReleaseMaxColdPages>;

template <typename Sink>
void AbslStringify(Sink& sink, const ParamOp& p) {
Expand Down Expand Up @@ -588,6 +597,9 @@ void FuzzHPAA(FuzzHugePageAwareAllocatorOptions fuzz_options,
} else if constexpr (std::is_same_v<
P, SetEnableUnfilteredCollapse>) {
forwarder.set_enable_unfiltered_collapse(param_arg.value);
} else if constexpr (std::is_same_v<
P, SetReleaseMaxColdPages>) {
forwarder.set_release_max_cold_pages(param_arg.value);
}
},
arg.op);
Expand Down Expand Up @@ -726,7 +738,9 @@ fuzztest::Domain<ChangeParam> GetChangeParamDomain(int depth) {
.WithSize(0)),
fuzztest::Map(
[](SetEnableUnfilteredCollapse s) { return ChangeParam{s}; },
fuzztest::Arbitrary<SetEnableUnfilteredCollapse>()));
fuzztest::Arbitrary<SetEnableUnfilteredCollapse>()),
fuzztest::Map([](SetReleaseMaxColdPages s) { return ChangeParam{s}; },
fuzztest::Arbitrary<SetReleaseMaxColdPages>()));
} else {
return fuzztest::OneOf(
fuzztest::Map([](ResetSubreleaseIntervals r) { return ChangeParam{r}; },
Expand Down Expand Up @@ -767,7 +781,9 @@ fuzztest::Domain<ChangeParam> GetChangeParamDomain(int depth) {
fuzztest::VectorOf(GetInstructionDomain(depth - 1))),
fuzztest::Map(
[](SetEnableUnfilteredCollapse s) { return ChangeParam{s}; },
fuzztest::Arbitrary<SetEnableUnfilteredCollapse>()));
fuzztest::Arbitrary<SetEnableUnfilteredCollapse>()),
fuzztest::Map([](SetReleaseMaxColdPages s) { return ChangeParam{s}; },
fuzztest::Arbitrary<SetReleaseMaxColdPages>()));
}
}

Expand Down
43 changes: 43 additions & 0 deletions tcmalloc/huge_page_aware_allocator_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2091,6 +2091,49 @@ TEST_F(GetReleaseStatsTest, b339535705) {
}));
}

TEST(HugePageAwareAllocatorTest, ReleaseMaxColdPages) {
constexpr SpanAllocInfo kAllocInfo = {
.objects_per_span = 1,
.density = AccessDensityPrediction::kSparse,
};
constexpr Length kAllocPages = kPagesPerHugePage / 2;

for (bool release_max_cold_pages : {false, true}) {
FakeHugePageAwareAllocator cold_allocator({.tag = MemoryTag::kCold});
cold_allocator.forwarder().set_filler_skip_subrelease_short_interval(
absl::ZeroDuration());
cold_allocator.forwarder().set_filler_skip_subrelease_long_interval(
absl::ZeroDuration());
cold_allocator.forwarder().set_release_max_cold_pages(
release_max_cold_pages);

Span* s1 = cold_allocator.New(kAllocPages, kAllocInfo);
Span* s2 = cold_allocator.New(kAllocPages, kAllocInfo);
Span* s3 = cold_allocator.New(kAllocPages, kAllocInfo);
Span* s4 = cold_allocator.New(kAllocPages, kAllocInfo);

SpanDeleter deleter(&cold_allocator);
deleter(s1);
deleter(s3);

Length released;
{
PageHeapSpinLockHolder l;
released = cold_allocator.ReleaseAtLeastNPages(
kAllocPages, PageReleaseReason::kReleaseMemoryToSystem);
}

if (release_max_cold_pages) {
EXPECT_EQ(released, 2 * kAllocPages);
} else {
EXPECT_EQ(released, kAllocPages);
}

deleter(s2);
deleter(s4);
}
}

} // namespace
} // namespace tcmalloc_internal
} // namespace tcmalloc
2 changes: 2 additions & 0 deletions tcmalloc/internal/parameter_accessors.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,8 @@ ABSL_ATTRIBUTE_WEAK bool
TCMalloc_Internal_GetHugeRegionAdaptiveReleaseEnabled();
ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetHugeRegionAdaptiveReleaseEnabled(
bool v);
ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetReleaseMaxColdPages();
ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetReleaseMaxColdPages(bool v);

ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_GetSizeClasses(
std::vector<tcmalloc::tcmalloc_internal::TracerSizeClassInfo>* absl_nonnull
Expand Down
5 changes: 5 additions & 0 deletions tcmalloc/mock_huge_page_static_forwarder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,10 @@ class FakeStaticForwarder {
void set_huge_region_adaptive_release(bool value) {
huge_region_adaptive_release_ = value;
}
bool release_max_cold_pages() const { return release_max_cold_pages_; }
void set_release_max_cold_pages(bool value) {
release_max_cold_pages_ = value;
}

bool BackAllocations() const { return back_allocations_; }
void SetBackAllocations(bool value) { back_allocations_ = value; }
Expand Down Expand Up @@ -228,6 +232,7 @@ class FakeStaticForwarder {
int error_number_ = 0;
bool huge_region_demand_based_release_ = false;
bool huge_region_adaptive_release_ = false;
bool release_max_cold_pages_ = false;

bool back_allocations_ = false;
int32_t back_size_threshold_bytes_ = kPageSize;
Expand Down
9 changes: 9 additions & 0 deletions tcmalloc/parameters.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,7 @@ ABSL_CONST_INIT std::atomic<int32_t> Parameters::back_size_threshold_bytes_(
kPageSize);
ABSL_CONST_INIT std::atomic<bool> Parameters::enable_unfiltered_collapse_(
false);
ABSL_CONST_INIT std::atomic<bool> Parameters::release_max_cold_pages_(false);
static std::atomic<bool>& huge_region_adaptive_release_enabled() {
ABSL_CONST_INIT static absl::once_flag flag;
ABSL_CONST_INIT static std::atomic<bool> v{false};
Expand Down Expand Up @@ -664,6 +665,14 @@ void TCMalloc_Internal_SetHugeRegionAdaptiveReleaseEnabled(bool v) {
v, std::memory_order_relaxed);
}

bool TCMalloc_Internal_GetReleaseMaxColdPages() {
return Parameters::release_max_cold_pages();
}

void TCMalloc_Internal_SetReleaseMaxColdPages(bool v) {
Parameters::release_max_cold_pages_.store(v, std::memory_order_relaxed);
}

} // extern "C"

GOOGLE_MALLOC_SECTION_END
10 changes: 10 additions & 0 deletions tcmalloc/parameters.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,14 @@ class Parameters {

static bool huge_region_adaptive_release();

static bool release_max_cold_pages() {
return release_max_cold_pages_.load(std::memory_order_relaxed);
}

static void set_release_max_cold_pages(bool value) {
TCMalloc_Internal_SetReleaseMaxColdPages(value);
}

static void set_per_cpu_caches(bool value) {
#if !defined(TCMALLOC_DEPRECATED_PERTHREAD)
if (!value) {
Expand Down Expand Up @@ -221,6 +229,7 @@ class Parameters {
friend void ::TCMalloc_Internal_SetBackSizeThresholdBytes(int32_t v);
friend void ::TCMalloc_Internal_SetEnableUnfilteredCollapse(bool v);
friend void ::TCMalloc_Internal_SetHugeRegionAdaptiveReleaseEnabled(bool v);
friend void ::TCMalloc_Internal_SetReleaseMaxColdPages(bool v);

static std::atomic<int64_t> guarded_sampling_interval_;
static std::atomic<int32_t> max_per_cpu_cache_size_;
Expand All @@ -240,6 +249,7 @@ class Parameters {

static std::atomic<int32_t> back_size_threshold_bytes_;
static std::atomic<bool> enable_unfiltered_collapse_;
static std::atomic<bool> release_max_cold_pages_;
};

} // namespace tcmalloc_internal
Expand Down
Loading