Skip to content

Refactor sizemap class array regions and systematize terminology #631

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Sep 2, 2026
+63 −94
Merged

Refactor sizemap class array regions and systematize terminology #631

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 30 additions & 67 deletions tcmalloc/sizemap.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,91 +238,54 @@ bool SizeMap::Init(absl::Span<const SizeClassInfo> size_classes) {
return false;
}

int next_size = 0;
for (int c = 1; c < kNumClasses; c++) {
const int max_size_in_class = class_to_size_[c];

for (int s = next_size; s <= max_size_in_class;
s += static_cast<size_t>(kAlignment)) {
// Fill in the canonical class array in region 0.
for (int c = 1, s = 0; c < kNumClasses && s <= kMaxSize; c++) {
for (; s <= class_to_size_[c]; s += static_cast<size_t>(kAlignment)) {
class_array_[ClassIndex(s)] = c;
}
next_size = max_size_in_class + static_cast<size_t>(kAlignment);
if (next_size > kMaxSize) {
break;
}
}

// Point all lookups in hot registers (Malloc P0, New P1, Malloc P1)
// directly to New's P0. We only overwrite the lookups if heap
// Point all lookups in hot regions (Malloc R0, New R1, Malloc R1)
// directly to New R0. We only overwrite the lookups if heap
// partitioning is active with the dedicated size classes.
for (size_t i = 1; i < kHotRegisters; ++i) {
std::copy(&class_array_[0], &class_array_[kClassArraySize],
&class_array_[kClassArraySize * i]);
for (size_t i = 1; i < kHotRegions; ++i) {
SetClassArrayRegion(i, 0);
}

const bool heap_partitioning_active =
tc_globals.multiple_non_numa_partitions();
const bool heap_partitioning_full =
Parameters::heap_partitioning_mode() == HeapPartitioningMode::kFull;
if (kSecurityPartitions > 1 && heap_partitioning_active) {
next_size = 0;
for (int c = kNumBaseClasses + 1; c < kColdClassesStart; ++c) {
const int max_size_in_class = class_to_size_[c];

for (int s = next_size; s <= max_size_in_class;
s += static_cast<size_t>(kAlignment)) {
// Route Hot Malloc P1 to security partition P1.
class_array_[ClassIndex(s) +
kClassArraySize * (kSecurityPartitions + 1)] = c;
// Route Hot New P1 to security partition P1.
class_array_[ClassIndex(s) + kClassArraySize] = c;
if (heap_partitioning_full) continue;
// In kLight mode, route Hot New P0 to P1.
class_array_[ClassIndex(s)] = c;
}
next_size = max_size_in_class + static_cast<size_t>(kAlignment);
if (next_size > kMaxSize) {
break;
}
if (ColdFeatureActive()) {
SetClassArrayRegion(kColdRegionsStart, +kColdClassesStart);
if (kSecurityPartitions > 1) {
// Point all lookups in Cold New R1 to either Hot New R1 or Cold New R0.
SetClassArrayRegion(kColdRegionsStart + 1, heap_partitioning_full
? +kNumBaseClasses
: +kColdClassesStart);
}
}

if (ColdFeatureActive()) {
next_size = 0;
for (int c = kColdClassesStart + 1; c < kNumClasses; c++) {
size_t max_size_in_class = class_to_size_[c];
if (max_size_in_class == 0) {
next_size = max_size_in_class + static_cast<size_t>(kAlignment);
continue;
}

for (int s = next_size; s <= max_size_in_class;
s += static_cast<size_t>(kAlignment)) {
class_array_[ClassIndex(s) + kClassArraySize * kColdRegisterStride] = c;
}
next_size = max_size_in_class + static_cast<size_t>(kAlignment);
if (next_size > kMaxSize) {
break;
}
}
if (kSecurityPartitions > 1) {
if (heap_partitioning_full) {
// Point all lookups in Cold New's P1 register to Hot New's P1.
std::copy(&class_array_[kClassArraySize],
&class_array_[kClassArraySize * 2],
&class_array_[kClassArraySize * (kColdRegisterStride + 1)]);
} else {
// Point all lookups in Cold New's P1 register to Cold New's P0.
std::copy(&class_array_[kClassArraySize * kColdRegisterStride],
&class_array_[kClassArraySize * (kColdRegisterStride + 1)],
&class_array_[kClassArraySize * (kColdRegisterStride + 1)]);
}
if (kSecurityPartitions > 1 && tc_globals.multiple_non_numa_partitions()) {
// Route Hot Malloc R1 to security partition P1.
SetClassArrayRegion(kSecurityPartitions + 1, +kNumBaseClasses);
// Route Hot New R1 to security partition P1.
SetClassArrayRegion(1, +kNumBaseClasses);
if (!heap_partitioning_full) {
// In kLight mode, route Hot New R0 to P1.
SetClassArrayRegion(0, +kNumBaseClasses);
}
}

return true;
}

void SizeMap::SetClassArrayRegion(size_t region, CompactSizeClass adjust) {
// Ensure R0 is the canonical non-adjusted array.
TC_CHECK_EQ(class_array_[0], 1);
for (size_t i = 0; i < kClassArraySize; ++i) {
class_array_[region * kClassArraySize + i] = class_array_[i] + adjust;
}
}

} // namespace tcmalloc_internal
} // namespace tcmalloc

Expand Down
60 changes: 33 additions & 27 deletions tcmalloc/sizemap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,38 +89,40 @@ class SizeMap {
// If TCMalloc is compiled without NUMA support, and with cold allocations
// (cold classes), then the class_array_ will consist of 6 regions:
//
// [0, kClassArraySize) : Hot New P0
// [kClassArraySize, 2*kClassArraySize) : Hot New P1
// [2*kClassArraySize, 3*kClassArraySize) : Hot Malloc P0
// [3*kClassArraySize, 4*kClassArraySize) : Hot Malloc P1
// [4*kClassArraySize, 5*kClassArraySize) : Cold New P0
// [5*kClassArraySize, 6*kClassArraySize) : Cold New P1
// [0, kClassArraySize) : Hot New R0
// [kClassArraySize, 2*kClassArraySize) : Hot New R1
// [2*kClassArraySize, 3*kClassArraySize) : Hot Malloc R0
// [3*kClassArraySize, 4*kClassArraySize) : Hot Malloc R1
// [4*kClassArraySize, 5*kClassArraySize) : Cold New R0
// [5*kClassArraySize, 6*kClassArraySize) : Cold New R1
//
// * If the heap partitioning feature is not active, then the lookups for
// partition 1 will contain the same information as for partition 0.
// region 1 will contain the same information as for region 0,
// and both Hot New and Hot Malloc point to P0.
// * If the heap partitioning feature is active in kFull mode:
// Cold & partition 1 will be the same as hot & partition 1. Namely, it
// will point to the [kNumBaseClasses, kColdClassesStart) size classes.
// * If the heap partitioning feature is active in kLight mode: Malloc P0 is
// exclusive to P0; Hot New P0 maps to Hot New P1; Cold P1 maps to Cold P0.
// Cold R1 will be the same as Hot R1. Namely, it will point to the
// [kNumBaseClasses, kColdClassesStart) size classes.
// * If the heap partitioning feature is active in kLight mode: Hot Malloc R0
// exclusively points to P0; Hot New R0 and Hot New R1 point to P1;
// Cold New R0 and Cold New R1 point to Cold P0.
//
// If NUMA support is compiled in, the partition 1 regions won't exist.
// If NUMA support is compiled in, the R1 regions won't exist.
// Similarly, for cold memory, if cold classes are not compiled in.
static constexpr size_t kHotRegisters = 2 * kSecurityPartitions;
static constexpr size_t kColdRegisterStride = kHotRegisters;
static constexpr size_t kColdRegisters =
static constexpr size_t kHotRegions = 2 * kSecurityPartitions;
static constexpr size_t kColdRegionsStart = kHotRegions;
static constexpr size_t kColdRegions =
(kHasColdClasses ? 1 : 0) * kSecurityPartitions;
static constexpr size_t kClassArraySizePartitions =
kClassArraySize * (kHotRegisters + kColdRegisters);
static constexpr size_t kTotalClassArraySize =
kClassArraySize * (kHotRegions + kColdRegions);

// class_array_ is accessed on every malloc, so is very hot. We make it the
// first member so that it inherits the overall alignment of a SizeMap
// instance. In particular, if we create a SizeMap instance that's cache-line
// aligned, this member is also aligned to the width of a cache line.
//
// For the mapping with the cold and/or security partitions, see the comment
// for kClassArraySizePartitions.
CompactSizeClass class_array_[kClassArraySizePartitions] = {0};
// for kTotalClassArraySize.
CompactSizeClass class_array_[kTotalClassArraySize] = {0};

// Number of objects to move between a per-thread list and a central
// list in one shot. We want this to be not too small so we can
Expand Down Expand Up @@ -163,6 +165,10 @@ class SizeMap {
return ret;
}

// Set the specified class_array_ region from region 0 adjusting all
// values by `adjust`.
void SetClassArrayRegion(size_t region, CompactSizeClass adjust);

// Mapping from size class to number of pages to allocate at a time
unsigned char class_to_pages_[kNumClasses] = {0};

Expand Down Expand Up @@ -230,24 +236,24 @@ class SizeMap {
return {false};
}
size_t size_class;
// Note, if security heap partitioning is enabled, only data (partition 0)
// is added to the cold heap. See the comment for kClassArraySizePartitions
// Note, if security heap partitioning is enabled, only data (region 0)
// is added to the cold heap. See the comment for kTotalClassArraySize
// for more details.
if (kHasColdClasses && policy.is_cold()) {
TC_ASSERT(policy.allocation_type() == AllocationType::New);
TC_ASSERT_LT(idx + (policy.security_partition() + kColdRegisterStride) *
TC_ASSERT_LT(idx + (policy.security_partition() + kColdRegionsStart) *
kClassArraySize,
kClassArraySizePartitions);
size_class = class_array_[idx + (policy.security_partition() +
kColdRegisterStride) *
kClassArraySize];
kTotalClassArraySize);
size_class =
class_array_[idx + (policy.security_partition() + kColdRegionsStart) *
kClassArraySize];
} else {
constexpr size_t kTypeOffset =
policy.allocation_type() != AllocationType::New ? kSecurityPartitions
: 0;
TC_ASSERT_LT(
idx + (policy.security_partition() + kTypeOffset) * kClassArraySize,
kClassArraySizePartitions);
kTotalClassArraySize);
size_class =
class_array_[idx + (policy.security_partition() + kTypeOffset) *
kClassArraySize] +
Expand Down
Loading