diff --git a/tcmalloc/sizemap.cc b/tcmalloc/sizemap.cc index f234d2176..3560b4522 100644 --- a/tcmalloc/sizemap.cc +++ b/tcmalloc/sizemap.cc @@ -238,91 +238,54 @@ bool SizeMap::Init(absl::Span size_classes) { return false; } - int next_size = 0; - for (int c = 1; c < kNumClasses; c++) { - const int max_size_in_class = class_to_size_[c]; - - for (int s = next_size; s <= max_size_in_class; - s += static_cast(kAlignment)) { + // Fill in the canonical class array in region 0. + for (int c = 1, s = 0; c < kNumClasses && s <= kMaxSize; c++) { + for (; s <= class_to_size_[c]; s += static_cast(kAlignment)) { class_array_[ClassIndex(s)] = c; } - next_size = max_size_in_class + static_cast(kAlignment); - if (next_size > kMaxSize) { - break; - } } - // Point all lookups in hot registers (Malloc P0, New P1, Malloc P1) - // directly to New's P0. We only overwrite the lookups if heap + // Point all lookups in hot regions (Malloc R0, New R1, Malloc R1) + // directly to New R0. We only overwrite the lookups if heap // partitioning is active with the dedicated size classes. - for (size_t i = 1; i < kHotRegisters; ++i) { - std::copy(&class_array_[0], &class_array_[kClassArraySize], - &class_array_[kClassArraySize * i]); + for (size_t i = 1; i < kHotRegions; ++i) { + SetClassArrayRegion(i, 0); } - const bool heap_partitioning_active = - tc_globals.multiple_non_numa_partitions(); const bool heap_partitioning_full = Parameters::heap_partitioning_mode() == HeapPartitioningMode::kFull; - if (kSecurityPartitions > 1 && heap_partitioning_active) { - next_size = 0; - for (int c = kNumBaseClasses + 1; c < kColdClassesStart; ++c) { - const int max_size_in_class = class_to_size_[c]; - - for (int s = next_size; s <= max_size_in_class; - s += static_cast(kAlignment)) { - // Route Hot Malloc P1 to security partition P1. - class_array_[ClassIndex(s) + - kClassArraySize * (kSecurityPartitions + 1)] = c; - // Route Hot New P1 to security partition P1. - class_array_[ClassIndex(s) + kClassArraySize] = c; - if (heap_partitioning_full) continue; - // In kLight mode, route Hot New P0 to P1. - class_array_[ClassIndex(s)] = c; - } - next_size = max_size_in_class + static_cast(kAlignment); - if (next_size > kMaxSize) { - break; - } + if (ColdFeatureActive()) { + SetClassArrayRegion(kColdRegionsStart, +kColdClassesStart); + if (kSecurityPartitions > 1) { + // Point all lookups in Cold New R1 to either Hot New R1 or Cold New R0. + SetClassArrayRegion(kColdRegionsStart + 1, heap_partitioning_full + ? +kNumBaseClasses + : +kColdClassesStart); } } - if (ColdFeatureActive()) { - next_size = 0; - for (int c = kColdClassesStart + 1; c < kNumClasses; c++) { - size_t max_size_in_class = class_to_size_[c]; - if (max_size_in_class == 0) { - next_size = max_size_in_class + static_cast(kAlignment); - continue; - } - - for (int s = next_size; s <= max_size_in_class; - s += static_cast(kAlignment)) { - class_array_[ClassIndex(s) + kClassArraySize * kColdRegisterStride] = c; - } - next_size = max_size_in_class + static_cast(kAlignment); - if (next_size > kMaxSize) { - break; - } - } - if (kSecurityPartitions > 1) { - if (heap_partitioning_full) { - // Point all lookups in Cold New's P1 register to Hot New's P1. - std::copy(&class_array_[kClassArraySize], - &class_array_[kClassArraySize * 2], - &class_array_[kClassArraySize * (kColdRegisterStride + 1)]); - } else { - // Point all lookups in Cold New's P1 register to Cold New's P0. - std::copy(&class_array_[kClassArraySize * kColdRegisterStride], - &class_array_[kClassArraySize * (kColdRegisterStride + 1)], - &class_array_[kClassArraySize * (kColdRegisterStride + 1)]); - } + if (kSecurityPartitions > 1 && tc_globals.multiple_non_numa_partitions()) { + // Route Hot Malloc R1 to security partition P1. + SetClassArrayRegion(kSecurityPartitions + 1, +kNumBaseClasses); + // Route Hot New R1 to security partition P1. + SetClassArrayRegion(1, +kNumBaseClasses); + if (!heap_partitioning_full) { + // In kLight mode, route Hot New R0 to P1. + SetClassArrayRegion(0, +kNumBaseClasses); } } return true; } +void SizeMap::SetClassArrayRegion(size_t region, CompactSizeClass adjust) { + // Ensure R0 is the canonical non-adjusted array. + TC_CHECK_EQ(class_array_[0], 1); + for (size_t i = 0; i < kClassArraySize; ++i) { + class_array_[region * kClassArraySize + i] = class_array_[i] + adjust; + } +} + } // namespace tcmalloc_internal } // namespace tcmalloc diff --git a/tcmalloc/sizemap.h b/tcmalloc/sizemap.h index 1260d57a3..9d2184a66 100644 --- a/tcmalloc/sizemap.h +++ b/tcmalloc/sizemap.h @@ -89,29 +89,31 @@ class SizeMap { // If TCMalloc is compiled without NUMA support, and with cold allocations // (cold classes), then the class_array_ will consist of 6 regions: // - // [0, kClassArraySize) : Hot New P0 - // [kClassArraySize, 2*kClassArraySize) : Hot New P1 - // [2*kClassArraySize, 3*kClassArraySize) : Hot Malloc P0 - // [3*kClassArraySize, 4*kClassArraySize) : Hot Malloc P1 - // [4*kClassArraySize, 5*kClassArraySize) : Cold New P0 - // [5*kClassArraySize, 6*kClassArraySize) : Cold New P1 + // [0, kClassArraySize) : Hot New R0 + // [kClassArraySize, 2*kClassArraySize) : Hot New R1 + // [2*kClassArraySize, 3*kClassArraySize) : Hot Malloc R0 + // [3*kClassArraySize, 4*kClassArraySize) : Hot Malloc R1 + // [4*kClassArraySize, 5*kClassArraySize) : Cold New R0 + // [5*kClassArraySize, 6*kClassArraySize) : Cold New R1 // // * If the heap partitioning feature is not active, then the lookups for - // partition 1 will contain the same information as for partition 0. + // region 1 will contain the same information as for region 0, + // and both Hot New and Hot Malloc point to P0. // * If the heap partitioning feature is active in kFull mode: - // Cold & partition 1 will be the same as hot & partition 1. Namely, it - // will point to the [kNumBaseClasses, kColdClassesStart) size classes. - // * If the heap partitioning feature is active in kLight mode: Malloc P0 is - // exclusive to P0; Hot New P0 maps to Hot New P1; Cold P1 maps to Cold P0. + // Cold R1 will be the same as Hot R1. Namely, it will point to the + // [kNumBaseClasses, kColdClassesStart) size classes. + // * If the heap partitioning feature is active in kLight mode: Hot Malloc R0 + // exclusively points to P0; Hot New R0 and Hot New R1 point to P1; + // Cold New R0 and Cold New R1 point to Cold P0. // - // If NUMA support is compiled in, the partition 1 regions won't exist. + // If NUMA support is compiled in, the R1 regions won't exist. // Similarly, for cold memory, if cold classes are not compiled in. - static constexpr size_t kHotRegisters = 2 * kSecurityPartitions; - static constexpr size_t kColdRegisterStride = kHotRegisters; - static constexpr size_t kColdRegisters = + static constexpr size_t kHotRegions = 2 * kSecurityPartitions; + static constexpr size_t kColdRegionsStart = kHotRegions; + static constexpr size_t kColdRegions = (kHasColdClasses ? 1 : 0) * kSecurityPartitions; - static constexpr size_t kClassArraySizePartitions = - kClassArraySize * (kHotRegisters + kColdRegisters); + static constexpr size_t kTotalClassArraySize = + kClassArraySize * (kHotRegions + kColdRegions); // class_array_ is accessed on every malloc, so is very hot. We make it the // first member so that it inherits the overall alignment of a SizeMap @@ -119,8 +121,8 @@ class SizeMap { // aligned, this member is also aligned to the width of a cache line. // // For the mapping with the cold and/or security partitions, see the comment - // for kClassArraySizePartitions. - CompactSizeClass class_array_[kClassArraySizePartitions] = {0}; + // for kTotalClassArraySize. + CompactSizeClass class_array_[kTotalClassArraySize] = {0}; // Number of objects to move between a per-thread list and a central // list in one shot. We want this to be not too small so we can @@ -163,6 +165,10 @@ class SizeMap { return ret; } + // Set the specified class_array_ region from region 0 adjusting all + // values by `adjust`. + void SetClassArrayRegion(size_t region, CompactSizeClass adjust); + // Mapping from size class to number of pages to allocate at a time unsigned char class_to_pages_[kNumClasses] = {0}; @@ -230,24 +236,24 @@ class SizeMap { return {false}; } size_t size_class; - // Note, if security heap partitioning is enabled, only data (partition 0) - // is added to the cold heap. See the comment for kClassArraySizePartitions + // Note, if security heap partitioning is enabled, only data (region 0) + // is added to the cold heap. See the comment for kTotalClassArraySize // for more details. if (kHasColdClasses && policy.is_cold()) { TC_ASSERT(policy.allocation_type() == AllocationType::New); - TC_ASSERT_LT(idx + (policy.security_partition() + kColdRegisterStride) * + TC_ASSERT_LT(idx + (policy.security_partition() + kColdRegionsStart) * kClassArraySize, - kClassArraySizePartitions); - size_class = class_array_[idx + (policy.security_partition() + - kColdRegisterStride) * - kClassArraySize]; + kTotalClassArraySize); + size_class = + class_array_[idx + (policy.security_partition() + kColdRegionsStart) * + kClassArraySize]; } else { constexpr size_t kTypeOffset = policy.allocation_type() != AllocationType::New ? kSecurityPartitions : 0; TC_ASSERT_LT( idx + (policy.security_partition() + kTypeOffset) * kClassArraySize, - kClassArraySizePartitions); + kTotalClassArraySize); size_class = class_array_[idx + (policy.security_partition() + kTypeOffset) * kClassArraySize] +