Skip to content

Remove fragmentation profile #405

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
copybara-service wants to merge 1 commit into
base: master
Choose a base branch
from
+144 −56
Draft

Remove fragmentation profile #405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions tcmalloc/allocation_sampling.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,5 +58,19 @@ std::unique_ptr<const ProfileBase> DumpHeapProfile(Static& state) {
return profile;
}

ABSL_ATTRIBUTE_NOINLINE void FreeProxyObject(Static& state, void* ptr,
size_t size_class) {
if (ABSL_PREDICT_TRUE(UsePerCpuCache(state))) {
state.cpu_cache().Deallocate(ptr, size_class);
} else if (ThreadCache* cache = ThreadCache::GetCacheIfPresent();
ABSL_PREDICT_TRUE(cache)) {
cache->Deallocate(ptr, size_class);
} else {
// This thread doesn't have thread-cache yet or already. Delete directly
// into transfer cache.
state.transfer_cache().InsertRange(size_class, absl::Span<void*>(&ptr, 1));
}
}

} // namespace tcmalloc::tcmalloc_internal
GOOGLE_MALLOC_SECTION_END
78 changes: 66 additions & 12 deletions tcmalloc/allocation_sampling.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,16 @@ inline Sampler& GetThreadSampler() {
#endif
}

void FreeProxyObject(Static& state, void* absl_nonnull ptr, size_t size_class);

// Performs sampling for already occurred allocation of object.
//
// For small object sizes, we allocate a new object in a sampled span.
// For very small object sizes, object is used as 'proxy' and full
// page with sampled marked is allocated instead.
//
// For medium-sized objects that have single instance per span,
// they're simply freed and fresh page span is allocated to represent
// sampling.
//
// For large objects (i.e. allocated with do_malloc_pages) they are
// also fully reused and their span is marked as sampled.
Expand All @@ -88,10 +95,12 @@ inline Sampler& GetThreadSampler() {
template <typename Policy>
ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation(
Static& state, Policy policy, size_t requested_size, size_t weight,
size_t size_class, Span* absl_nullable span) {
TC_CHECK_EQ(size_class != 0, span == nullptr);
size_t size_class, void* absl_nullable obj, Span* absl_nullable span) {
TC_CHECK((size_class != 0 && obj != nullptr && span == nullptr) ||
(size_class == 0 && obj == nullptr && span != nullptr));

StackTrace stack_trace;
stack_trace.proxy = nullptr;
stack_trace.requested_size = requested_size;
// Grab the stack trace outside the heap lock.
stack_trace.depth = absl::GetStackTrace(stack_trace.stack, kMaxStackDepth, 0);
Expand Down Expand Up @@ -122,6 +131,8 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation(
: MemoryTag::kSampled;
size_t capacity = 0;
if (size_class != 0) {
TC_ASSERT_EQ(size_class,
state.pagemap().sizeclass(PageIdContainingTagged(obj)));
state.per_size_class_counts()[size_class].Add(allocation_estimate);

stack_trace.allocated_size = state.sizemap().class_to_size(size_class);
Expand Down Expand Up @@ -150,10 +161,24 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation(
}
capacity = requested_size;
} else if ((span = state.page_allocator().New(
num_pages, {1, AccessDensityPrediction::kSparse}, tag))) {
num_pages, {1, AccessDensityPrediction::kSparse}, tag)) ==
nullptr) {
capacity = stack_trace.allocated_size;
return {obj, capacity};
} else {
return {};
capacity = stack_trace.allocated_size;
}

size_t span_size =
Length(state.sizemap().class_to_pages(size_class)).in_bytes();
size_t objects_per_span = stack_trace.allocated_size != 0
? span_size / stack_trace.allocated_size
: -1ul;

if (objects_per_span != 1) {
TC_ASSERT_GT(objects_per_span, 1);
stack_trace.proxy = obj;
obj = nullptr;
}
} else {
// Set stack_trace.allocated_size to the exact size for a page allocation.
Expand Down Expand Up @@ -219,6 +244,12 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation(

state.peak_heap_tracker().MaybeSaveSample();

if (obj != nullptr) {
// We are not maintaining precise statistics on malloc hit/miss rates at our
// cache tiers. We can deallocate into our ordinary cache.
TC_ASSERT_NE(size_class, 0);
FreeProxyObject(state, obj, size_class);
}
TC_ASSERT_EQ(state.pagemap().sizeclass(span->first_page()), 0);
return {(alloc_with_status.alloc != nullptr) ? alloc_with_status.alloc
: span->start_address(),
Expand All @@ -233,15 +264,16 @@ template <typename Policy>
static sized_ptr_t SampleLargeAllocation(Static& state, Policy policy,
size_t requested_size, size_t weight,
Span* span) {
return SampleifyAllocation(state, policy, requested_size, weight, 0, span);
return SampleifyAllocation(state, policy, requested_size, weight, 0, nullptr,
span);
}

template <typename Policy>
static sized_ptr_t SampleSmallAllocation(Static& state, Policy policy,
size_t requested_size, size_t weight,
size_t size_class) {
size_t size_class, sized_ptr_t res) {
return SampleifyAllocation(state, policy, requested_size, weight, size_class,
nullptr);
res.p, nullptr);
}

// Rewrite type so that the allocation type falls into one of the categories we
Expand Down Expand Up @@ -291,6 +323,7 @@ void MaybeUnsampleAllocation(Static& state, Policy policy,

TC_ASSERT_EQ(state.pagemap().sizeclass(PageIdContainingTagged(ptr)), 0);

void* const proxy = sampled_allocation->sampled_stack.proxy;
const size_t weight = sampled_allocation->sampled_stack.weight;
const size_t requested_size =
sampled_allocation->sampled_stack.requested_size;
Expand Down Expand Up @@ -333,14 +366,18 @@ void MaybeUnsampleAllocation(Static& state, Policy policy,
sampled_allocation->sampled_stack.depth));
}

const size_t allocated_alignment = static_cast<size_t>(
sampled_allocation->sampled_stack.requested_alignment.value_or(
std::align_val_t{1}));
const std::optional<std::align_val_t> deallocated_alignment =
policy.has_explicit_alignment()
? std::make_optional<std::align_val_t>(policy.align())
: std::nullopt;

if ((size.has_value() || policy.allocation_type() == AllocationType::New)) {
const bool type_mismatch =
policy.allocation_type() !=
sampled_allocation->sampled_stack.allocation_type;
const std::optional<std::align_val_t> deallocated_alignment =
policy.has_explicit_alignment()
? std::make_optional<std::align_val_t>(policy.align())
: std::nullopt;
const bool alignment_mismatch =
deallocated_alignment !=
sampled_allocation->sampled_stack.requested_alignment;
Expand Down Expand Up @@ -391,6 +428,23 @@ void MaybeUnsampleAllocation(Static& state, Policy policy,
MallocHook::InvokeSampledDeleteHook(sampled_alloc);

state.deallocation_samples.ReportFree(sampled_alloc_handle);

if (proxy) {
const auto proxy_policy = policy.InSamePartitionAs(proxy);
size_t size_class;
if (AccessFromPointer(proxy) == AllocationAccess::kCold) {
size_class = state.sizemap().SizeClass(
proxy_policy.AccessAsCold().AlignAs(allocated_alignment),
allocated_size);
} else {
size_class = state.sizemap().SizeClass(
proxy_policy.AccessAsHot().AlignAs(allocated_alignment),
allocated_size);
}
TC_ASSERT_EQ(size_class,
state.pagemap().sizeclass(PageIdContainingTagged(proxy)));
FreeProxyObject(state, proxy, size_class);
}
}

} // namespace tcmalloc::tcmalloc_internal
Expand Down
11 changes: 11 additions & 0 deletions tcmalloc/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,17 @@ enum class AllocationAccess {
kCold,
};

inline AllocationAccess AccessFromPointer(void* ptr) {
if (!kHasExpandedClasses) {
TC_ASSERT_NE(GetMemoryTag(ptr), MemoryTag::kCold);
return AllocationAccess::kHot;
}

return ABSL_PREDICT_FALSE(GetMemoryTag(ptr) == MemoryTag::kCold)
? AllocationAccess::kCold
: AllocationAccess::kHot;
}

inline MemoryTag MultiNormalTag(size_t partition) {
switch (partition) {
case 0:
Expand Down
4 changes: 2 additions & 2 deletions tcmalloc/experimental_pow2_size_class.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ static constexpr SizeClassInfo List[] = {
{ 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00%
{ 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00%
{ 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00%
{131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00%
{131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00%
{262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00%
};
#elif TCMALLOC_PAGE_SHIFT == 12
Expand Down Expand Up @@ -251,7 +251,7 @@ static constexpr SizeClassInfo List[] = {
{ 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00%
{ 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00%
{ 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00%
{131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00%
{131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00%
{262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00%
};
#elif TCMALLOC_PAGE_SHIFT == 12
Expand Down
9 changes: 9 additions & 0 deletions tcmalloc/internal/logging.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,15 @@ struct StackTrace {
// memory block.
AllocHandle sampled_alloc_handle;

// For small sampled objects, we allocate a full span to hold the
// sampled object. However to avoid disturbing fragmentation
// profiles, in such cases we also allocate a small proxy object
// using the normal mechanism.
//
// proxy field is defined only for heap sample stack traces.
// For heap samples, proxy==NULL iff size > kMaxSize.
void* proxy;

uintptr_t requested_size;
std::optional<std::align_val_t> requested_alignment;
uintptr_t allocated_size; // size after sizeclass/page rounding
Expand Down
30 changes: 15 additions & 15 deletions tcmalloc/legacy_size_classes.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,9 @@ static constexpr SizeClassInfo List[] = {
{ 640, 8192, 32}, // 38 12 6.98% 0.43% 11.11%
{ 704, 8192, 32}, // 39 11 6.20% 0.43% 10.00%
{ 768, 8192, 32}, // 40 10 6.98% 0.43% 9.09%
{ 896, 8192, 32}, // 41 9 2.33% 0.42% 16.67%
{ 896, 8192, 32}, // 41 9 2.33% 0.43% 16.67%
{ 1024, 8192, 32}, // 42 8 0.78% 0.42% 14.29%
{ 1152, 16384, 32}, // 43 14 1.95% 0.42% 12.50%
{ 1152, 16384, 32}, // 43 14 1.95% 0.43% 12.50%
{ 1280, 16384, 32}, // 44 12 6.61% 0.43% 11.11%
{ 1408, 16384, 32}, // 45 11 5.84% 0.43% 10.00%
{ 1536, 16384, 32}, // 46 10 6.61% 0.43% 9.09%
Expand Down Expand Up @@ -228,7 +228,7 @@ static constexpr SizeClassInfo List[] = {
{ 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29%
{ 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00%
{ 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00%
{114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67%
{114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67%
{131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29%
{163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00%
{196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00%
Expand Down Expand Up @@ -296,17 +296,17 @@ static constexpr SizeClassInfo List[] = {
{ 2304, 262144, 28}, // 46 113 0.71% 12.53% 12.50%
{ 2560, 262144, 25}, // 47 102 0.41% 12.53% 11.11%
{ 2816, 262144, 23}, // 48 93 0.12% 12.53% 10.00%
{ 3200, 262144, 20}, // 49 81 1.15% 12.53% 13.64%
{ 3200, 262144, 20}, // 49 81 1.15% 12.54% 13.64%
{ 3584, 262144, 18}, // 50 73 0.22% 12.53% 12.00%
{ 4096, 262144, 16}, // 51 64 0.02% 12.53% 14.29%
{ 4224, 262144, 15}, // 52 62 0.12% 12.53% 3.12%
{ 4736, 262144, 13}, // 53 55 0.66% 12.53% 12.12%
{ 4736, 262144, 13}, // 53 55 0.66% 12.54% 12.12%
{ 5120, 262144, 12}, // 54 51 0.41% 12.53% 8.11%
{ 5632, 262144, 11}, // 55 46 1.20% 12.54% 10.00%
{ 6144, 262144, 10}, // 56 42 1.59% 12.54% 9.09%
{ 6528, 262144, 10}, // 57 40 0.41% 12.53% 6.25%
{ 7040, 262144, 9}, // 58 37 0.66% 12.54% 7.84%
{ 7680, 262144, 8}, // 59 34 0.41% 12.53% 9.09%
{ 7680, 262144, 8}, // 59 34 0.41% 12.54% 9.09%
{ 8192, 262144, 8}, // 60 32 0.02% 12.53% 6.67%
{ 9344, 262144, 7}, // 61 28 0.22% 12.53% 14.06%
{ 11392, 262144, 5}, // 62 23 0.07% 12.53% 21.92%
Expand All @@ -330,7 +330,7 @@ static constexpr SizeClassInfo List[] = {
{ 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58%
{104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09%
{112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08%
{131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76%
{131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76%
{149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26%
{174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67%
{196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53%
Expand Down Expand Up @@ -439,17 +439,17 @@ static constexpr SizeClassInfo List[] = {
{ 368, 8192, 32}, // 24 22 1.94% 0.42% 4.55%
{ 384, 8192, 32}, // 25 21 2.33% 0.42% 4.35%
{ 400, 8192, 32}, // 26 20 3.10% 0.42% 4.17%
{ 416, 8192, 32}, // 27 19 4.26% 0.42% 4.00%
{ 416, 8192, 32}, // 27 19 4.26% 0.43% 4.00%
{ 448, 8192, 32}, // 28 18 2.33% 0.42% 7.69%
{ 480, 8192, 32}, // 29 17 1.16% 0.42% 7.14%
{ 512, 8192, 32}, // 30 16 0.78% 0.42% 6.67%
{ 576, 8192, 32}, // 31 14 2.33% 0.42% 12.50%
{ 640, 8192, 32}, // 32 12 6.98% 0.43% 11.11%
{ 704, 8192, 32}, // 33 11 6.20% 0.43% 10.00%
{ 768, 8192, 32}, // 34 10 6.98% 0.43% 9.09%
{ 896, 8192, 32}, // 35 9 2.33% 0.42% 16.67%
{ 896, 8192, 32}, // 35 9 2.33% 0.43% 16.67%
{ 1024, 8192, 32}, // 36 8 0.78% 0.42% 14.29%
{ 1152, 16384, 32}, // 37 14 1.95% 0.42% 12.50%
{ 1152, 16384, 32}, // 37 14 1.95% 0.43% 12.50%
{ 1280, 16384, 32}, // 38 12 6.61% 0.43% 11.11%
{ 1408, 16384, 32}, // 39 11 5.84% 0.43% 10.00%
{ 1536, 16384, 32}, // 40 10 6.61% 0.43% 9.09%
Expand Down Expand Up @@ -583,7 +583,7 @@ static constexpr SizeClassInfo List[] = {
{ 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29%
{ 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00%
{ 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00%
{114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67%
{114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67%
{131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29%
{163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00%
{196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00%
Expand Down Expand Up @@ -647,19 +647,19 @@ static constexpr SizeClassInfo List[] = {
{ 2304, 262144, 28}, // 42 113 0.71% 12.53% 12.50%
{ 2560, 262144, 25}, // 43 102 0.41% 12.53% 11.11%
{ 2816, 262144, 23}, // 44 93 0.12% 12.53% 10.00%
{ 3200, 262144, 20}, // 45 81 1.15% 12.53% 13.64%
{ 3200, 262144, 20}, // 45 81 1.15% 12.54% 13.64%
{ 3584, 262144, 18}, // 46 73 0.22% 12.53% 12.00%
{ 3840, 262144, 17}, // 47 68 0.41% 12.53% 7.14%
{ 4096, 262144, 16}, // 48 64 0.02% 12.53% 6.67%
{ 4224, 262144, 15}, // 49 62 0.12% 12.53% 3.12%
{ 4480, 262144, 14}, // 50 58 0.90% 12.54% 6.06%
{ 4736, 262144, 13}, // 51 55 0.66% 12.53% 5.71%
{ 4736, 262144, 13}, // 51 55 0.66% 12.54% 5.71%
{ 5120, 262144, 12}, // 52 51 0.41% 12.53% 8.11%
{ 5632, 262144, 11}, // 53 46 1.20% 12.54% 10.00%
{ 6144, 262144, 10}, // 54 42 1.59% 12.54% 9.09%
{ 6528, 262144, 10}, // 55 40 0.41% 12.53% 6.25%
{ 7040, 262144, 9}, // 56 37 0.66% 12.54% 7.84%
{ 7680, 262144, 8}, // 57 34 0.41% 12.53% 9.09%
{ 7680, 262144, 8}, // 57 34 0.41% 12.54% 9.09%
{ 8192, 262144, 8}, // 58 32 0.02% 12.53% 6.67%
{ 9344, 262144, 7}, // 59 28 0.22% 12.53% 14.06%
{ 10368, 262144, 6}, // 60 25 1.15% 12.54% 10.96%
Expand All @@ -685,7 +685,7 @@ static constexpr SizeClassInfo List[] = {
{ 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58%
{104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09%
{112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08%
{131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76%
{131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76%
{149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26%
{174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67%
{196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53%
Expand Down
3 changes: 1 addition & 2 deletions tcmalloc/malloc_extension.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,7 @@ enum class ProfileType {
kHeap,

// Fragmentation report
kFragmentation ABSL_DEPRECATED(
"Fragmentation profiles are no longer collected"),
kFragmentation,

// Sample of objects that were live at a recent peak of total heap usage. The
// specifics of when exactly this profile is collected are subject to change.
Expand Down
Loading
Loading