From e3fd078caf3e38830229f8c6dd16321e470f722b Mon Sep 17 00:00:00 2001 From: TCMalloc Team Date: Mon, 20 Jul 2026 16:27:35 -0700 Subject: [PATCH] Remove fragmentation profile The fragmentation profile is not useful in the current form. The proxy allocation stacks are not correlated and are not the cause of external fragmentation. They are just random objects of a particular size class. We don't see that these profiles are used in practice. External fragmentation of more of a global tcmalloc property that depends on CFL span prioritization, etc. The information about external fragmentation is available in a more complete form in mallocz CFL info, and if necessary can be crossed with heapz for representative stacks. PiperOrigin-RevId: 951115964 --- tcmalloc/allocation_sampling.cc | 14 ++++ tcmalloc/allocation_sampling.h | 78 ++++++++++++++++--- tcmalloc/common.h | 11 +++ tcmalloc/experimental_pow2_size_class.cc | 4 +- tcmalloc/internal/logging.h | 9 +++ tcmalloc/legacy_size_classes.cc | 30 +++---- tcmalloc/malloc_extension.h | 3 +- .../reuse_relaxed_below_64_size_classes.cc | 14 ++-- tcmalloc/size_classes.cc | 16 ++-- tcmalloc/tcmalloc.cc | 21 ++--- 10 files changed, 144 insertions(+), 56 deletions(-) diff --git a/tcmalloc/allocation_sampling.cc b/tcmalloc/allocation_sampling.cc index 5af8d2358..65115749d 100644 --- a/tcmalloc/allocation_sampling.cc +++ b/tcmalloc/allocation_sampling.cc @@ -58,5 +58,19 @@ std::unique_ptr DumpHeapProfile(Static& state) { return profile; } +ABSL_ATTRIBUTE_NOINLINE void FreeProxyObject(Static& state, void* ptr, + size_t size_class) { + if (ABSL_PREDICT_TRUE(UsePerCpuCache(state))) { + state.cpu_cache().Deallocate(ptr, size_class); + } else if (ThreadCache* cache = ThreadCache::GetCacheIfPresent(); + ABSL_PREDICT_TRUE(cache)) { + cache->Deallocate(ptr, size_class); + } else { + // This thread doesn't have thread-cache yet or already. Delete directly + // into transfer cache. + state.transfer_cache().InsertRange(size_class, absl::Span(&ptr, 1)); + } +} + } // namespace tcmalloc::tcmalloc_internal GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/allocation_sampling.h b/tcmalloc/allocation_sampling.h index 7db362dcc..9753aa7b7 100644 --- a/tcmalloc/allocation_sampling.h +++ b/tcmalloc/allocation_sampling.h @@ -67,9 +67,16 @@ inline Sampler& GetThreadSampler() { #endif } +void FreeProxyObject(Static& state, void* absl_nonnull ptr, size_t size_class); + // Performs sampling for already occurred allocation of object. // -// For small object sizes, we allocate a new object in a sampled span. +// For very small object sizes, object is used as 'proxy' and full +// page with sampled marked is allocated instead. +// +// For medium-sized objects that have single instance per span, +// they're simply freed and fresh page span is allocated to represent +// sampling. // // For large objects (i.e. allocated with do_malloc_pages) they are // also fully reused and their span is marked as sampled. @@ -88,10 +95,12 @@ inline Sampler& GetThreadSampler() { template ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( Static& state, Policy policy, size_t requested_size, size_t weight, - size_t size_class, Span* absl_nullable span) { - TC_CHECK_EQ(size_class != 0, span == nullptr); + size_t size_class, void* absl_nullable obj, Span* absl_nullable span) { + TC_CHECK((size_class != 0 && obj != nullptr && span == nullptr) || + (size_class == 0 && obj == nullptr && span != nullptr)); StackTrace stack_trace; + stack_trace.proxy = nullptr; stack_trace.requested_size = requested_size; // Grab the stack trace outside the heap lock. stack_trace.depth = absl::GetStackTrace(stack_trace.stack, kMaxStackDepth, 0); @@ -122,6 +131,8 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( : MemoryTag::kSampled; size_t capacity = 0; if (size_class != 0) { + TC_ASSERT_EQ(size_class, + state.pagemap().sizeclass(PageIdContainingTagged(obj))); state.per_size_class_counts()[size_class].Add(allocation_estimate); stack_trace.allocated_size = state.sizemap().class_to_size(size_class); @@ -150,10 +161,24 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( } capacity = requested_size; } else if ((span = state.page_allocator().New( - num_pages, {1, AccessDensityPrediction::kSparse}, tag))) { + num_pages, {1, AccessDensityPrediction::kSparse}, tag)) == + nullptr) { capacity = stack_trace.allocated_size; + return {obj, capacity}; } else { - return {}; + capacity = stack_trace.allocated_size; + } + + size_t span_size = + Length(state.sizemap().class_to_pages(size_class)).in_bytes(); + size_t objects_per_span = stack_trace.allocated_size != 0 + ? span_size / stack_trace.allocated_size + : -1ul; + + if (objects_per_span != 1) { + TC_ASSERT_GT(objects_per_span, 1); + stack_trace.proxy = obj; + obj = nullptr; } } else { // Set stack_trace.allocated_size to the exact size for a page allocation. @@ -219,6 +244,12 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( state.peak_heap_tracker().MaybeSaveSample(); + if (obj != nullptr) { + // We are not maintaining precise statistics on malloc hit/miss rates at our + // cache tiers. We can deallocate into our ordinary cache. + TC_ASSERT_NE(size_class, 0); + FreeProxyObject(state, obj, size_class); + } TC_ASSERT_EQ(state.pagemap().sizeclass(span->first_page()), 0); return {(alloc_with_status.alloc != nullptr) ? alloc_with_status.alloc : span->start_address(), @@ -233,15 +264,16 @@ template static sized_ptr_t SampleLargeAllocation(Static& state, Policy policy, size_t requested_size, size_t weight, Span* span) { - return SampleifyAllocation(state, policy, requested_size, weight, 0, span); + return SampleifyAllocation(state, policy, requested_size, weight, 0, nullptr, + span); } template static sized_ptr_t SampleSmallAllocation(Static& state, Policy policy, size_t requested_size, size_t weight, - size_t size_class) { + size_t size_class, sized_ptr_t res) { return SampleifyAllocation(state, policy, requested_size, weight, size_class, - nullptr); + res.p, nullptr); } // Rewrite type so that the allocation type falls into one of the categories we @@ -291,6 +323,7 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, TC_ASSERT_EQ(state.pagemap().sizeclass(PageIdContainingTagged(ptr)), 0); + void* const proxy = sampled_allocation->sampled_stack.proxy; const size_t weight = sampled_allocation->sampled_stack.weight; const size_t requested_size = sampled_allocation->sampled_stack.requested_size; @@ -333,14 +366,18 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, sampled_allocation->sampled_stack.depth)); } + const size_t allocated_alignment = static_cast( + sampled_allocation->sampled_stack.requested_alignment.value_or( + std::align_val_t{1})); + const std::optional deallocated_alignment = + policy.has_explicit_alignment() + ? std::make_optional(policy.align()) + : std::nullopt; + if ((size.has_value() || policy.allocation_type() == AllocationType::New)) { const bool type_mismatch = policy.allocation_type() != sampled_allocation->sampled_stack.allocation_type; - const std::optional deallocated_alignment = - policy.has_explicit_alignment() - ? std::make_optional(policy.align()) - : std::nullopt; const bool alignment_mismatch = deallocated_alignment != sampled_allocation->sampled_stack.requested_alignment; @@ -391,6 +428,23 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, MallocHook::InvokeSampledDeleteHook(sampled_alloc); state.deallocation_samples.ReportFree(sampled_alloc_handle); + + if (proxy) { + const auto proxy_policy = policy.InSamePartitionAs(proxy); + size_t size_class; + if (AccessFromPointer(proxy) == AllocationAccess::kCold) { + size_class = state.sizemap().SizeClass( + proxy_policy.AccessAsCold().AlignAs(allocated_alignment), + allocated_size); + } else { + size_class = state.sizemap().SizeClass( + proxy_policy.AccessAsHot().AlignAs(allocated_alignment), + allocated_size); + } + TC_ASSERT_EQ(size_class, + state.pagemap().sizeclass(PageIdContainingTagged(proxy))); + FreeProxyObject(state, proxy, size_class); + } } } // namespace tcmalloc::tcmalloc_internal diff --git a/tcmalloc/common.h b/tcmalloc/common.h index 704861807..4e7165c73 100644 --- a/tcmalloc/common.h +++ b/tcmalloc/common.h @@ -248,6 +248,17 @@ enum class AllocationAccess { kCold, }; +inline AllocationAccess AccessFromPointer(void* ptr) { + if (!kHasExpandedClasses) { + TC_ASSERT_NE(GetMemoryTag(ptr), MemoryTag::kCold); + return AllocationAccess::kHot; + } + + return ABSL_PREDICT_FALSE(GetMemoryTag(ptr) == MemoryTag::kCold) + ? AllocationAccess::kCold + : AllocationAccess::kHot; +} + inline MemoryTag MultiNormalTag(size_t partition) { switch (partition) { case 0: diff --git a/tcmalloc/experimental_pow2_size_class.cc b/tcmalloc/experimental_pow2_size_class.cc index 820be1e33..be170038b 100644 --- a/tcmalloc/experimental_pow2_size_class.cc +++ b/tcmalloc/experimental_pow2_size_class.cc @@ -132,7 +132,7 @@ static constexpr SizeClassInfo List[] = { { 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00% { 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00% { 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00% - {131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00% + {131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00% {262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00% }; #elif TCMALLOC_PAGE_SHIFT == 12 @@ -251,7 +251,7 @@ static constexpr SizeClassInfo List[] = { { 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00% { 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00% { 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00% - {131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00% + {131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00% {262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00% }; #elif TCMALLOC_PAGE_SHIFT == 12 diff --git a/tcmalloc/internal/logging.h b/tcmalloc/internal/logging.h index c0cd16d04..f2b675bc2 100644 --- a/tcmalloc/internal/logging.h +++ b/tcmalloc/internal/logging.h @@ -64,6 +64,15 @@ struct StackTrace { // memory block. AllocHandle sampled_alloc_handle; + // For small sampled objects, we allocate a full span to hold the + // sampled object. However to avoid disturbing fragmentation + // profiles, in such cases we also allocate a small proxy object + // using the normal mechanism. + // + // proxy field is defined only for heap sample stack traces. + // For heap samples, proxy==NULL iff size > kMaxSize. + void* proxy; + uintptr_t requested_size; std::optional requested_alignment; uintptr_t allocated_size; // size after sizeclass/page rounding diff --git a/tcmalloc/legacy_size_classes.cc b/tcmalloc/legacy_size_classes.cc index 38f70e817..26d70179e 100644 --- a/tcmalloc/legacy_size_classes.cc +++ b/tcmalloc/legacy_size_classes.cc @@ -98,9 +98,9 @@ static constexpr SizeClassInfo List[] = { { 640, 8192, 32}, // 38 12 6.98% 0.43% 11.11% { 704, 8192, 32}, // 39 11 6.20% 0.43% 10.00% { 768, 8192, 32}, // 40 10 6.98% 0.43% 9.09% - { 896, 8192, 32}, // 41 9 2.33% 0.42% 16.67% + { 896, 8192, 32}, // 41 9 2.33% 0.43% 16.67% { 1024, 8192, 32}, // 42 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 43 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 43 14 1.95% 0.43% 12.50% { 1280, 16384, 32}, // 44 12 6.61% 0.43% 11.11% { 1408, 16384, 32}, // 45 11 5.84% 0.43% 10.00% { 1536, 16384, 32}, // 46 10 6.61% 0.43% 9.09% @@ -228,7 +228,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29% { 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67% + {114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67% {131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00% {196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00% @@ -296,17 +296,17 @@ static constexpr SizeClassInfo List[] = { { 2304, 262144, 28}, // 46 113 0.71% 12.53% 12.50% { 2560, 262144, 25}, // 47 102 0.41% 12.53% 11.11% { 2816, 262144, 23}, // 48 93 0.12% 12.53% 10.00% - { 3200, 262144, 20}, // 49 81 1.15% 12.53% 13.64% + { 3200, 262144, 20}, // 49 81 1.15% 12.54% 13.64% { 3584, 262144, 18}, // 50 73 0.22% 12.53% 12.00% { 4096, 262144, 16}, // 51 64 0.02% 12.53% 14.29% { 4224, 262144, 15}, // 52 62 0.12% 12.53% 3.12% - { 4736, 262144, 13}, // 53 55 0.66% 12.53% 12.12% + { 4736, 262144, 13}, // 53 55 0.66% 12.54% 12.12% { 5120, 262144, 12}, // 54 51 0.41% 12.53% 8.11% { 5632, 262144, 11}, // 55 46 1.20% 12.54% 10.00% { 6144, 262144, 10}, // 56 42 1.59% 12.54% 9.09% { 6528, 262144, 10}, // 57 40 0.41% 12.53% 6.25% { 7040, 262144, 9}, // 58 37 0.66% 12.54% 7.84% - { 7680, 262144, 8}, // 59 34 0.41% 12.53% 9.09% + { 7680, 262144, 8}, // 59 34 0.41% 12.54% 9.09% { 8192, 262144, 8}, // 60 32 0.02% 12.53% 6.67% { 9344, 262144, 7}, // 61 28 0.22% 12.53% 14.06% { 11392, 262144, 5}, // 62 23 0.07% 12.53% 21.92% @@ -330,7 +330,7 @@ static constexpr SizeClassInfo List[] = { { 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09% {112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08% - {131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76% + {131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76% {149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67% {196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53% @@ -439,7 +439,7 @@ static constexpr SizeClassInfo List[] = { { 368, 8192, 32}, // 24 22 1.94% 0.42% 4.55% { 384, 8192, 32}, // 25 21 2.33% 0.42% 4.35% { 400, 8192, 32}, // 26 20 3.10% 0.42% 4.17% - { 416, 8192, 32}, // 27 19 4.26% 0.42% 4.00% + { 416, 8192, 32}, // 27 19 4.26% 0.43% 4.00% { 448, 8192, 32}, // 28 18 2.33% 0.42% 7.69% { 480, 8192, 32}, // 29 17 1.16% 0.42% 7.14% { 512, 8192, 32}, // 30 16 0.78% 0.42% 6.67% @@ -447,9 +447,9 @@ static constexpr SizeClassInfo List[] = { { 640, 8192, 32}, // 32 12 6.98% 0.43% 11.11% { 704, 8192, 32}, // 33 11 6.20% 0.43% 10.00% { 768, 8192, 32}, // 34 10 6.98% 0.43% 9.09% - { 896, 8192, 32}, // 35 9 2.33% 0.42% 16.67% + { 896, 8192, 32}, // 35 9 2.33% 0.43% 16.67% { 1024, 8192, 32}, // 36 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 37 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 37 14 1.95% 0.43% 12.50% { 1280, 16384, 32}, // 38 12 6.61% 0.43% 11.11% { 1408, 16384, 32}, // 39 11 5.84% 0.43% 10.00% { 1536, 16384, 32}, // 40 10 6.61% 0.43% 9.09% @@ -583,7 +583,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29% { 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67% + {114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67% {131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00% {196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00% @@ -647,19 +647,19 @@ static constexpr SizeClassInfo List[] = { { 2304, 262144, 28}, // 42 113 0.71% 12.53% 12.50% { 2560, 262144, 25}, // 43 102 0.41% 12.53% 11.11% { 2816, 262144, 23}, // 44 93 0.12% 12.53% 10.00% - { 3200, 262144, 20}, // 45 81 1.15% 12.53% 13.64% + { 3200, 262144, 20}, // 45 81 1.15% 12.54% 13.64% { 3584, 262144, 18}, // 46 73 0.22% 12.53% 12.00% { 3840, 262144, 17}, // 47 68 0.41% 12.53% 7.14% { 4096, 262144, 16}, // 48 64 0.02% 12.53% 6.67% { 4224, 262144, 15}, // 49 62 0.12% 12.53% 3.12% { 4480, 262144, 14}, // 50 58 0.90% 12.54% 6.06% - { 4736, 262144, 13}, // 51 55 0.66% 12.53% 5.71% + { 4736, 262144, 13}, // 51 55 0.66% 12.54% 5.71% { 5120, 262144, 12}, // 52 51 0.41% 12.53% 8.11% { 5632, 262144, 11}, // 53 46 1.20% 12.54% 10.00% { 6144, 262144, 10}, // 54 42 1.59% 12.54% 9.09% { 6528, 262144, 10}, // 55 40 0.41% 12.53% 6.25% { 7040, 262144, 9}, // 56 37 0.66% 12.54% 7.84% - { 7680, 262144, 8}, // 57 34 0.41% 12.53% 9.09% + { 7680, 262144, 8}, // 57 34 0.41% 12.54% 9.09% { 8192, 262144, 8}, // 58 32 0.02% 12.53% 6.67% { 9344, 262144, 7}, // 59 28 0.22% 12.53% 14.06% { 10368, 262144, 6}, // 60 25 1.15% 12.54% 10.96% @@ -685,7 +685,7 @@ static constexpr SizeClassInfo List[] = { { 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09% {112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08% - {131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76% + {131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76% {149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67% {196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53% diff --git a/tcmalloc/malloc_extension.h b/tcmalloc/malloc_extension.h index 06f954a30..a3bf70e7b 100644 --- a/tcmalloc/malloc_extension.h +++ b/tcmalloc/malloc_extension.h @@ -151,8 +151,7 @@ enum class ProfileType { kHeap, // Fragmentation report - kFragmentation ABSL_DEPRECATED( - "Fragmentation profiles are no longer collected"), + kFragmentation, // Sample of objects that were live at a recent peak of total heap usage. The // specifics of when exactly this profile is collected are subject to change. diff --git a/tcmalloc/reuse_relaxed_below_64_size_classes.cc b/tcmalloc/reuse_relaxed_below_64_size_classes.cc index 03846e8e3..b00ce9eaa 100644 --- a/tcmalloc/reuse_relaxed_below_64_size_classes.cc +++ b/tcmalloc/reuse_relaxed_below_64_size_classes.cc @@ -78,9 +78,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 18 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 19 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 20 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 21 9 2.33% 0.42% 27.27% + { 896, 8192, 32}, // 21 9 2.33% 0.43% 27.27% { 1024, 8192, 32}, // 22 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 23 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 23 14 1.95% 0.43% 12.50% { 1408, 16384, 32}, // 24 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 25 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 26 4 0.78% 0.42% 14.29% @@ -227,7 +227,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% @@ -308,9 +308,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 17 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 18 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 19 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 20 9 2.33% 0.42% 27.27% + { 896, 8192, 32}, // 20 9 2.33% 0.43% 27.27% { 1024, 8192, 32}, // 21 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 22 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 22 14 1.95% 0.43% 12.50% { 1408, 16384, 32}, // 23 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 24 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 25 4 0.78% 0.42% 14.29% @@ -393,7 +393,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -457,7 +457,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% diff --git a/tcmalloc/size_classes.cc b/tcmalloc/size_classes.cc index 951a012c8..a61a0b0e5 100644 --- a/tcmalloc/size_classes.cc +++ b/tcmalloc/size_classes.cc @@ -76,9 +76,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 16 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 17 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 18 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 19 9 2.33% 0.42% 27.27% + { 896, 8192, 32}, // 19 9 2.33% 0.43% 27.27% { 1024, 8192, 32}, // 20 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 21 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 21 14 1.95% 0.43% 12.50% { 1408, 16384, 32}, // 22 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 23 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 24 4 0.78% 0.42% 14.29% @@ -161,7 +161,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -225,7 +225,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% @@ -305,9 +305,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 16 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 17 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 18 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 19 9 2.33% 0.42% 27.27% + { 896, 8192, 32}, // 19 9 2.33% 0.43% 27.27% { 1024, 8192, 32}, // 20 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 21 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 21 14 1.95% 0.43% 12.50% { 1408, 16384, 32}, // 22 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 23 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 24 4 0.78% 0.42% 14.29% @@ -390,7 +390,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -454,7 +454,7 @@ static constexpr SizeClassInfo List[] = { { 74880, 524288, 2}, // 43 7 0.04% 12.53% 14.26% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% diff --git a/tcmalloc/tcmalloc.cc b/tcmalloc/tcmalloc.cc index b9b0fa377..7b2754439 100644 --- a/tcmalloc/tcmalloc.cc +++ b/tcmalloc/tcmalloc.cc @@ -119,7 +119,6 @@ #include "tcmalloc/sampler.h" #include "tcmalloc/segv_handler.h" #include "tcmalloc/span.h" -#include "tcmalloc/stack_trace_table.h" #include "tcmalloc/static_vars.h" #include "tcmalloc/stats.h" #include "tcmalloc/tcmalloc_policy.h" @@ -1194,19 +1193,21 @@ alloc_small_sampled_hooks_or_perthread(size_t size, size_t size_class, size_class = ret.size_class; TC_CHECK(ret.is_small); } - __sized_ptr_t ptr; - if (ABSL_PREDICT_FALSE(weight != 0)) { - ptr = SampleSmallAllocation(tc_globals, policy, size, weight, size_class); - } else { + void* res; + // If we are here because of sampling, try AllocateFast first. + if (ABSL_PREDICT_TRUE(weight == 0) || + (res = tc_globals.cpu_cache().AllocateFast(size_class)) == nullptr) { if (UsePerCpuCache(tc_globals)) { - ptr.p = tc_globals.cpu_cache().AllocateSlow(size_class); + res = tc_globals.cpu_cache().AllocateSlow(size_class); } else { - ptr.p = ThreadCache::GetCache()->Allocate(size_class); + res = ThreadCache::GetCache()->Allocate(size_class); } - ptr.n = tc_globals.sizemap().class_to_size(size_class); + if (ABSL_PREDICT_FALSE(res == nullptr)) return policy.handle_oom(size); } - if (ABSL_PREDICT_FALSE(ptr.p == nullptr)) { - return policy.handle_oom(size); + __sized_ptr_t ptr = {res, tc_globals.sizemap().class_to_size(size_class)}; + if (ABSL_PREDICT_FALSE(weight != 0)) { + ptr = SampleSmallAllocation(tc_globals, policy, size, weight, size_class, + ptr); } if (Policy::invoke_hooks()) { // Size returning tcmallocs call NewHooks with capacity as requested_size.