diff --git a/tcmalloc/allocation_sampling.cc b/tcmalloc/allocation_sampling.cc index 5af8d2358..65115749d 100644 --- a/tcmalloc/allocation_sampling.cc +++ b/tcmalloc/allocation_sampling.cc @@ -58,5 +58,19 @@ std::unique_ptr DumpHeapProfile(Static& state) { return profile; } +ABSL_ATTRIBUTE_NOINLINE void FreeProxyObject(Static& state, void* ptr, + size_t size_class) { + if (ABSL_PREDICT_TRUE(UsePerCpuCache(state))) { + state.cpu_cache().Deallocate(ptr, size_class); + } else if (ThreadCache* cache = ThreadCache::GetCacheIfPresent(); + ABSL_PREDICT_TRUE(cache)) { + cache->Deallocate(ptr, size_class); + } else { + // This thread doesn't have thread-cache yet or already. Delete directly + // into transfer cache. + state.transfer_cache().InsertRange(size_class, absl::Span(&ptr, 1)); + } +} + } // namespace tcmalloc::tcmalloc_internal GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/allocation_sampling.h b/tcmalloc/allocation_sampling.h index 7db362dcc..9753aa7b7 100644 --- a/tcmalloc/allocation_sampling.h +++ b/tcmalloc/allocation_sampling.h @@ -67,9 +67,16 @@ inline Sampler& GetThreadSampler() { #endif } +void FreeProxyObject(Static& state, void* absl_nonnull ptr, size_t size_class); + // Performs sampling for already occurred allocation of object. // -// For small object sizes, we allocate a new object in a sampled span. +// For very small object sizes, object is used as 'proxy' and full +// page with sampled marked is allocated instead. +// +// For medium-sized objects that have single instance per span, +// they're simply freed and fresh page span is allocated to represent +// sampling. // // For large objects (i.e. allocated with do_malloc_pages) they are // also fully reused and their span is marked as sampled. @@ -88,10 +95,12 @@ inline Sampler& GetThreadSampler() { template ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( Static& state, Policy policy, size_t requested_size, size_t weight, - size_t size_class, Span* absl_nullable span) { - TC_CHECK_EQ(size_class != 0, span == nullptr); + size_t size_class, void* absl_nullable obj, Span* absl_nullable span) { + TC_CHECK((size_class != 0 && obj != nullptr && span == nullptr) || + (size_class == 0 && obj == nullptr && span != nullptr)); StackTrace stack_trace; + stack_trace.proxy = nullptr; stack_trace.requested_size = requested_size; // Grab the stack trace outside the heap lock. stack_trace.depth = absl::GetStackTrace(stack_trace.stack, kMaxStackDepth, 0); @@ -122,6 +131,8 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( : MemoryTag::kSampled; size_t capacity = 0; if (size_class != 0) { + TC_ASSERT_EQ(size_class, + state.pagemap().sizeclass(PageIdContainingTagged(obj))); state.per_size_class_counts()[size_class].Add(allocation_estimate); stack_trace.allocated_size = state.sizemap().class_to_size(size_class); @@ -150,10 +161,24 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( } capacity = requested_size; } else if ((span = state.page_allocator().New( - num_pages, {1, AccessDensityPrediction::kSparse}, tag))) { + num_pages, {1, AccessDensityPrediction::kSparse}, tag)) == + nullptr) { capacity = stack_trace.allocated_size; + return {obj, capacity}; } else { - return {}; + capacity = stack_trace.allocated_size; + } + + size_t span_size = + Length(state.sizemap().class_to_pages(size_class)).in_bytes(); + size_t objects_per_span = stack_trace.allocated_size != 0 + ? span_size / stack_trace.allocated_size + : -1ul; + + if (objects_per_span != 1) { + TC_ASSERT_GT(objects_per_span, 1); + stack_trace.proxy = obj; + obj = nullptr; } } else { // Set stack_trace.allocated_size to the exact size for a page allocation. @@ -219,6 +244,12 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( state.peak_heap_tracker().MaybeSaveSample(); + if (obj != nullptr) { + // We are not maintaining precise statistics on malloc hit/miss rates at our + // cache tiers. We can deallocate into our ordinary cache. + TC_ASSERT_NE(size_class, 0); + FreeProxyObject(state, obj, size_class); + } TC_ASSERT_EQ(state.pagemap().sizeclass(span->first_page()), 0); return {(alloc_with_status.alloc != nullptr) ? alloc_with_status.alloc : span->start_address(), @@ -233,15 +264,16 @@ template static sized_ptr_t SampleLargeAllocation(Static& state, Policy policy, size_t requested_size, size_t weight, Span* span) { - return SampleifyAllocation(state, policy, requested_size, weight, 0, span); + return SampleifyAllocation(state, policy, requested_size, weight, 0, nullptr, + span); } template static sized_ptr_t SampleSmallAllocation(Static& state, Policy policy, size_t requested_size, size_t weight, - size_t size_class) { + size_t size_class, sized_ptr_t res) { return SampleifyAllocation(state, policy, requested_size, weight, size_class, - nullptr); + res.p, nullptr); } // Rewrite type so that the allocation type falls into one of the categories we @@ -291,6 +323,7 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, TC_ASSERT_EQ(state.pagemap().sizeclass(PageIdContainingTagged(ptr)), 0); + void* const proxy = sampled_allocation->sampled_stack.proxy; const size_t weight = sampled_allocation->sampled_stack.weight; const size_t requested_size = sampled_allocation->sampled_stack.requested_size; @@ -333,14 +366,18 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, sampled_allocation->sampled_stack.depth)); } + const size_t allocated_alignment = static_cast( + sampled_allocation->sampled_stack.requested_alignment.value_or( + std::align_val_t{1})); + const std::optional deallocated_alignment = + policy.has_explicit_alignment() + ? std::make_optional(policy.align()) + : std::nullopt; + if ((size.has_value() || policy.allocation_type() == AllocationType::New)) { const bool type_mismatch = policy.allocation_type() != sampled_allocation->sampled_stack.allocation_type; - const std::optional deallocated_alignment = - policy.has_explicit_alignment() - ? std::make_optional(policy.align()) - : std::nullopt; const bool alignment_mismatch = deallocated_alignment != sampled_allocation->sampled_stack.requested_alignment; @@ -391,6 +428,23 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, MallocHook::InvokeSampledDeleteHook(sampled_alloc); state.deallocation_samples.ReportFree(sampled_alloc_handle); + + if (proxy) { + const auto proxy_policy = policy.InSamePartitionAs(proxy); + size_t size_class; + if (AccessFromPointer(proxy) == AllocationAccess::kCold) { + size_class = state.sizemap().SizeClass( + proxy_policy.AccessAsCold().AlignAs(allocated_alignment), + allocated_size); + } else { + size_class = state.sizemap().SizeClass( + proxy_policy.AccessAsHot().AlignAs(allocated_alignment), + allocated_size); + } + TC_ASSERT_EQ(size_class, + state.pagemap().sizeclass(PageIdContainingTagged(proxy))); + FreeProxyObject(state, proxy, size_class); + } } } // namespace tcmalloc::tcmalloc_internal diff --git a/tcmalloc/common.h b/tcmalloc/common.h index 704861807..4e7165c73 100644 --- a/tcmalloc/common.h +++ b/tcmalloc/common.h @@ -248,6 +248,17 @@ enum class AllocationAccess { kCold, }; +inline AllocationAccess AccessFromPointer(void* ptr) { + if (!kHasExpandedClasses) { + TC_ASSERT_NE(GetMemoryTag(ptr), MemoryTag::kCold); + return AllocationAccess::kHot; + } + + return ABSL_PREDICT_FALSE(GetMemoryTag(ptr) == MemoryTag::kCold) + ? AllocationAccess::kCold + : AllocationAccess::kHot; +} + inline MemoryTag MultiNormalTag(size_t partition) { switch (partition) { case 0: diff --git a/tcmalloc/experimental_pow2_size_class.cc b/tcmalloc/experimental_pow2_size_class.cc index 820be1e33..be170038b 100644 --- a/tcmalloc/experimental_pow2_size_class.cc +++ b/tcmalloc/experimental_pow2_size_class.cc @@ -132,7 +132,7 @@ static constexpr SizeClassInfo List[] = { { 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00% { 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00% { 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00% - {131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00% + {131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00% {262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00% }; #elif TCMALLOC_PAGE_SHIFT == 12 @@ -251,7 +251,7 @@ static constexpr SizeClassInfo List[] = { { 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00% { 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00% { 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00% - {131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00% + {131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00% {262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00% }; #elif TCMALLOC_PAGE_SHIFT == 12 diff --git a/tcmalloc/internal/logging.h b/tcmalloc/internal/logging.h index c0cd16d04..f2b675bc2 100644 --- a/tcmalloc/internal/logging.h +++ b/tcmalloc/internal/logging.h @@ -64,6 +64,15 @@ struct StackTrace { // memory block. AllocHandle sampled_alloc_handle; + // For small sampled objects, we allocate a full span to hold the + // sampled object. However to avoid disturbing fragmentation + // profiles, in such cases we also allocate a small proxy object + // using the normal mechanism. + // + // proxy field is defined only for heap sample stack traces. + // For heap samples, proxy==NULL iff size > kMaxSize. + void* proxy; + uintptr_t requested_size; std::optional requested_alignment; uintptr_t allocated_size; // size after sizeclass/page rounding diff --git a/tcmalloc/legacy_size_classes.cc b/tcmalloc/legacy_size_classes.cc index 38f70e817..26d70179e 100644 --- a/tcmalloc/legacy_size_classes.cc +++ b/tcmalloc/legacy_size_classes.cc @@ -98,9 +98,9 @@ static constexpr SizeClassInfo List[] = { { 640, 8192, 32}, // 38 12 6.98% 0.43% 11.11% { 704, 8192, 32}, // 39 11 6.20% 0.43% 10.00% { 768, 8192, 32}, // 40 10 6.98% 0.43% 9.09% - { 896, 8192, 32}, // 41 9 2.33% 0.42% 16.67% + { 896, 8192, 32}, // 41 9 2.33% 0.43% 16.67% { 1024, 8192, 32}, // 42 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 43 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 43 14 1.95% 0.43% 12.50% { 1280, 16384, 32}, // 44 12 6.61% 0.43% 11.11% { 1408, 16384, 32}, // 45 11 5.84% 0.43% 10.00% { 1536, 16384, 32}, // 46 10 6.61% 0.43% 9.09% @@ -228,7 +228,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29% { 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67% + {114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67% {131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00% {196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00% @@ -296,17 +296,17 @@ static constexpr SizeClassInfo List[] = { { 2304, 262144, 28}, // 46 113 0.71% 12.53% 12.50% { 2560, 262144, 25}, // 47 102 0.41% 12.53% 11.11% { 2816, 262144, 23}, // 48 93 0.12% 12.53% 10.00% - { 3200, 262144, 20}, // 49 81 1.15% 12.53% 13.64% + { 3200, 262144, 20}, // 49 81 1.15% 12.54% 13.64% { 3584, 262144, 18}, // 50 73 0.22% 12.53% 12.00% { 4096, 262144, 16}, // 51 64 0.02% 12.53% 14.29% { 4224, 262144, 15}, // 52 62 0.12% 12.53% 3.12% - { 4736, 262144, 13}, // 53 55 0.66% 12.53% 12.12% + { 4736, 262144, 13}, // 53 55 0.66% 12.54% 12.12% { 5120, 262144, 12}, // 54 51 0.41% 12.53% 8.11% { 5632, 262144, 11}, // 55 46 1.20% 12.54% 10.00% { 6144, 262144, 10}, // 56 42 1.59% 12.54% 9.09% { 6528, 262144, 10}, // 57 40 0.41% 12.53% 6.25% { 7040, 262144, 9}, // 58 37 0.66% 12.54% 7.84% - { 7680, 262144, 8}, // 59 34 0.41% 12.53% 9.09% + { 7680, 262144, 8}, // 59 34 0.41% 12.54% 9.09% { 8192, 262144, 8}, // 60 32 0.02% 12.53% 6.67% { 9344, 262144, 7}, // 61 28 0.22% 12.53% 14.06% { 11392, 262144, 5}, // 62 23 0.07% 12.53% 21.92% @@ -330,7 +330,7 @@ static constexpr SizeClassInfo List[] = { { 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09% {112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08% - {131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76% + {131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76% {149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67% {196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53% @@ -439,7 +439,7 @@ static constexpr SizeClassInfo List[] = { { 368, 8192, 32}, // 24 22 1.94% 0.42% 4.55% { 384, 8192, 32}, // 25 21 2.33% 0.42% 4.35% { 400, 8192, 32}, // 26 20 3.10% 0.42% 4.17% - { 416, 8192, 32}, // 27 19 4.26% 0.42% 4.00% + { 416, 8192, 32}, // 27 19 4.26% 0.43% 4.00% { 448, 8192, 32}, // 28 18 2.33% 0.42% 7.69% { 480, 8192, 32}, // 29 17 1.16% 0.42% 7.14% { 512, 8192, 32}, // 30 16 0.78% 0.42% 6.67% @@ -447,9 +447,9 @@ static constexpr SizeClassInfo List[] = { { 640, 8192, 32}, // 32 12 6.98% 0.43% 11.11% { 704, 8192, 32}, // 33 11 6.20% 0.43% 10.00% { 768, 8192, 32}, // 34 10 6.98% 0.43% 9.09% - { 896, 8192, 32}, // 35 9 2.33% 0.42% 16.67% + { 896, 8192, 32}, // 35 9 2.33% 0.43% 16.67% { 1024, 8192, 32}, // 36 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 37 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 37 14 1.95% 0.43% 12.50% { 1280, 16384, 32}, // 38 12 6.61% 0.43% 11.11% { 1408, 16384, 32}, // 39 11 5.84% 0.43% 10.00% { 1536, 16384, 32}, // 40 10 6.61% 0.43% 9.09% @@ -583,7 +583,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29% { 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67% + {114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67% {131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00% {196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00% @@ -647,19 +647,19 @@ static constexpr SizeClassInfo List[] = { { 2304, 262144, 28}, // 42 113 0.71% 12.53% 12.50% { 2560, 262144, 25}, // 43 102 0.41% 12.53% 11.11% { 2816, 262144, 23}, // 44 93 0.12% 12.53% 10.00% - { 3200, 262144, 20}, // 45 81 1.15% 12.53% 13.64% + { 3200, 262144, 20}, // 45 81 1.15% 12.54% 13.64% { 3584, 262144, 18}, // 46 73 0.22% 12.53% 12.00% { 3840, 262144, 17}, // 47 68 0.41% 12.53% 7.14% { 4096, 262144, 16}, // 48 64 0.02% 12.53% 6.67% { 4224, 262144, 15}, // 49 62 0.12% 12.53% 3.12% { 4480, 262144, 14}, // 50 58 0.90% 12.54% 6.06% - { 4736, 262144, 13}, // 51 55 0.66% 12.53% 5.71% + { 4736, 262144, 13}, // 51 55 0.66% 12.54% 5.71% { 5120, 262144, 12}, // 52 51 0.41% 12.53% 8.11% { 5632, 262144, 11}, // 53 46 1.20% 12.54% 10.00% { 6144, 262144, 10}, // 54 42 1.59% 12.54% 9.09% { 6528, 262144, 10}, // 55 40 0.41% 12.53% 6.25% { 7040, 262144, 9}, // 56 37 0.66% 12.54% 7.84% - { 7680, 262144, 8}, // 57 34 0.41% 12.53% 9.09% + { 7680, 262144, 8}, // 57 34 0.41% 12.54% 9.09% { 8192, 262144, 8}, // 58 32 0.02% 12.53% 6.67% { 9344, 262144, 7}, // 59 28 0.22% 12.53% 14.06% { 10368, 262144, 6}, // 60 25 1.15% 12.54% 10.96% @@ -685,7 +685,7 @@ static constexpr SizeClassInfo List[] = { { 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09% {112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08% - {131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76% + {131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76% {149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67% {196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53% diff --git a/tcmalloc/malloc_extension.h b/tcmalloc/malloc_extension.h index 06f954a30..a3bf70e7b 100644 --- a/tcmalloc/malloc_extension.h +++ b/tcmalloc/malloc_extension.h @@ -151,8 +151,7 @@ enum class ProfileType { kHeap, // Fragmentation report - kFragmentation ABSL_DEPRECATED( - "Fragmentation profiles are no longer collected"), + kFragmentation, // Sample of objects that were live at a recent peak of total heap usage. The // specifics of when exactly this profile is collected are subject to change. diff --git a/tcmalloc/reuse_relaxed_below_64_size_classes.cc b/tcmalloc/reuse_relaxed_below_64_size_classes.cc index 03846e8e3..b00ce9eaa 100644 --- a/tcmalloc/reuse_relaxed_below_64_size_classes.cc +++ b/tcmalloc/reuse_relaxed_below_64_size_classes.cc @@ -78,9 +78,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 18 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 19 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 20 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 21 9 2.33% 0.42% 27.27% + { 896, 8192, 32}, // 21 9 2.33% 0.43% 27.27% { 1024, 8192, 32}, // 22 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 23 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 23 14 1.95% 0.43% 12.50% { 1408, 16384, 32}, // 24 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 25 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 26 4 0.78% 0.42% 14.29% @@ -227,7 +227,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% @@ -308,9 +308,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 17 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 18 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 19 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 20 9 2.33% 0.42% 27.27% + { 896, 8192, 32}, // 20 9 2.33% 0.43% 27.27% { 1024, 8192, 32}, // 21 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 22 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 22 14 1.95% 0.43% 12.50% { 1408, 16384, 32}, // 23 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 24 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 25 4 0.78% 0.42% 14.29% @@ -393,7 +393,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -457,7 +457,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% diff --git a/tcmalloc/size_classes.cc b/tcmalloc/size_classes.cc index 951a012c8..a61a0b0e5 100644 --- a/tcmalloc/size_classes.cc +++ b/tcmalloc/size_classes.cc @@ -76,9 +76,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 16 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 17 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 18 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 19 9 2.33% 0.42% 27.27% + { 896, 8192, 32}, // 19 9 2.33% 0.43% 27.27% { 1024, 8192, 32}, // 20 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 21 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 21 14 1.95% 0.43% 12.50% { 1408, 16384, 32}, // 22 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 23 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 24 4 0.78% 0.42% 14.29% @@ -161,7 +161,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -225,7 +225,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% @@ -305,9 +305,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 16 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 17 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 18 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 19 9 2.33% 0.42% 27.27% + { 896, 8192, 32}, // 19 9 2.33% 0.43% 27.27% { 1024, 8192, 32}, // 20 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 21 14 1.95% 0.42% 12.50% + { 1152, 16384, 32}, // 21 14 1.95% 0.43% 12.50% { 1408, 16384, 32}, // 22 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 23 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 24 4 0.78% 0.42% 14.29% @@ -390,7 +390,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -454,7 +454,7 @@ static constexpr SizeClassInfo List[] = { { 74880, 524288, 2}, // 43 7 0.04% 12.53% 14.26% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% diff --git a/tcmalloc/tcmalloc.cc b/tcmalloc/tcmalloc.cc index b9b0fa377..7b2754439 100644 --- a/tcmalloc/tcmalloc.cc +++ b/tcmalloc/tcmalloc.cc @@ -119,7 +119,6 @@ #include "tcmalloc/sampler.h" #include "tcmalloc/segv_handler.h" #include "tcmalloc/span.h" -#include "tcmalloc/stack_trace_table.h" #include "tcmalloc/static_vars.h" #include "tcmalloc/stats.h" #include "tcmalloc/tcmalloc_policy.h" @@ -1194,19 +1193,21 @@ alloc_small_sampled_hooks_or_perthread(size_t size, size_t size_class, size_class = ret.size_class; TC_CHECK(ret.is_small); } - __sized_ptr_t ptr; - if (ABSL_PREDICT_FALSE(weight != 0)) { - ptr = SampleSmallAllocation(tc_globals, policy, size, weight, size_class); - } else { + void* res; + // If we are here because of sampling, try AllocateFast first. + if (ABSL_PREDICT_TRUE(weight == 0) || + (res = tc_globals.cpu_cache().AllocateFast(size_class)) == nullptr) { if (UsePerCpuCache(tc_globals)) { - ptr.p = tc_globals.cpu_cache().AllocateSlow(size_class); + res = tc_globals.cpu_cache().AllocateSlow(size_class); } else { - ptr.p = ThreadCache::GetCache()->Allocate(size_class); + res = ThreadCache::GetCache()->Allocate(size_class); } - ptr.n = tc_globals.sizemap().class_to_size(size_class); + if (ABSL_PREDICT_FALSE(res == nullptr)) return policy.handle_oom(size); } - if (ABSL_PREDICT_FALSE(ptr.p == nullptr)) { - return policy.handle_oom(size); + __sized_ptr_t ptr = {res, tc_globals.sizemap().class_to_size(size_class)}; + if (ABSL_PREDICT_FALSE(weight != 0)) { + ptr = SampleSmallAllocation(tc_globals, policy, size, weight, size_class, + ptr); } if (Policy::invoke_hooks()) { // Size returning tcmallocs call NewHooks with capacity as requested_size.