feat(profiling): add sample count to internal payload (#15177)

## Description

This PR adds data to the `internal` payload we send to `libdatadog`. In
this `internal` payload, we can push custom metrics (that are not
exposed to customers) but that we can access for analytics.

For the time being, I only added the number of Samples taken (one per
thread) and Sampling Events (one for each time we run `for_each_interp`)
for the current Profile. We may want to add more in the short term (e.g.
number of times adaptive sampling was adjusted, history of adaptive
sampling intervals, etc.)

In order to implement this feature in a way that keeps clear semantics
around locking the Profile object, I refactored the code not to `borrow`
only the single `Profile` but instead both the `Profile` and the
`ProfilerStats` (in which we store our metrics). To keep locking clear
and explicit, locking returns a `ProfileBorrow` objects which allows to
access both the `Profile` and the `ProfilerStats` and that is
RAII-compatible (automatically unlocks when it goes out of scope).

Note the PR does add some "lock contention" because we now need to take
the Profile lock in two new places (one for each metric we increment) –
it should be the same order of magnitude as what we already do though
(in number of lock acquisitions).
We plan to refactor Stack V2 in the short to avoid having to hold the
Profile / ProfilerStats lock during the upload, by using double
buffering or by releasing earlier (after the Profile data has been
serialised). This should improve the performance and reduce lost
Samples.

The PR also adds an integration test, ensuring that the generated JSON
string is correct (we can't guarantee exact numbers, but we can check
the numbers we have are meaningful).

**Note** I marked the PR as `no-changelog` as this feature isn't
public/visible by customers.

Open questions:
*  What other metrics do we want to add?

This is an example of querying through the Events UI.

<img width="886" height="604" alt="image"
src="https://github.com/user-attachments/assets/53c3eae0-d29a-4446-9f1b-72ac29a75d60"
/>

Author: KowalskiThomas

ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt

@@ -54,6 +54,8 @@ add_library(
     src/code_provenance_interface.cpp
     src/ddup_interface.cpp
     src/profile.cpp
+    src/profile_borrow.cpp
+    src/profiler_stats.cpp
     src/sample.cpp
     src/sample_manager.cpp
     src/static_sample_pool.cpp

ddtrace/internal/datadog/profiling/dd_wrapper/include/ddup_interface.hpp

@@ -1,14 +1,13 @@
 #pragma once
 
-#include <stddef.h>
-#include <stdint.h>
+#include <cstdint>
 #include <string_view>
 #include <unordered_map>
 
 // Forward decl of the return pointer
 namespace Datadog {
 class Sample;
-}
+} // namespace Datadog
 
 #ifdef __cplusplus
 extern "C"
@@ -68,6 +67,10 @@ extern "C"
                          int64_t line);
     void ddup_push_absolute_ns(Datadog::Sample* sample, int64_t timestamp_ns);
     void ddup_push_monotonic_ns(Datadog::Sample* sample, int64_t monotonic_ns);
+
+    void ddup_increment_sampling_event_count();
+    void ddup_increment_sample_count();
+
     void ddup_flush_sample(Datadog::Sample* sample);
     // Stack v2 specific flush, which reverses the locations
     void ddup_flush_sample_v2(Datadog::Sample* sample);

ddtrace/internal/datadog/profiling/dd_wrapper/include/profile.hpp

@@ -1,13 +1,11 @@
 #pragma once
 
 #include "constants.hpp"
+#include "profiler_stats.hpp"
 #include "types.hpp"
 
 #include <atomic>
-#include <memory>
 #include <mutex>
-#include <string>
-#include <string_view>
 #include <vector>
 
 extern "C"
@@ -17,9 +15,13 @@ extern "C"
 
 namespace Datadog {
 
+class ProfileBorrow;
+
 // Serves to collect individual samples, as well as lengthen the scope of string data
 class Profile
 {
+    friend class ProfileBorrow;
+
   private:
     // Serialization for static state
     // - string table
@@ -45,6 +47,12 @@ class Profile
     // cannot be used until it's initialized by libdatadog
     ddog_prof_Profile cur_profile{};
 
+    Datadog::ProfilerStats profiler_stats{};
+
+    // Internal access methods - not for direct use
+    ddog_prof_Profile& profile_borrow_internal();
+    void profile_release();
+
   public:
     // State management
     void one_time_init(SampleType type, unsigned int _max_nframes);
@@ -53,8 +61,8 @@ class Profile
 
     // Getters
     size_t get_sample_type_length();
-    ddog_prof_Profile& profile_borrow();
-    void profile_release();
+
+    ProfileBorrow borrow();
 
     // constref getters
     const ValueIndex& val();

ddtrace/internal/datadog/profiling/dd_wrapper/include/profile_borrow.hpp

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "profile.hpp"
+
+namespace Datadog {
+
+// Forward declaration
+class Profile;
+
+// RAII wrapper for borrowing both profile and stats under a single lock
+class ProfileBorrow
+{
+  private:
+    Profile* profile_ptr;
+
+  public:
+    explicit ProfileBorrow(Profile& profile);
+    ~ProfileBorrow();
+
+    // Disable copy
+    ProfileBorrow(const ProfileBorrow&) = delete;
+    ProfileBorrow& operator=(const ProfileBorrow&) = delete;
+
+    // Enable move
+    ProfileBorrow(ProfileBorrow&& other) noexcept;
+    ProfileBorrow& operator=(ProfileBorrow&& other) noexcept;
+
+    // Accessors
+    ddog_prof_Profile& profile();
+    ProfilerStats& stats();
+};
+
+} // namespace Datadog

ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <cstddef>
+
+#include <string>
+#include <string_view>
+
+namespace Datadog {
+
+/*
+ProfilerStats holds statistics around Profiling to be sent along
+with the actual Profiles.
+
+None of its methods are thread-safe and it should typically used with
+a mutex to protect access to the data.
+*/
+class ProfilerStats
+{
+  private:
+    std::string internal_metadata_json;
+
+    // Number of samples collected (one per thread)
+    size_t sample_count = 0;
+
+    // Number of sampling events (one per collection cycle)
+    size_t sampling_event_count = 0;
+
+  public:
+    ProfilerStats() = default;
+    ~ProfilerStats() = default;
+
+    void increment_sample_count(size_t k_sample_count = 1);
+    size_t get_sample_count();
+
+    void increment_sampling_event_count(size_t k_sampling_event_count = 1);
+    size_t get_sampling_event_count();
+
+    // Returns a JSON string containing relevant Profiler Stats to be included
+    // in the libdatadog payload.
+    // The function returned a string_view to a statically allocated string that
+    // is updated every time the function is called.
+    std::string_view get_internal_metadata_json();
+
+    void reset_state();
+};
+
+} // namespace Datadog

ddtrace/internal/datadog/profiling/dd_wrapper/include/sample.hpp

@@ -2,6 +2,7 @@
 
 #include "libdatadog_helpers.hpp"
 #include "profile.hpp"
+#include "profile_borrow.hpp"
 #include "types.hpp"
 
 #include <string>
@@ -134,8 +135,7 @@ class Sample
     // Flushes the current buffer, clearing it
     bool flush_sample(bool reverse_locations = false);
 
-    static ddog_prof_Profile& profile_borrow();
-    static void profile_release();
+    static ProfileBorrow profile_borrow();
     static void postfork_child();
     Sample(SampleType _type_mask, unsigned int _max_nframes);
 

ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader.hpp

@@ -1,11 +1,10 @@
 #pragma once
 
-#include "sample.hpp"
-#include "types.hpp"
+#include "profiler_stats.hpp"
 
 #include <atomic>
-#include <memory>
 #include <mutex>
+#include <string>
 
 extern "C"
 {
@@ -24,10 +23,10 @@ class Uploader
     std::string output_filename;
     ddog_prof_ProfileExporter ddog_exporter{ .inner = nullptr };
 
-    bool export_to_file(ddog_prof_EncodedProfile* encoded);
+    bool export_to_file(ddog_prof_EncodedProfile* encoded, std::string_view internal_metadata_json);
 
   public:
-    bool upload(ddog_prof_Profile& profile);
+    bool upload(ddog_prof_Profile& profile, Datadog::ProfilerStats& profiler_stats);
     static void cancel_inflight();
     static void lock();
     static void unlock();

ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader_builder.hpp

@@ -1,8 +1,8 @@
 #pragma once
 
+#include "constants.hpp"
 #include "uploader.hpp"
 
-#include <mutex>
 #include <string>
 #include <string_view>
 #include <unordered_map>

ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp

@@ -1,8 +1,7 @@
 #include "ddup_interface.hpp"
 
-#include "code_provenance.hpp"
 #include "libdatadog_helpers.hpp"
-#include "profile.hpp"
+#include "profiler_stats.hpp"
 #include "sample.hpp"
 #include "sample_manager.hpp"
 #include "uploader.hpp"
@@ -304,6 +303,20 @@ ddup_push_monotonic_ns(Datadog::Sample* sample, int64_t monotonic_ns) // cppchec
     sample->push_monotonic_ns(monotonic_ns);
 }
 
+void
+ddup_increment_sampling_event_count() // cppcheck-suppress unusedFunction
+{
+    auto borrowed = Datadog::Sample::profile_borrow();
+    borrowed.stats().increment_sampling_event_count();
+}
+
+void
+ddup_increment_sample_count() // cppcheck-suppress unusedFunction
+{
+    auto borrowed = Datadog::Sample::profile_borrow();
+    borrowed.stats().increment_sample_count();
+}
+
 void
 ddup_flush_sample(Datadog::Sample* sample) // cppcheck-suppress unusedFunction
 {
@@ -351,17 +364,19 @@ ddup_upload() // cppcheck-suppress unusedFunction
     //  be modified.  It gets released and cleared after uploading.
     // * Uploading cancels inflight uploads. There are better ways to do this, but this is what
     //   we have for now.
-    uploader.upload(Datadog::Sample::profile_borrow());
-    Datadog::Sample::profile_release();
-    return true;
+    auto borrowed = Datadog::Sample::profile_borrow();
+    bool success = uploader.upload(borrowed.profile(), borrowed.stats());
+    borrowed.stats().reset_state();
+    return success;
 }
 
 void
 ddup_profile_set_endpoints(
   std::unordered_map<int64_t, std::string_view> span_ids_to_endpoints) // cppcheck-suppress unusedFunction
 {
     static bool already_warned = false; // cppcheck-suppress threadsafety-threadsafety
-    ddog_prof_Profile& profile = Datadog::Sample::profile_borrow();
+    auto borrowed = Datadog::Sample::profile_borrow();
+    ddog_prof_Profile& profile = borrowed.profile();
     for (const auto& [span_id, trace_endpoint] : span_ids_to_endpoints) {
         ddog_CharSlice trace_endpoint_slice = Datadog::to_slice(trace_endpoint);
         auto res = ddog_prof_Profile_set_endpoint(&profile, span_id, trace_endpoint_slice);
@@ -375,14 +390,14 @@ ddup_profile_set_endpoints(
             ddog_Error_drop(&err);
         }
     }
-    Datadog::Sample::profile_release();
 }
 
 void
 ddup_profile_add_endpoint_counts(std::unordered_map<std::string_view, int64_t> trace_endpoints_to_counts)
 {
     static bool already_warned = false; // cppcheck-suppress threadsafety-threadsafety
-    ddog_prof_Profile& profile = Datadog::Sample::profile_borrow();
+    auto borrowed = Datadog::Sample::profile_borrow();
+    ddog_prof_Profile& profile = borrowed.profile();
     for (const auto& [trace_endpoint, count] : trace_endpoints_to_counts) {
         ddog_CharSlice trace_endpoint_slice = Datadog::to_slice(trace_endpoint);
         auto res = ddog_prof_Profile_add_endpoint_count(&profile, trace_endpoint_slice, count);
@@ -396,5 +411,4 @@ ddup_profile_add_endpoint_counts(std::unordered_map<std::string_view, int64_t> t
             ddog_Error_drop(&err);
         }
     }
-    Datadog::Sample::profile_release();
 }

ddtrace/internal/datadog/profiling/dd_wrapper/src/profile.cpp

@@ -1,6 +1,8 @@
 #include "profile.hpp"
 
 #include "libdatadog_helpers.hpp"
+#include "profile_borrow.hpp"
+#include "profiler_stats.hpp"
 
 #include <functional>
 #include <iostream>
@@ -55,6 +57,8 @@ Datadog::Profile::reset_profile()
         ddog_Error_drop(&err);
         return false;
     }
+
+    profiler_stats.reset_state();
     return true;
 }
 
@@ -127,11 +131,16 @@ Datadog::Profile::get_sample_type_length()
     return samplers.size();
 }
 
+Datadog::ProfileBorrow
+Datadog::Profile::borrow()
+{
+    return ProfileBorrow(*this);
+}
+
 ddog_prof_Profile&
-Datadog::Profile::profile_borrow()
+Datadog::Profile::profile_borrow_internal()
 {
-    // We could wrap this in an object for better RAII, but since this
-    // sequence is only used in a single place, we'll hold off on that sidequest.
+    // Note: Caller is responsible for ensuring profile_release() is called
     profile_mtx.lock();
     return cur_profile;
 }
@@ -219,5 +228,6 @@ Datadog::Profile::postfork_child()
 {
     new (&profile_mtx) std::mutex();
     // Reset the profile to clear any samples collected in the parent process
+    profiler_stats.reset_state();
     reset_profile();
 }