feat: OTel context sharing

Makefile

@@ -48,7 +48,7 @@ RUN_TESTS_CMD := DD_SERVICE= DD_ENV= DD_TRACE_RETRY_INTERVAL=1 REPORT_EXIT_STATU
 
 C_FILES = $(shell find components components-rs ext src/dogstatsd tracer zend_abstract_interface -name '*.c' -o -name '*.h' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )
 TEST_FILES = $(shell find tests/ext -name '*.php*' -o -name '*.inc' -o -name '*.json' -o -name '*.yaml' -o -name 'CONFLICTS' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )
-RUST_FILES = $(BUILD_DIR)/Cargo.toml $(BUILD_DIR)/Cargo.lock $(shell find components-rs -name '*.c' -o -name '*.rs' -o -name 'Cargo.toml' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' ) $(shell find libdatadog/{build-common,datadog-ffe,datadog-ipc,datadog-ipc-macros,datadog-live-debugger,datadog-live-debugger-ffi,libdd-remote-config,datadog-sidecar,datadog-sidecar-ffi,datadog-sidecar-macros,libdd-alloc,libdd-capabilities,libdd-capabilities-impl,libdd-common,libdd-common-ffi,libdd-crashtracker,libdd-crashtracker-ffi,libdd-data-pipeline,libdd-ddsketch,libdd-dogstatsd-client,libdd-library-config,libdd-library-config-ffi,libdd-log,libdd-shared-runtime,libdd-telemetry,libdd-telemetry-ffi,libdd-tinybytes,libdd-trace-*,spawn_worker,tools/{cc_utils,sidecar_mockgen},libdd-trace-*,Cargo.toml} \( -type l -o -type f \) \( -path "*/src*" -o -path "*/examples*" -o -path "*Cargo.toml" -o -path "*/build.rs" -o -path "*/tests/dataservice.rs" -o -path "*/tests/service_functional.rs" \) -not -path "*/datadog-ipc/build.rs" -not -path "*/datadog-sidecar-ffi/build.rs")
+RUST_FILES = $(BUILD_DIR)/Cargo.toml $(BUILD_DIR)/Cargo.lock $(shell find components-rs -name '*.c' -o -name '*.rs' -o -name 'Cargo.toml' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' ) $(shell find libdatadog/{build-common,datadog-ffe,datadog-ipc,datadog-ipc-macros,datadog-live-debugger,datadog-live-debugger-ffi,libdd-remote-config,datadog-sidecar,datadog-sidecar-ffi,datadog-sidecar-macros,libdd-alloc,libdd-capabilities,libdd-capabilities-impl,libdd-common,libdd-common-ffi,libdd-crashtracker,libdd-crashtracker-ffi,libdd-data-pipeline,libdd-ddsketch,libdd-dogstatsd-client,libdd-library-config,libdd-library-config-ffi,libdd-log,libdd-otel-thread-ctx*,libdd-shared-runtime,libdd-telemetry,libdd-telemetry-ffi,libdd-tinybytes,libdd-trace-*,spawn_worker,tools/{cc_utils,sidecar_mockgen},libdd-trace-*,Cargo.toml} \( -type l -o -type f \) \( -path "*/src*" -o -path "*/examples*" -o -path "*Cargo.toml" -o -path "*/build.rs" -o -path "*/tests/dataservice.rs" -o -path "*/tests/service_functional.rs" -o -name "tls-dynamic-list.txt" \) -not -path "*/datadog-ipc/build.rs" -not -path "*/datadog-sidecar-ffi/build.rs")
 ALL_OBJECT_FILES = $(C_FILES) $(RUST_FILES) $(BUILD_DIR)/Makefile
 TEST_OPCACHE_FILES = $(shell find tests/opcache -name '*.php*' -o -name '.gitkeep' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )
 TEST_STUB_FILES = $(shell find tests/ext -type d -name 'stubs' -exec find '{}' -type f \; | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )
@@ -423,9 +423,9 @@ clang_format_fix:
 cbindgen: remove_cbindgen generate_cbindgen
 
 remove_cbindgen:
-	rm -f components-rs/datadog.h components-rs/live-debugger.h components-rs/telemetry.h components-rs/sidecar.h components-rs/common.h components-rs/crashtracker.h components-rs/library-config.h
+	rm -f components-rs/datadog.h components-rs/live-debugger.h components-rs/telemetry.h components-rs/sidecar.h components-rs/common.h components-rs/crashtracker.h components-rs/library-config.h components-rs/otel-thread-ctx.h
 
-generate_cbindgen: cbindgen_binary # Regenerate components-rs/datadog.h components-rs/live-debugger.h components-rs/telemetry.h components-rs/sidecar.h components-rs/common.h components-rs/crashtracker.h components-rs/library-config.h
+generate_cbindgen: cbindgen_binary # Regenerate components-rs/datadog.h components-rs/live-debugger.h components-rs/telemetry.h components-rs/sidecar.h components-rs/common.h components-rs/crashtracker.h components-rs/library-config.h components-rs/otel-thread-ctx.h
 	( \
 		$(command rustup && echo run nightly --) cbindgen --crate datadog-php  \
 			--config cbindgen.toml \
@@ -449,11 +449,14 @@ generate_cbindgen: cbindgen_binary # Regenerate components-rs/datadog.h componen
 		$(command rustup && echo run nightly --) cbindgen --crate libdd-library-config-ffi  \
 			--config libdd-library-config-ffi/cbindgen.toml \
 			--output $(PROJECT_ROOT)/components-rs/library-config.h; \
+		$(command rustup && echo run nightly --) cbindgen --crate libdd-otel-thread-ctx-ffi  \
+			--config libdd-otel-thread-ctx-ffi/cbindgen.toml \
+			--output $(PROJECT_ROOT)/components-rs/otel-thread-ctx.h; \
 		if test -d $(PROJECT_ROOT)/tmp; then \
 			mkdir -pv "$(BUILD_DIR)"; \
 			export CARGO_TARGET_DIR="$(BUILD_DIR)/target"; \
 		fi; \
-		cargo run -p tools --bin dedup_headers -- $(PROJECT_ROOT)/components-rs/common.h $(PROJECT_ROOT)/components-rs/datadog.h $(PROJECT_ROOT)/components-rs/live-debugger.h $(PROJECT_ROOT)/components-rs/telemetry.h $(PROJECT_ROOT)/components-rs/sidecar.h $(PROJECT_ROOT)/components-rs/crashtracker.h $(PROJECT_ROOT)/components-rs/library-config.h \
+		cargo run -p tools --bin dedup_headers -- $(PROJECT_ROOT)/components-rs/common.h $(PROJECT_ROOT)/components-rs/datadog.h $(PROJECT_ROOT)/components-rs/live-debugger.h $(PROJECT_ROOT)/components-rs/telemetry.h $(PROJECT_ROOT)/components-rs/sidecar.h $(PROJECT_ROOT)/components-rs/crashtracker.h $(PROJECT_ROOT)/components-rs/library-config.h $(PROJECT_ROOT)/components-rs/otel-thread-ctx.h \
 	)
 
 cbindgen_binary:

appsec/cmake/ddtrace.cmake

@@ -34,9 +34,11 @@ add_custom_target(ddtrace_exports
 elseif(APPLE)
 set(EXPORTS_FILE "${CMAKE_BINARY_DIR}/datadog_exports.sym")
 add_custom_target(ddtrace_exports
-    COMMAND sed "s/^/_/" "${CMAKE_SOURCE_DIR}/../datadog.sym" > "${EXPORTS_FILE}"
+    # otel_thread_ctx_v1 is a Linux-only TLS symbol; omit it on macOS where tls_shim.c is not compiled
+    COMMAND bash -c "grep -v ^otel_thread_ctx_v1$ '${CMAKE_SOURCE_DIR}'/../datadog.sym | sed 's/^/_/' > '${EXPORTS_FILE}'"
     BYPRODUCTS ${EXPORTS_FILE}
     DEPENDS ${CMAKE_SOURCE_DIR}/../datadog.sym
+    VERBATIM
 )
 endif()
 
@@ -92,6 +94,8 @@ file(GLOB_RECURSE FILES_DDTRACE
     CONFIGURE_DEPENDS
     "${CMAKE_SOURCE_DIR}/../ext/*.c"
     "${CMAKE_SOURCE_DIR}/../ext/**/*.c"
+    "${CMAKE_SOURCE_DIR}/../tracer/*.c"
+    "${CMAKE_SOURCE_DIR}/../tracer/**/*.c"
     "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/*.c"
     "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/**/*.c"
 )
@@ -103,28 +107,28 @@ list(APPEND FILES_DDTRACE
     "${CMAKE_SOURCE_DIR}/../components/string_view/string_view.c"
 )
 if (PhpConfig_VERNUM GREATER_EQUAL 80000)
-    list(REMOVE_ITEM FILES_DDTRACE "${CMAKE_SOURCE_DIR}/../ext/handlers_curl_php7.c"
+    list(REMOVE_ITEM FILES_DDTRACE "${CMAKE_SOURCE_DIR}/../tracer/handlers_curl_php7.c"
         "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/interceptor/php7/interceptor.c"
         "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/interceptor/php7/resolver.c"
         "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/sandbox/php7/sandbox.c")
 else() # PHP 7
-    list(REMOVE_ITEM FILES_DDTRACE "${CMAKE_SOURCE_DIR}/../ext/handlers_curl.c"
-        "${CMAKE_SOURCE_DIR}/../ext/hook/uhook_attributes.c"
-        "${CMAKE_SOURCE_DIR}/../ext/hook/uhook_otel.c"
+    list(REMOVE_ITEM FILES_DDTRACE "${CMAKE_SOURCE_DIR}/../tracer/handlers_curl.c"
+        "${CMAKE_SOURCE_DIR}/../tracer/hook/uhook_attributes.c"
+        "${CMAKE_SOURCE_DIR}/../tracer/hook/uhook_otel.c"
         "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/interceptor/php8/interceptor.c"
         "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/interceptor/php8/resolver.c"
         "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/interceptor/php8/resolver_pre-8_2.c"
         "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/jit_utils/jit_blacklist.c"
         "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/sandbox/php8/sandbox.c")
 endif()
 if (PhpConfig_VERNUM LESS 80200)
-    list(REMOVE_ITEM FILES_DDTRACE "${CMAKE_SOURCE_DIR}/../ext/weakrefs.c")
+    list(REMOVE_ITEM FILES_DDTRACE "${CMAKE_SOURCE_DIR}/../tracer/weakrefs.c")
     list(REMOVE_ITEM FILES_DDTRACE "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/interceptor/php8/resolver.c")
 else() # PHP 8.2+
     list(REMOVE_ITEM FILES_DDTRACE "${CMAKE_SOURCE_DIR}/../zend_abstract_interface/interceptor/php8/resolver_pre-8_2.c")
 endif()
 if (PhpConfig_VERNUM LESS 80100)
-    list(REMOVE_ITEM FILES_DDTRACE "${CMAKE_SOURCE_DIR}/../ext/handlers_fiber.c")
+    list(REMOVE_ITEM FILES_DDTRACE "${CMAKE_SOURCE_DIR}/../tracer/handlers_fiber.c")
 endif()
 list(REMOVE_ITEM FILES_DDTRACE "${CMAKE_SOURCE_DIR}/../ext/crashtracking_windows.c")
 
@@ -162,7 +166,7 @@ endif()
 if(CURL_DEFINITIONS)
     target_compile_definitions(ddtrace PRIVATE ${CURL_DEFINITIONS})
 endif()
-target_compile_definitions(ddtrace PRIVATE ZEND_ENABLE_STATIC_TSRMLS_CACHE=1 COMPILE_DL_DDTRACE=1)
+target_compile_definitions(ddtrace PRIVATE ZEND_ENABLE_STATIC_TSRMLS_CACHE=1 COMPILE_DL_DDTRACE=1 DDTRACE)
 target_include_directories(ddtrace PRIVATE
     ${CURL_INCLUDE_DIRS}
     ${CMAKE_SOURCE_DIR}/..
@@ -171,6 +175,8 @@ target_include_directories(ddtrace PRIVATE
     ${CMAKE_SOURCE_DIR}/../ext
     ${CMAKE_SOURCE_DIR}/../ext/vendor
     ${CMAKE_SOURCE_DIR}/../ext/vendor/mt19937
+    ${CMAKE_SOURCE_DIR}/../tracer
+    ${CMAKE_SOURCE_DIR}/../tracer/vendor
     ${CMAKE_BINARY_DIR}/gen_ddtrace
 )
 add_dependencies(ddtrace ddtrace_exports update_version_h)

appsec/src/extension/ddappsec.c

@@ -14,6 +14,7 @@
 #include <string.h>
 #include <sys/stat.h>
 #include <sys/types.h>
+#include <unistd.h>
 
 #include <pthread.h>
 #include <stdatomic.h>
@@ -55,6 +56,7 @@ static atomic_int _thread_count;
 static void _check_enabled(void);
 #ifdef TESTING
 static void _register_testing_objects(void);
+volatile int ddappsec_debugger_wait_continue;
 #endif
 
 static PHP_MINIT_FUNCTION(ddappsec);
@@ -481,6 +483,32 @@ static PHP_FUNCTION(datadog_appsec_testing_stop_for_debugger)
     RETURN_TRUE;
 }
 
+static PHP_FUNCTION(datadog_appsec_testing_wait_for_debugger)
+{
+    if (zend_parse_parameters_none() == FAILURE) {
+        RETURN_FALSE;
+    }
+    ddappsec_debugger_wait_continue = 0;
+
+    int fd = open(
+        "/tmp/pid", O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0600); // NOLINT
+    if (fd < 0) {
+        RETURN_FALSE;
+    }
+    char pid[sizeof("-2147483648")] = "";
+    sprintf(pid, "%" PRIi32, (int32_t)getpid()); // NOLINT
+    ATTR_UNUSED ssize_t unused_ = write(fd, pid, strlen(pid));
+    close(fd);
+
+    while (!ddappsec_debugger_wait_continue) {
+        usleep(10000); // NOLINT
+    }
+    ddappsec_debugger_wait_continue = 0;
+    unlink("/tmp/pid"); // NOLINT
+
+    RETURN_TRUE;
+}
+
 static PHP_FUNCTION(datadog_appsec_testing_request_exec)
 {
     zend_array *data = NULL;
@@ -632,6 +660,7 @@ static const zend_function_entry testing_request_control_functions[] = {
     ZEND_RAW_FENTRY(DD_TESTING_NS "rinit", PHP_FN(datadog_appsec_testing_rinit), void_ret_bool_arginfo, 0, NULL, NULL)
     ZEND_RAW_FENTRY(DD_TESTING_NS "rshutdown", PHP_FN(datadog_appsec_testing_rshutdown), void_ret_bool_arginfo, 0, NULL, NULL)
     ZEND_RAW_FENTRY(DD_TESTING_NS "request_exec", PHP_FN(datadog_appsec_testing_request_exec), request_exec_arginfo, 0, NULL, NULL)
+    ZEND_RAW_FENTRY(DD_TESTING_NS "wait_for_debugger", PHP_FN(datadog_appsec_testing_wait_for_debugger), void_ret_bool_arginfo, 0, NULL, NULL)
     PHP_FE_END
 };
 static const zend_function_entry testing_functions[] = {

appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/TelemetryHandler.groovy

@@ -23,7 +23,7 @@ class TelemetryHandler implements Handler {
             ctx.bodyInputStream().withCloseable {
                 message = readTelemetryMessage(it)
             }
-            log.debug("Read telemetry message: ${message['request_type']}")
+            log.debug("Read telemetry message: ${describeTelemetryMessage(message)}")
         } catch (AssertionError e) {
             log.error("Error reading traces: $e.message")
             error = e
@@ -52,6 +52,48 @@ class TelemetryHandler implements Handler {
         jsonSlurper.parse(is)
     }
 
+    private static String describeTelemetryMessage(Object message) {
+        def application = message['application'] ?: [:]
+        def requestType = message['request_type']
+        def payload = message['payload']
+        def details = [
+                "request_type=${requestType}",
+                "seq_id=${message['seq_id']}",
+                "service=${application['service_name']}",
+                "runtime_id=${application['runtime_id']}",
+        ]
+        def payloadSummary = describeTelemetryPayload(requestType, payload)
+        if (payloadSummary) {
+            details << "payload=${payloadSummary}"
+        }
+        details.join(', ')
+    }
+
+    private static String describeTelemetryPayload(String requestType, Object payload) {
+        if (requestType == 'message-batch' && payload instanceof List) {
+            return payload.collect { describeTelemetryPayload(it['request_type'], it['payload']) }
+                    .findAll { it }
+                    .join('; ')
+        }
+
+        if (!(payload instanceof Map)) {
+            return null
+        }
+
+        def fields = []
+        if (payload['integrations'] instanceof List) {
+            fields << "integrations=${payload['integrations'].collect { it['name'] }}"
+        }
+        if (payload['dependencies'] instanceof List) {
+            fields << "dependencies=${payload['dependencies'].size()}"
+        }
+        if (payload['configuration'] instanceof List) {
+            fields << "configuration=${payload['configuration'].size()}"
+        }
+
+        return "${requestType}{${fields.join(', ')}}"
+    }
+
     List<Object> drain(long timeoutInMs) {
         synchronized (capturedTelemetryMessages) {
             if (!savedError && capturedTelemetryMessages.isEmpty()) {