diff --git a/.gitlab/compile_extension.sh b/.gitlab/compile_extension.sh index f52a043d41f..c3d38d16ba8 100755 --- a/.gitlab/compile_extension.sh +++ b/.gitlab/compile_extension.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -eo pipefail +set -xeo pipefail SWITCH_PHP_VERSION=${SWITCH_PHP_VERSION:-} WITH_ASAN=${WITH_ASAN:-} @@ -20,7 +20,7 @@ if [ "${WITH_ASAN}" -eq "1" ]; then export COMPILE_ASAN=1 fi # Compile Rust and PHP in parallel -./compile_rust.sh & +SHARED=1 ./compile_rust.sh & make -j static & wait diff --git a/.gitlab/generate-tracer.php b/.gitlab/generate-tracer.php index cb016abf388..42510ff8ce5 100644 --- a/.gitlab/generate-tracer.php +++ b/.gitlab/generate-tracer.php @@ -74,7 +74,6 @@ function before_script_steps($with_docker_auth = false) { ARCH: *arch_targets variables: host_os: linux-gnu - SHARED: "1" WITH_ASAN: "0" CARGO_HOME: "/rust/cargo/" SWITCH_PHP_VERSION: "debug" @@ -204,6 +203,7 @@ function before_script_steps($with_docker_auth = false) { DATADOG_HAVE_DEV_ENV: 1 HTTPBIN_HOSTNAME: httpbin-integration HTTPBIN_PORT: 8080 + RUST_BACKTRACE: 1 before_script: diff --git a/Cargo.lock b/Cargo.lock index 90e10780ede..00790dc7c64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1079,12 +1079,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "current_platform" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a74858bcfe44b22016cb49337d7b6f04618c58e5dbfdef61b06b8c434324a0bc" - [[package]] name = "cxx" version = "1.0.194" @@ -1361,6 +1355,7 @@ name = "datadog-sidecar" version = "0.0.1" dependencies = [ "anyhow", + "arc-swap", "arrayref", "base64 0.22.1", "bincode", @@ -2745,6 +2740,7 @@ dependencies = [ "http-body-util", "libdd-capabilities", "libdd-common", + "tokio", ] [[package]] @@ -2880,11 +2876,13 @@ dependencies = [ "libdd-capabilities", "libdd-capabilities-impl", "libdd-common", + "libdd-ddsketch", "libdd-dogstatsd-client", "libdd-log", "libdd-shared-runtime", "libdd-telemetry", "libdd-tinybytes", + "libdd-trace-obfuscation", "libdd-trace-protobuf", "libdd-trace-stats", "libdd-trace-utils", @@ -3037,11 +3035,14 @@ version = "0.1.0" dependencies = [ "async-trait", "futures", + "futures-util", "libdd-capabilities", + "libdd-capabilities-impl", "libdd-common", "tokio", "tokio-util", "tracing", + "wasm-bindgen-futures", ] [[package]] @@ -3056,6 +3057,7 @@ dependencies = [ "hashbrown 0.15.2", "http", "http-body-util", + "httpmock", "libc 0.2.177", "libdd-common", "libdd-ddsketch", @@ -3147,6 +3149,7 @@ name = "libdd-trace-stats" version = "2.0.0" dependencies = [ "anyhow", + "arc-swap", "async-trait", "criterion", "hashbrown 0.15.2", @@ -3906,8 +3909,6 @@ dependencies = [ name = "php_sidecar_mockgen" version = "0.1.0" dependencies = [ - "cc_utils", - "current_platform", "sidecar_mockgen", ] diff --git a/Cargo.toml b/Cargo.toml index 050c56c67a2..88b0eb587ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ inherits = "release" # components-rs. There may be a better way to fix this, but I'm alreday two # tasks removed from what I'm trying to do, so pushing forward. [workspace.dependencies] +arc-swap = "1.7.1" hyper = { version = "1.6", features = [ "http1", "client", diff --git a/components-rs/build.rs b/components-rs/build.rs new file mode 100644 index 00000000000..64a910dd093 --- /dev/null +++ b/components-rs/build.rs @@ -0,0 +1,8 @@ +fn main() { + // On Linux, set ddog_spawn_direct_entry as the ELF entry point for the + // cdylib build (libddtrace_php.so in SSI deployments). This allows ld.so + // to exec the library directly without a trampoline binary. + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("linux") { + println!("cargo:rustc-cdylib-link-arg=-Wl,-e,ddog_spawn_direct_entry"); + } +} diff --git a/components-rs/php_sidecar_mockgen/Cargo.toml b/components-rs/php_sidecar_mockgen/Cargo.toml index 4b010850c28..ac3a03a172c 100644 --- a/components-rs/php_sidecar_mockgen/Cargo.toml +++ b/components-rs/php_sidecar_mockgen/Cargo.toml @@ -4,8 +4,6 @@ name = "php_sidecar_mockgen" version = "0.1.0" [dependencies] -cc_utils = { path = "../../libdatadog/tools/cc_utils" } -current_platform = "0.2.0" sidecar_mockgen = { path = "../../libdatadog/tools/sidecar_mockgen" } [[bin]] diff --git a/components-rs/php_sidecar_mockgen/src/bin/php_sidecar_mockgen.rs b/components-rs/php_sidecar_mockgen/src/bin/php_sidecar_mockgen.rs index a31c034e07e..d8f3ca315b5 100644 --- a/components-rs/php_sidecar_mockgen/src/bin/php_sidecar_mockgen.rs +++ b/components-rs/php_sidecar_mockgen/src/bin/php_sidecar_mockgen.rs @@ -1,89 +1,27 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. -pub use cc_utils::cc; -pub use sidecar_mockgen::generate_mock_symbols; +pub use sidecar_mockgen::weaken_object_symbols; use std::path::Path; -use std::{env, fs, process}; +use std::{env, process}; fn main() { - // This script is necessary to avoid the linker puking when the sidecar tries to load our ddtrace.so - // As php itself is not available within the sidecar, it needs to make sure that all symbols ddtrace.so depends on, are available. - // The mock_generator takes care of generating these symbols. let args: Vec<_> = env::args_os().collect(); - if args.len() < 3 { - eprintln!( - "Needs at least 3 args: the output file, the shared object file followed by at least one object file" - ); - process::exit(1); - } - - let output_path = Path::new(&args[1]); - let binary_path = Path::new(&args[2]); - let object_paths: Vec<_> = args.iter().skip(3).map(Path::new).collect(); - let mock_symbols = match generate_mock_symbols(binary_path, object_paths.as_slice()) { - Ok(symbols) => symbols, - Err(err) => { - eprintln!("Failed generating mock_php_syms.c: {}", err); - process::exit(1); - } - }; - if fs::read("mock_php_syms.c") - .ok() - .map(|contents| contents == mock_symbols.as_str().as_bytes()) - != Some(true) - { - if let Err(err) = fs::write("mock_php_syms.c", mock_symbols) { - eprintln!("Failed generating mock_php_syms.c: {}", err); - process::exit(1); - } + if args.get(1).and_then(|a| a.to_str()) != Some("weaken-dynsym") || args.len() < 4 { + eprintln!("Usage: php_sidecar_mockgen weaken-dynsym "); + process::exit(1); } - let source_modified = fs::metadata("mock_php_syms.c").unwrap().modified().unwrap(); - if fs::metadata("mock_php.shared_lib").map_or(true, |m| m.modified().unwrap() < source_modified) { - env::set_var("OPT_LEVEL", "2"); - - let mut cc_build = cc::Build::new(); - - // The 'host' and 'target' options are required to compile. - // They can be provided using the HOST and TARGET environment variables. - // On Linux, these environment variables can be empty strings, but it's not the case on macOS. - let host = std::env::var("HOST").unwrap_or("".to_string()); - if host == "" { - cc_build.host(current_platform::CURRENT_PLATFORM); - } - let target = std::env::var("TARGET").unwrap_or("".to_string()); - if target == "" { - cc_build.target(current_platform::CURRENT_PLATFORM); - } - - cc_utils::ImprovedBuild::new() - .set_cc_builder(cc_build) - .file("mock_php_syms.c") - .link_dynamically("dl") - .warnings(true) - .warnings_into_errors(true) - .emit_rerun_if_env_changed(false) - .try_compile_shared_lib("mock_php.shared_lib") - .unwrap(); - - let bin = match fs::read("mock_php.shared_lib") { - Ok(bin) => bin, - Err(err) => { - eprintln!("Failed opening generated mock_php.shared_lib: {}", err); - process::exit(1); - } - }; - - let comma_separated = bin.iter().map(|byte| format!("{byte:#X}")).collect::>().join(","); - let out = format!(r#" - const unsigned char DDTRACE_MOCK_PHP[] = {{{comma_separated}}}; - const void *DDTRACE_MOCK_PHP_SIZE = (void *) sizeof(DDTRACE_MOCK_PHP); - "#); + let php_binary = Path::new(args.last().unwrap()); + let targets: Vec<_> = args[2..args.len() - 1] + .iter() + .map(|a| Path::new(a.as_os_str())) + .collect(); - if let Err(err) = fs::write(output_path, out) { - eprintln!("Failed generating {:?}: {}", output_path, err); + for target in targets { + if let Err(e) = weaken_object_symbols(target, php_binary) { + eprintln!("Warning: weaken-dynsym {}: {e}", target.display()); process::exit(1); } } diff --git a/components-rs/sidecar.rs b/components-rs/sidecar.rs index d7e6a687666..499a7b3e7b5 100644 --- a/components-rs/sidecar.rs +++ b/components-rs/sidecar.rs @@ -16,26 +16,15 @@ use libdd_common::Endpoint; use libdd_common_ffi::slice::AsBytes; use libdd_common_ffi::{CharSlice, self as ffi, MaybeError}; use libdd_telemetry_ffi::try_c; -#[cfg(any(windows, php_shared_build))] -use spawn_worker::LibDependency; #[cfg(windows)] -use spawn_worker::get_trampoline_target_data; - - -#[cfg(php_shared_build)] -extern "C" { - #[linkage="extern_weak"] - static DDTRACE_MOCK_PHP: *mut u8; - #[linkage="extern_weak"] - static DDTRACE_MOCK_PHP_SIZE: *mut usize; -} +use spawn_worker::{get_trampoline_target_data, LibDependency}; #[cfg(php_shared_build)] fn run_sidecar(mut cfg: config::Config) -> anyhow::Result { - if !unsafe { DDTRACE_MOCK_PHP_SIZE }.is_null() { - let mock = unsafe { std::slice::from_raw_parts(DDTRACE_MOCK_PHP, *DDTRACE_MOCK_PHP_SIZE) }; - cfg.library_dependencies - .push(LibDependency::Binary(mock)); + #[cfg(target_os = "linux")] + if std::env::var_os("DD_SIDECAR_DISABLE_DIRECT_EXEC").map(|s| s.is_empty()).unwrap_or(true) + && std::env::var_os("DD_SPAWN_WORKER_USE_EXEC").map(|s| s.is_empty()).unwrap_or(true) { + cfg.spawn_without_trampoline = true; } datadog_sidecar::start_or_connect_to_sidecar(cfg) } diff --git a/config.m4 b/config.m4 index a2de863caac..8a7ab3c1c68 100644 --- a/config.m4 +++ b/config.m4 @@ -7,7 +7,7 @@ PHP_ARG_ENABLE(ddtrace-sanitize, whether to enable AddressSanitizer for ddtrace, PHP_ARG_WITH(ddtrace-rust-library, the rust library is located; i.e. to compile without cargo, [ --with-ddtrace-rust-library Location to rust library for linking against], -, will be compiled) -PHP_ARG_WITH(ddtrace-sidecar-mockgen, binary to generate mock_php.c, +PHP_ARG_WITH(ddtrace-sidecar-mockgen, binary to weaken PHP symbols in object files, [ --with-ddtrace-sidecar-library Location to cargo binary produced by components-rs/php_sidecar_mockgen], -, will be compiled) PHP_ARG_WITH(ddtrace-cargo, where cargo is located for rust code compilation, @@ -281,6 +281,14 @@ if test "$PHP_DDTRACE" != "no"; then EXTRA_CFLAGS="$EXTRA_CFLAGS -fvisibility=hidden" EXTRA_LDFLAGS="$EXTRA_LDFLAGS -export-symbols $ext_srcdir/ddtrace.sym -flto -fuse-linker-plugin" + dnl On Linux: set the ELF entry point so ddtrace.so can be exec'd directly by ld.so + dnl for sidecar spawning (no trampoline binary, no memfd, no temp files). + case $host_os in + linux*) + EXTRA_LDFLAGS="$EXTRA_LDFLAGS -Wl,-e,ddog_spawn_direct_entry" + ;; + esac + PHP_SUBST(EXTRA_CFLAGS) PHP_SUBST(EXTRA_LDFLAGS) PHP_SUBST(DDTRACE_SHARED_LIBADD) @@ -365,6 +373,17 @@ EOT $(LIBTOOL) --mode=link $(CC) -static $(COMMON_FLAGS) $(CFLAGS_CLEAN) $(EXTRA_CFLAGS) $(LDFLAGS) -o $@ -avoid-version -prefer-pic -module $(shared_objects_ddtrace) EOT + if test "$ext_shared" = "yes"; then + all_object_files=$(for src in $DD_TRACE_PHP_SOURCES $ZAI_SOURCES; do printf ' %s' "${src%?}lo"; done) + all_object_files_absolute=$(for src in $DD_TRACE_PHP_SOURCES $ZAI_SOURCES; do printf ' $(builddir)/%s' "$(dirname "$src")/$objdir/$(basename "${src%?}o")"; done) + php_binary=$("$PHP_CONFIG" --php-binary) + if test "$PHP_DDTRACE_SIDECAR_MOCKGEN" != "-"; then + ddtrace_mockgen_invocation="HOST= TARGET= $PHP_DDTRACE_SIDECAR_MOCKGEN" + else + ddtrace_mockgen_invocation="cd \"$ext_srcdir/components-rs/php_sidecar_mockgen\"; HOST= TARGET= CARGO_TARGET_DIR=\$(builddir)/target_mockgen/ \$(DDTRACE_CARGO) run" + fi + fi + if test "$PHP_DDTRACE_RUST_LIBRARY_SPLIT" != "no"; then ddtrace_rust_lib="" elif test "$PHP_DDTRACE_RUST_LIBRARY" != "-"; then @@ -380,25 +399,17 @@ $ddtrace_rust_lib: $( (find "$ext_srcdir/components-rs" -name "*.rs" -o -name "C EOT fi + dnl Weaken PHP-origin symbols in all .o files before the link step so that + dnl the resulting .so/.a naturally has weak references. if test "$ext_shared" = "yes"; then - all_object_files=$(for src in $DD_TRACE_PHP_SOURCES $ZAI_SOURCES; do printf ' %s' "${src%?}lo"; done) - all_object_files_absolute=$(for src in $DD_TRACE_PHP_SOURCES $ZAI_SOURCES; do printf ' $(builddir)/%s' "$(dirname "$src")/$objdir/$(basename "${src%?}o")"; done) - php_binary=$("$PHP_CONFIG" --php-binary) - if test "$PHP_DDTRACE_SIDECAR_MOCKGEN" != "-"; then - ddtrace_mockgen_invocation="HOST= TARGET= $PHP_DDTRACE_SIDECAR_MOCKGEN" - else - ddtrace_mockgen_invocation="cd \"$ext_srcdir/components-rs/php_sidecar_mockgen\"; HOST= TARGET= CARGO_TARGET_DIR=\$(builddir)/target_mockgen/ \$(DDTRACE_CARGO) run" - fi - cat <> Makefile.fragments - -/\$(builddir)/components-rs/mock_php.c: $all_object_files - ($ddtrace_mockgen_invocation \$(builddir)/components-rs/mock_php.c $php_binary $all_object_files_absolute) - -# avoid cargo running simultaneously for libddtrace_php and php_sidecar_mockgen -/\$(builddir)/components-rs/mock_php.c: | \$(filter-out \$(builddir)/components-rs/mock_php.lo,\$(shared_objects_ddtrace)) -EOT - - PHP_ADD_SOURCES_X("/$ext_dir", "\$(builddir)/components-rs/mock_php.c", $ac_extra, shared_objects_ddtrace, yes) + _ddtrace_weaken_tmp=$(mktemp) + cat > "$_ddtrace_weaken_tmp" << WEAKEN + ($ddtrace_mockgen_invocation weaken-dynsym $all_object_files_absolute $php_binary) +WEAKEN + sed -i $({ sed --version 2>&1 || echo ''; } | grep GNU >/dev/null || echo "''") -e "/\/ddtrace\.la:\ \\$/r $_ddtrace_weaken_tmp" Makefile.objects + dnl run weaken only once, create a dependency on .la for .a + echo "./modules/ddtrace.a: | ./ddtrace.la" >> Makefile.fragments + rm -f "$_ddtrace_weaken_tmp" fi if test "$ext_shared" = "shared" || test "$ext_shared" = "yes"; then diff --git a/docker-compose.yml b/docker-compose.yml index a77598f7ed3..1ae0b5f428b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,7 +40,6 @@ x-aliases: - PHP_IDE_CONFIG=serverName=docker - DD_TRACE_DOCKER_DEBUG - DATADOG_HAVE_DEV_ENV=1 - - DD_SPAWN_WORKER_USE_EXEC=1 cap_add: - SYS_PTRACE security_opt: diff --git a/libdatadog b/libdatadog index 91fd13c8a0c..c33c315448c 160000 --- a/libdatadog +++ b/libdatadog @@ -1 +1 @@ -Subproject commit 91fd13c8a0ca5335fe39940f8764cd825bbef7e8 +Subproject commit c33c315448ccbcf4120e9997002a329fbaa2d584 diff --git a/loader/tests/functional/test_configuration_telemetry.php b/loader/tests/functional/test_configuration_telemetry.php index 6a15c892a45..565e667a9cb 100644 --- a/loader/tests/functional/test_configuration_telemetry.php +++ b/loader/tests/functional/test_configuration_telemetry.php @@ -16,7 +16,7 @@ assertMatchesFormat($output, '%A"loaded_by_ssi":true%s%A'); // Let time to write the telemetry log -usleep(10000); +usleep(300000); $content = file_get_contents($telemetryLogPath); assertContains($content, '{"name":"instrumentation_source","value":"ssi","origin":"default","config_id":null,"seq_id":null}'); diff --git a/tests/ext/telemetry/integration.phpt b/tests/ext/telemetry/integration.phpt index 3405c58b4c5..921f2c57229 100644 --- a/tests/ext/telemetry/integration.phpt +++ b/tests/ext/telemetry/integration.phpt @@ -52,6 +52,10 @@ namespace $json = json_decode($l, true); $batch = $json["request_type"] == "message-batch" ? $json["payload"] : [$json]; foreach ($batch as $json) { + if ($json["request_type"] == "app-started" && !empty($json["payload"]["integrations"])) { + var_dump(["integrations" => $json["payload"]]); + break 3; + } if ($json["request_type"] == "app-integrations-change") { var_dump($json["payload"]); break 3;