chaos: vthread-context-cascade antagonist + glibc heap-corruption defaults

.gitlab/reliability/chaos_check.sh

@@ -1,173 +1,17 @@
 #!/usr/bin/env bash
-
-set +e # Disable exit on error
+#
+# CI entry point for the chaos harness. Thin wrapper around
+# utils/run-chaos-harness.sh — this file only supplies the CI-specific
+# caching paths (so repeated scheduled runs on the same runner reuse the
+# downloaded JDK/agent jar) and the fixed hs_err.log location the pipeline's
+# `artifacts:` block expects at the repo root. All the actual build/run logic
+# lives in the standalone script, which is also runnable by hand.
 
 HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT="$( cd "${HERE}/../.." >/dev/null 2>&1 && pwd )"
 
-RUNTIME=${1}
-CONFIG=${2:-profiler+tracer}
-ALLOCATOR=${3:-gmalloc}
-
-echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}"
-
-CHAOS_JDK="${CHAOS_JDK:-21.0.3-tem}"
-# CHAOS_JDK uses sdkman notation (<version>-<dist>); extract major for Adoptium API.
-JDK_MAJOR="${CHAOS_JDK%%.*}"
-JDK_ARCH=$(uname -m | sed 's/x86_64/x64/')
-JDK_INSTALL_DIR="/opt/jdk-${CHAOS_JDK}"
-
-if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then
-  TMP=$(mktemp -d)
-  DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse"
-  echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..."
-  if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then
-    echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2
-    rm -rf "${TMP}"
-    exit 1
-  fi
-  mkdir -p "${JDK_INSTALL_DIR}"
-  tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1
-  rm -rf "${TMP}"
-fi
-
-if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then
-  echo "FAIL:JDK ${CHAOS_JDK} not available after install" >&2
-  exit 1
-fi
-export JAVA_HOME="${JDK_INSTALL_DIR}"
-export PATH="${JAVA_HOME}/bin:${PATH}"
-ACTIVE_JDK=$(java -version 2>&1 | head -1)
-# Check major version only — patch may differ from the Adoptium latest GA.
-if ! echo "${ACTIVE_JDK}" | grep -qE "\"${JDK_MAJOR}\."; then
-  echo "FAIL:wrong JDK active (expected major ${JDK_MAJOR}, got: ${ACTIVE_JDK})" >&2
-  exit 1
-fi
-
-# Resolve ddprof.jar: prefer local build artifact, fall back to Maven snapshot.
-# Running mvn from /tmp avoids the empty pom.xml at the repo root.
-DDPROF_JAR_LOCAL=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1)
-if [ -n "${DDPROF_JAR_LOCAL}" ] && [ -f "${DDPROF_JAR_LOCAL}" ]; then
-  DDPROF_JAR="${DDPROF_JAR_LOCAL}"
-  echo "Using local ddprof jar: ${DDPROF_JAR}"
-else
-  if [ -z "${CURRENT_VERSION:-}" ]; then
-    echo "FAIL:CURRENT_VERSION is empty and no local jar found (get-versions dotenv missing)" >&2
-    exit 1
-  fi
-  echo "Local ddprof jar not found — downloading ${CURRENT_VERSION} from Maven snapshots"
-  (cd /tmp && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \
-      -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \
-      -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION})
-  DDPROF_JAR="/root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar"
-fi
-
-if [ ! -f "${DDPROF_JAR}" ]; then
-  echo "FAIL:ddprof jar unavailable" >&2
-  exit 1
-fi
-
-mkdir -p /var/lib/datadog
-wget -q --timeout=120 --tries=3 -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer'
-
-# chaos.jar is produced once per pipeline by the chaos:build job (stresstest
-# stage) and pulled here as an artifact. Fall back to an inline build if the
-# artifact is absent (e.g. local repro outside CI).
-CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar"
-if [ ! -f "${CHAOS_JAR}" ]; then
-  echo "chaos.jar artifact not present — building inline"
-  ( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q )
-fi
-
-if [ ! -f "${CHAOS_JAR}" ]; then
-  echo "FAIL:chaos.jar unavailable" >&2
-  exit 1
-fi
-
-# Patch dd-java-agent.jar with the locally built ddprof contents so the agent's
-# (relocated) profiler classes match the version under test.
-DD_AGENT_JAR=/var/lib/datadog/dd-java-agent.jar \
-DDPROF_JAR=${DDPROF_JAR} \
-OUTPUT_JAR=/var/lib/datadog/dd-java-agent-patched.jar \
-"${ROOT}/utils/patch-dd-java-agent.sh"
-
-if [ ! -f /var/lib/datadog/dd-java-agent-patched.jar ]; then
-  echo "FAIL:dd-java-agent patching failed" >&2
-  exit 1
-fi
-
-PATCHED_AGENT=/var/lib/datadog/dd-java-agent-patched.jar
-
-case $CONFIG in
-  profiler)
-    echo "Running with profiler only"
-    ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false"
-    # @Trace is a no-op without the tracer, so trace-context is excluded here.
-    ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm"
-    ;;
-  profiler+tracer)
-    echo "Running with profiler and tracer"
-    ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true"
-    ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm"
-    ;;
-  *)
-    echo "Unknown configuration: $CONFIG"
-    exit 1
-    ;;
-esac
-
-case $ALLOCATOR in
-  gmalloc)
-    echo "Running with gmalloc"
-    ;;
-  tcmalloc)
-    echo "Running with tcmalloc"
-    export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4')
-    ;;
-  jemalloc)
-    echo "Running with jemalloc"
-    export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so')
-    ;;
-  *)
-    echo "Unknown allocator: $ALLOCATOR"
-    echo "Valid values are: gmalloc, tcmalloc, jemalloc"
-    exit 1
-    ;;
-esac
-
-echo "LD_PRELOAD=$LD_PRELOAD"
-
-timeout "$((RUNTIME + 300))" \
-java -javaagent:${PATCHED_AGENT} \
-     ${ENABLEMENT} \
-     -Ddd.profiling.upload.period=10 \
-     -Ddd.profiling.start-force-first=true \
-     -Ddd.profiling.ddprof.liveheap.enabled=true \
-     -Ddd.profiling.ddprof.alloc.enabled=true \
-     -Ddd.profiling.ddprof.wall.enabled=true \
-     -Ddd.profiling.ddprof.nativemem.enabled=true \
-     -Ddd.env=java-profiler-stability \
-     -Ddd.service=java-profiler-chaos \
-     -Xmx2g -Xms2g \
-     -XX:MaxMetaspaceSize=384m \
-     -XX:ErrorFile=${HERE}/../../hs_err.log \
-     -XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \
-     -jar ${CHAOS_JAR} \
-     --duration ${RUNTIME}s \
-     --antagonists ${ANTAGONISTS}
-
-RC=$?
-echo "RC=$RC"
+export CHAOS_JDK_DIR="${CHAOS_JDK_DIR:-/opt/jdk-${CHAOS_JDK}}"
+export CHAOS_WORK_DIR="${CHAOS_WORK_DIR:-/var/lib/datadog}"
+export CHAOS_ERROR_FILE="${ROOT}/hs_err.log"
 
-if [ $RC -ne 0 ]; then
-  CRASH_MSG="Chaos harness crashed (RC=${RC})"
-  HS_ERR="${HERE}/../../hs_err.log"
-  if [ -f "${HS_ERR}" ]; then
-    SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120)
-    FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120)
-    [ -n "${SIG}" ]   && CRASH_MSG="${CRASH_MSG};${SIG}"
-    [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}"
-  fi
-  echo "FAIL:${CRASH_MSG}" >&2
-  exit 1
-fi
+exec "${ROOT}/utils/run-chaos-harness.sh" "$@"

build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt

@@ -3,6 +3,7 @@ package com.datadoghq.profiler
 
 import com.datadoghq.native.NativeBuildExtension
 import com.datadoghq.native.model.BuildConfiguration
+import com.datadoghq.native.model.Platform
 import com.datadoghq.native.util.PlatformUtils
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
@@ -204,6 +205,21 @@ class ProfilerTestPlugin : Plugin<Project> {
 
         // Environment variables (explicit for consistency across both paths)
         val envVars = buildMap<String, String> {
+            // Turn silent glibc heap corruption (e.g. a use-after-free write into a
+            // freed chunk) into an immediate, attributable SIGABRT instead of a crash
+            // much later in an unrelated allocation. Only meaningful against glibc's
+            // own malloc: skip on musl (no MALLOC_CHECK_ support), skip whenever a
+            // sanitizer/allocator already replaces malloc via LD_PRELOAD, and skip on
+            // J9 (its own allocator usage trips a pre-existing, unrelated heap
+            // corruption that needs separate investigation — see PROF-15360).
+            if (PlatformUtils.currentPlatform == Platform.LINUX &&
+                !PlatformUtils.isMusl() &&
+                !testEnv.containsKey("LD_PRELOAD") &&
+                !PlatformUtils.isTestJvmJ9()
+            ) {
+                put("MALLOC_CHECK_", "3")
+                put("MALLOC_PERTURB_", (1..255).random().toString())
+            }
             putAll(testEnv)
             put("DDPROF_TEST_DISABLE_RATE_LIMIT", "1")
             put("CI", (project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false).toString())

ddprof-lib/src/main/cpp/frame.h

@@ -79,8 +79,12 @@ class FrameType {
     return (FrameTypeId)((bci >> TYPE_SHIFT) & FRAME_TYPE_MASK);
   }
 
+  // RAW_POINTER_MASK is only ever set by encode() (rawPointer=true), which
+  // unconditionally also sets ENCODED_MASK. Requiring ENCODED_MASK here
+  // prevents a raw, unencoded ASGCT BCI (never passed through encode())
+  // from false-positiving just because it happens to have bit 30 set.
   static inline bool isRawPointer(int bci) {
-    return bci > 0 && (bci & RAW_POINTER_MASK) != 0;
+    return bci > 0 && (bci & ENCODED_MASK) != 0 && (bci & RAW_POINTER_MASK) != 0;
   }
 };
 

ddprof-lib/src/main/cpp/threadLocalData.cpp

@@ -11,9 +11,125 @@
 #include <cstring>
 #include <time.h>
 
+#ifdef DEBUG
+#include <atomic>
+#include <chrono>
+#include <cstdio>
+#include <thread>
+#include <unistd.h>
+#endif
+
 pthread_key_t ProfiledThread::_tls_key;
 bool ProfiledThread::_tls_key_initialized = false;
 
+#ifdef DEBUG
+// Temporary chaos-testing diagnostic (jb/vt_churn): proves a stale-carrier UAF write actually
+// lands on freed ProfiledThread memory, without relying on glibc's malloc-metadata checks
+// (MALLOC_CHECK_/tcache safe-linking), which only catch corruption at chunk boundaries — the
+// OTel record header a Java-side stale write targets sits well inside the chunk, not at its
+// start. Snapshots the header right after free (MALLOC_PERTURB_ has already scribbled it if the
+// gmalloc harness is active), then re-reads the same dangling address after a delay exceeding the
+// stale-racer's max sleep window; a mismatch proves *something* wrote to memory this thread no
+// longer owns. Reading freed memory is deliberately undefined behavior — this exists only to get
+// a signal during the vthread-context-cascade UAF investigation and must not ship.
+//
+// Uses a fixed-size slot pool + one shared watcher thread instead of a thread per free: at this
+// antagonist's churn rate (hundreds of frees/sec), thread-per-free piles up thousands of
+// detached, sleeping threads and gets the harness process OOM-killed (observed as RC=137) —
+// an artifact of the diagnostic, not the bug under investigation. A dropped sample (no free
+// slot) just means fewer data points, not a wrong result.
+namespace {
+constexpr int kWatchSlotCount = 64;
+constexpr long kWatchDelayMillis = 900; // past FAST_STALE_RACER_SLEEP_MAX_MILLIS (800ms)
+enum SlotState : int { kFree = 0, kFilling = 1, kReady = 2 };
+
+struct WatchSlot {
+  std::atomic<int> state{kFree};
+  const void *recordAddr;
+  int tid;
+  uint8_t snapshot[OTEL_HEADER_SIZE];
+  std::chrono::steady_clock::time_point dueAt;
+};
+
+WatchSlot g_watchSlots[kWatchSlotCount];
+std::atomic<bool> g_watcherStarted{false};
+
+// A byte pattern that could plausibly be a live, *activated* OtelThreadContextRecord (see
+// otel_context.h). Requires valid==1 specifically rather than valid∈{0,1}: valid==0 is also the
+// record's zero-initialized default, so it's satisfied by plain uninitialized/zeroed memory and
+// proves nothing. valid==1 only happens once Java has actually written a real trace/span
+// activation into the record. _reserved must be 0 per the OTEP spec, which coincidental
+// allocator garbage (tcache/fastbin linkage, an unrelated object landing on the same address)
+// only matches by chance. Combined with attrs_data_size being in range, this cuts the chance of
+// a random-byte false positive to roughly 1 in 1.8e7 — reused-but-unrelated memory essentially
+// never passes all three checks together.
+bool looksLikeOtelRecord(const uint8_t *header) {
+  uint8_t valid = header[16 + 8];
+  uint8_t reserved = header[16 + 8 + 1];
+  uint16_t attrsSize;
+  memcpy(&attrsSize, header + 16 + 8 + 2, sizeof(attrsSize));
+  return valid == 1 && reserved == 0 && attrsSize <= OTEL_MAX_ATTRS_DATA_SIZE;
+}
+
+void watcherLoop() {
+  // Lazily spawned from inside freeKey()/release() while their SignalBlocker is still in scope,
+  // so this thread inherits SIGPROF/SIGVTALRM blocked from its creator. Unblock them here so a
+  // permanently-running debug thread doesn't carry that mask for the rest of the process.
+  sigset_t prof_signals;
+  sigemptyset(&prof_signals);
+  sigaddset(&prof_signals, SIGPROF);
+  sigaddset(&prof_signals, SIGVTALRM);
+  pthread_sigmask(SIG_UNBLOCK, &prof_signals, nullptr);
+  for (;;) {
+    usleep(50 * 1000);
+    auto now = std::chrono::steady_clock::now();
+    for (WatchSlot &slot : g_watchSlots) {
+      if (slot.state.load(std::memory_order_acquire) != kReady) {
+        continue;
+      }
+      if (now < slot.dueAt) {
+        continue;
+      }
+      uint8_t after[OTEL_HEADER_SIZE];
+      memcpy(after, slot.recordAddr, OTEL_HEADER_SIZE);
+      bool changed = memcmp(slot.snapshot, after, OTEL_HEADER_SIZE) != 0;
+      if (changed && looksLikeOtelRecord(after)) {
+        fprintf(stderr,
+                "[ddprof-debug] UAF WRITE DETECTED: freed ProfiledThread tid=%d otel-header "
+                "changed after free and looks like a live OTel record (addr=%p)\n",
+                slot.tid, slot.recordAddr);
+        fprintf(stderr, "[ddprof-debug]   before:");
+        for (uint8_t b : slot.snapshot) fprintf(stderr, " %02x", b);
+        fprintf(stderr, "\n[ddprof-debug]   after: ");
+        for (uint8_t b : after) fprintf(stderr, " %02x", b);
+        fprintf(stderr, "\n");
+      }
+      slot.state.store(kFree, std::memory_order_release);
+    }
+  }
+}
+} // namespace
+
+static void watchFreedContextMemory(const void *recordAddr, int tid) {
+  if (!g_watcherStarted.exchange(true, std::memory_order_acq_rel)) {
+    std::thread(watcherLoop).detach();
+  }
+  for (WatchSlot &slot : g_watchSlots) {
+    int expected = kFree;
+    if (slot.state.compare_exchange_strong(expected, kFilling, std::memory_order_acq_rel)) {
+      slot.recordAddr = recordAddr;
+      slot.tid = tid;
+      memcpy(slot.snapshot, recordAddr, OTEL_HEADER_SIZE);
+      slot.dueAt = std::chrono::steady_clock::now() + std::chrono::milliseconds(kWatchDelayMillis);
+      slot.state.store(kReady, std::memory_order_release);
+      return;
+    }
+  }
+  // No free slot: drop this sample rather than blocking the freeing thread or spawning
+  // another thread.
+}
+#endif
+
 void ProfiledThread::initTLSKey() {
   static pthread_once_t tls_initialized = PTHREAD_ONCE_INIT;
   pthread_once(&tls_initialized, doInitTLSKey);
@@ -32,7 +148,14 @@ inline void ProfiledThread::freeKey(void *key) {
   ProfiledThread *tls_ref = (ProfiledThread *)(key);
   if (tls_ref != NULL) {
     SignalBlocker blocker;
+#ifdef DEBUG
+    void *recordAddr = (void *)tls_ref->getOtelContextRecord();
+    int tid = tls_ref->tid();
+#endif
     delete tls_ref;
+#ifdef DEBUG
+    watchFreedContextMemory(recordAddr, tid);
+#endif
   }
 }
 
@@ -60,7 +183,14 @@ void ProfiledThread::release() {
   if (tls != NULL) {
     SignalBlocker blocker;
     pthread_setspecific(key, NULL);
+#ifdef DEBUG
+    void *recordAddr = (void *)tls->getOtelContextRecord();
+    int tid = tls->tid();
+#endif
     delete tls;
+#ifdef DEBUG
+    watchFreedContextMemory(recordAddr, tid);
+#endif
   }
 }