diff --git a/.gitlab/reliability/chaos_check.sh b/.gitlab/reliability/chaos_check.sh index 844a8522b5..6ac5c6605e 100755 --- a/.gitlab/reliability/chaos_check.sh +++ b/.gitlab/reliability/chaos_check.sh @@ -1,173 +1,17 @@ #!/usr/bin/env bash - -set +e # Disable exit on error +# +# CI entry point for the chaos harness. Thin wrapper around +# utils/run-chaos-harness.sh — this file only supplies the CI-specific +# caching paths (so repeated scheduled runs on the same runner reuse the +# downloaded JDK/agent jar) and the fixed hs_err.log location the pipeline's +# `artifacts:` block expects at the repo root. All the actual build/run logic +# lives in the standalone script, which is also runnable by hand. HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" ROOT="$( cd "${HERE}/../.." >/dev/null 2>&1 && pwd )" -RUNTIME=${1} -CONFIG=${2:-profiler+tracer} -ALLOCATOR=${3:-gmalloc} - -echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}" - -CHAOS_JDK="${CHAOS_JDK:-21.0.3-tem}" -# CHAOS_JDK uses sdkman notation (-); extract major for Adoptium API. -JDK_MAJOR="${CHAOS_JDK%%.*}" -JDK_ARCH=$(uname -m | sed 's/x86_64/x64/') -JDK_INSTALL_DIR="/opt/jdk-${CHAOS_JDK}" - -if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then - TMP=$(mktemp -d) - DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse" - echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..." - if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then - echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2 - rm -rf "${TMP}" - exit 1 - fi - mkdir -p "${JDK_INSTALL_DIR}" - tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1 - rm -rf "${TMP}" -fi - -if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then - echo "FAIL:JDK ${CHAOS_JDK} not available after install" >&2 - exit 1 -fi -export JAVA_HOME="${JDK_INSTALL_DIR}" -export PATH="${JAVA_HOME}/bin:${PATH}" -ACTIVE_JDK=$(java -version 2>&1 | head -1) -# Check major version only — patch may differ from the Adoptium latest GA. -if ! echo "${ACTIVE_JDK}" | grep -qE "\"${JDK_MAJOR}\."; then - echo "FAIL:wrong JDK active (expected major ${JDK_MAJOR}, got: ${ACTIVE_JDK})" >&2 - exit 1 -fi - -# Resolve ddprof.jar: prefer local build artifact, fall back to Maven snapshot. -# Running mvn from /tmp avoids the empty pom.xml at the repo root. -DDPROF_JAR_LOCAL=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1) -if [ -n "${DDPROF_JAR_LOCAL}" ] && [ -f "${DDPROF_JAR_LOCAL}" ]; then - DDPROF_JAR="${DDPROF_JAR_LOCAL}" - echo "Using local ddprof jar: ${DDPROF_JAR}" -else - if [ -z "${CURRENT_VERSION:-}" ]; then - echo "FAIL:CURRENT_VERSION is empty and no local jar found (get-versions dotenv missing)" >&2 - exit 1 - fi - echo "Local ddprof jar not found — downloading ${CURRENT_VERSION} from Maven snapshots" - (cd /tmp && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ - -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \ - -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION}) - DDPROF_JAR="/root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar" -fi - -if [ ! -f "${DDPROF_JAR}" ]; then - echo "FAIL:ddprof jar unavailable" >&2 - exit 1 -fi - -mkdir -p /var/lib/datadog -wget -q --timeout=120 --tries=3 -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer' - -# chaos.jar is produced once per pipeline by the chaos:build job (stresstest -# stage) and pulled here as an artifact. Fall back to an inline build if the -# artifact is absent (e.g. local repro outside CI). -CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar" -if [ ! -f "${CHAOS_JAR}" ]; then - echo "chaos.jar artifact not present — building inline" - ( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q ) -fi - -if [ ! -f "${CHAOS_JAR}" ]; then - echo "FAIL:chaos.jar unavailable" >&2 - exit 1 -fi - -# Patch dd-java-agent.jar with the locally built ddprof contents so the agent's -# (relocated) profiler classes match the version under test. -DD_AGENT_JAR=/var/lib/datadog/dd-java-agent.jar \ -DDPROF_JAR=${DDPROF_JAR} \ -OUTPUT_JAR=/var/lib/datadog/dd-java-agent-patched.jar \ -"${ROOT}/utils/patch-dd-java-agent.sh" - -if [ ! -f /var/lib/datadog/dd-java-agent-patched.jar ]; then - echo "FAIL:dd-java-agent patching failed" >&2 - exit 1 -fi - -PATCHED_AGENT=/var/lib/datadog/dd-java-agent-patched.jar - -case $CONFIG in - profiler) - echo "Running with profiler only" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" - # @Trace is a no-op without the tracer, so trace-context is excluded here. - ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" - ;; - profiler+tracer) - echo "Running with profiler and tracer" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" - ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" - ;; - *) - echo "Unknown configuration: $CONFIG" - exit 1 - ;; -esac - -case $ALLOCATOR in - gmalloc) - echo "Running with gmalloc" - ;; - tcmalloc) - echo "Running with tcmalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4') - ;; - jemalloc) - echo "Running with jemalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so') - ;; - *) - echo "Unknown allocator: $ALLOCATOR" - echo "Valid values are: gmalloc, tcmalloc, jemalloc" - exit 1 - ;; -esac - -echo "LD_PRELOAD=$LD_PRELOAD" - -timeout "$((RUNTIME + 300))" \ -java -javaagent:${PATCHED_AGENT} \ - ${ENABLEMENT} \ - -Ddd.profiling.upload.period=10 \ - -Ddd.profiling.start-force-first=true \ - -Ddd.profiling.ddprof.liveheap.enabled=true \ - -Ddd.profiling.ddprof.alloc.enabled=true \ - -Ddd.profiling.ddprof.wall.enabled=true \ - -Ddd.profiling.ddprof.nativemem.enabled=true \ - -Ddd.env=java-profiler-stability \ - -Ddd.service=java-profiler-chaos \ - -Xmx2g -Xms2g \ - -XX:MaxMetaspaceSize=384m \ - -XX:ErrorFile=${HERE}/../../hs_err.log \ - -XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \ - -jar ${CHAOS_JAR} \ - --duration ${RUNTIME}s \ - --antagonists ${ANTAGONISTS} - -RC=$? -echo "RC=$RC" +export CHAOS_JDK_DIR="${CHAOS_JDK_DIR:-/opt/jdk-${CHAOS_JDK}}" +export CHAOS_WORK_DIR="${CHAOS_WORK_DIR:-/var/lib/datadog}" +export CHAOS_ERROR_FILE="${ROOT}/hs_err.log" -if [ $RC -ne 0 ]; then - CRASH_MSG="Chaos harness crashed (RC=${RC})" - HS_ERR="${HERE}/../../hs_err.log" - if [ -f "${HS_ERR}" ]; then - SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120) - FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120) - [ -n "${SIG}" ] && CRASH_MSG="${CRASH_MSG};${SIG}" - [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}" - fi - echo "FAIL:${CRASH_MSG}" >&2 - exit 1 -fi +exec "${ROOT}/utils/run-chaos-harness.sh" "$@" diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt index 0aac0a91ae..73782379c7 100644 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt @@ -3,6 +3,7 @@ package com.datadoghq.profiler import com.datadoghq.native.NativeBuildExtension import com.datadoghq.native.model.BuildConfiguration +import com.datadoghq.native.model.Platform import com.datadoghq.native.util.PlatformUtils import org.gradle.api.DefaultTask import org.gradle.api.GradleException @@ -204,6 +205,21 @@ class ProfilerTestPlugin : Plugin { // Environment variables (explicit for consistency across both paths) val envVars = buildMap { + // Turn silent glibc heap corruption (e.g. a use-after-free write into a + // freed chunk) into an immediate, attributable SIGABRT instead of a crash + // much later in an unrelated allocation. Only meaningful against glibc's + // own malloc: skip on musl (no MALLOC_CHECK_ support), skip whenever a + // sanitizer/allocator already replaces malloc via LD_PRELOAD, and skip on + // J9 (its own allocator usage trips a pre-existing, unrelated heap + // corruption that needs separate investigation — see PROF-15360). + if (PlatformUtils.currentPlatform == Platform.LINUX && + !PlatformUtils.isMusl() && + !testEnv.containsKey("LD_PRELOAD") && + !PlatformUtils.isTestJvmJ9() + ) { + put("MALLOC_CHECK_", "3") + put("MALLOC_PERTURB_", (1..255).random().toString()) + } putAll(testEnv) put("DDPROF_TEST_DISABLE_RATE_LIMIT", "1") put("CI", (project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false).toString()) diff --git a/ddprof-lib/src/main/cpp/frame.h b/ddprof-lib/src/main/cpp/frame.h index 574ca41a11..c1c30633c0 100644 --- a/ddprof-lib/src/main/cpp/frame.h +++ b/ddprof-lib/src/main/cpp/frame.h @@ -79,8 +79,12 @@ class FrameType { return (FrameTypeId)((bci >> TYPE_SHIFT) & FRAME_TYPE_MASK); } + // RAW_POINTER_MASK is only ever set by encode() (rawPointer=true), which + // unconditionally also sets ENCODED_MASK. Requiring ENCODED_MASK here + // prevents a raw, unencoded ASGCT BCI (never passed through encode()) + // from false-positiving just because it happens to have bit 30 set. static inline bool isRawPointer(int bci) { - return bci > 0 && (bci & RAW_POINTER_MASK) != 0; + return bci > 0 && (bci & ENCODED_MASK) != 0 && (bci & RAW_POINTER_MASK) != 0; } }; diff --git a/ddprof-lib/src/main/cpp/threadLocalData.cpp b/ddprof-lib/src/main/cpp/threadLocalData.cpp index 1843514bd2..7e5ebaae2f 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalData.cpp @@ -11,9 +11,125 @@ #include #include +#ifdef DEBUG +#include +#include +#include +#include +#include +#endif + pthread_key_t ProfiledThread::_tls_key; bool ProfiledThread::_tls_key_initialized = false; +#ifdef DEBUG +// Temporary chaos-testing diagnostic (jb/vt_churn): proves a stale-carrier UAF write actually +// lands on freed ProfiledThread memory, without relying on glibc's malloc-metadata checks +// (MALLOC_CHECK_/tcache safe-linking), which only catch corruption at chunk boundaries — the +// OTel record header a Java-side stale write targets sits well inside the chunk, not at its +// start. Snapshots the header right after free (MALLOC_PERTURB_ has already scribbled it if the +// gmalloc harness is active), then re-reads the same dangling address after a delay exceeding the +// stale-racer's max sleep window; a mismatch proves *something* wrote to memory this thread no +// longer owns. Reading freed memory is deliberately undefined behavior — this exists only to get +// a signal during the vthread-context-cascade UAF investigation and must not ship. +// +// Uses a fixed-size slot pool + one shared watcher thread instead of a thread per free: at this +// antagonist's churn rate (hundreds of frees/sec), thread-per-free piles up thousands of +// detached, sleeping threads and gets the harness process OOM-killed (observed as RC=137) — +// an artifact of the diagnostic, not the bug under investigation. A dropped sample (no free +// slot) just means fewer data points, not a wrong result. +namespace { +constexpr int kWatchSlotCount = 64; +constexpr long kWatchDelayMillis = 900; // past FAST_STALE_RACER_SLEEP_MAX_MILLIS (800ms) +enum SlotState : int { kFree = 0, kFilling = 1, kReady = 2 }; + +struct WatchSlot { + std::atomic state{kFree}; + const void *recordAddr; + int tid; + uint8_t snapshot[OTEL_HEADER_SIZE]; + std::chrono::steady_clock::time_point dueAt; +}; + +WatchSlot g_watchSlots[kWatchSlotCount]; +std::atomic g_watcherStarted{false}; + +// A byte pattern that could plausibly be a live, *activated* OtelThreadContextRecord (see +// otel_context.h). Requires valid==1 specifically rather than valid∈{0,1}: valid==0 is also the +// record's zero-initialized default, so it's satisfied by plain uninitialized/zeroed memory and +// proves nothing. valid==1 only happens once Java has actually written a real trace/span +// activation into the record. _reserved must be 0 per the OTEP spec, which coincidental +// allocator garbage (tcache/fastbin linkage, an unrelated object landing on the same address) +// only matches by chance. Combined with attrs_data_size being in range, this cuts the chance of +// a random-byte false positive to roughly 1 in 1.8e7 — reused-but-unrelated memory essentially +// never passes all three checks together. +bool looksLikeOtelRecord(const uint8_t *header) { + uint8_t valid = header[16 + 8]; + uint8_t reserved = header[16 + 8 + 1]; + uint16_t attrsSize; + memcpy(&attrsSize, header + 16 + 8 + 2, sizeof(attrsSize)); + return valid == 1 && reserved == 0 && attrsSize <= OTEL_MAX_ATTRS_DATA_SIZE; +} + +void watcherLoop() { + // Lazily spawned from inside freeKey()/release() while their SignalBlocker is still in scope, + // so this thread inherits SIGPROF/SIGVTALRM blocked from its creator. Unblock them here so a + // permanently-running debug thread doesn't carry that mask for the rest of the process. + sigset_t prof_signals; + sigemptyset(&prof_signals); + sigaddset(&prof_signals, SIGPROF); + sigaddset(&prof_signals, SIGVTALRM); + pthread_sigmask(SIG_UNBLOCK, &prof_signals, nullptr); + for (;;) { + usleep(50 * 1000); + auto now = std::chrono::steady_clock::now(); + for (WatchSlot &slot : g_watchSlots) { + if (slot.state.load(std::memory_order_acquire) != kReady) { + continue; + } + if (now < slot.dueAt) { + continue; + } + uint8_t after[OTEL_HEADER_SIZE]; + memcpy(after, slot.recordAddr, OTEL_HEADER_SIZE); + bool changed = memcmp(slot.snapshot, after, OTEL_HEADER_SIZE) != 0; + if (changed && looksLikeOtelRecord(after)) { + fprintf(stderr, + "[ddprof-debug] UAF WRITE DETECTED: freed ProfiledThread tid=%d otel-header " + "changed after free and looks like a live OTel record (addr=%p)\n", + slot.tid, slot.recordAddr); + fprintf(stderr, "[ddprof-debug] before:"); + for (uint8_t b : slot.snapshot) fprintf(stderr, " %02x", b); + fprintf(stderr, "\n[ddprof-debug] after: "); + for (uint8_t b : after) fprintf(stderr, " %02x", b); + fprintf(stderr, "\n"); + } + slot.state.store(kFree, std::memory_order_release); + } + } +} +} // namespace + +static void watchFreedContextMemory(const void *recordAddr, int tid) { + if (!g_watcherStarted.exchange(true, std::memory_order_acq_rel)) { + std::thread(watcherLoop).detach(); + } + for (WatchSlot &slot : g_watchSlots) { + int expected = kFree; + if (slot.state.compare_exchange_strong(expected, kFilling, std::memory_order_acq_rel)) { + slot.recordAddr = recordAddr; + slot.tid = tid; + memcpy(slot.snapshot, recordAddr, OTEL_HEADER_SIZE); + slot.dueAt = std::chrono::steady_clock::now() + std::chrono::milliseconds(kWatchDelayMillis); + slot.state.store(kReady, std::memory_order_release); + return; + } + } + // No free slot: drop this sample rather than blocking the freeing thread or spawning + // another thread. +} +#endif + void ProfiledThread::initTLSKey() { static pthread_once_t tls_initialized = PTHREAD_ONCE_INIT; pthread_once(&tls_initialized, doInitTLSKey); @@ -32,7 +148,14 @@ inline void ProfiledThread::freeKey(void *key) { ProfiledThread *tls_ref = (ProfiledThread *)(key); if (tls_ref != NULL) { SignalBlocker blocker; +#ifdef DEBUG + void *recordAddr = (void *)tls_ref->getOtelContextRecord(); + int tid = tls_ref->tid(); +#endif delete tls_ref; +#ifdef DEBUG + watchFreedContextMemory(recordAddr, tid); +#endif } } @@ -60,7 +183,14 @@ void ProfiledThread::release() { if (tls != NULL) { SignalBlocker blocker; pthread_setspecific(key, NULL); +#ifdef DEBUG + void *recordAddr = (void *)tls->getOtelContextRecord(); + int tid = tls->tid(); +#endif delete tls; +#ifdef DEBUG + watchFreedContextMemory(recordAddr, tid); +#endif } } diff --git a/ddprof-lib/src/test/cpp/frame_ut.cpp b/ddprof-lib/src/test/cpp/frame_ut.cpp index c31d18f770..a211eb02a3 100644 --- a/ddprof-lib/src/test/cpp/frame_ut.cpp +++ b/ddprof-lib/src/test/cpp/frame_ut.cpp @@ -56,13 +56,6 @@ TEST(FrameTypeEncodeTest, RawPointerBitNotSetByDefault) { EXPECT_EQ(encoded & (1 << 30), 0) << "rawPointer flag (bit 30) must not be set by default"; } -TEST(FrameTypeEncodeTest, EncodedValuesArePositive) { - for (int t = FRAME_INTERPRETED; t <= FRAME_TYPE_MAX; ++t) { - int encoded = FrameType::encode(t, 0); - EXPECT_GT(encoded, 0) << "encode() must return a positive value for type " << t; - } -} - // ---- decode ---------------------------------------------------------------- TEST(FrameTypeDecodeTest, DecodeZeroReturnsJitCompiled) { @@ -110,15 +103,6 @@ TEST(FrameTypeDecodeTest, RoundTripAllTypesNonZeroBci) { } } -TEST(FrameTypeDecodeTest, DecodedTypeIsInValidRange) { - for (int t = FRAME_INTERPRETED; t <= FRAME_TYPE_MAX; ++t) { - int encoded = FrameType::encode(t, 42); - FrameTypeId decoded = FrameType::decode(encoded); - EXPECT_GE(decoded, FRAME_INTERPRETED); - EXPECT_LE(decoded, FRAME_TYPE_MAX); - } -} - // ---- isRawPointer ---------------------------------------------------------- TEST(FrameTypeIsRawPointerTest, FalseForZero) { @@ -154,3 +138,15 @@ TEST(FrameTypeIsRawPointerTest, FalseWithOnlyBit30SetOnNegative) { EXPECT_LT(negativeWithBit30, 0); EXPECT_FALSE(FrameType::isRawPointer(negativeWithBit30)); } + +TEST(FrameTypeIsRawPointerTest, FalseForRawAsgctBciWithBit30SetButNoEncodedMarker) { + // Regression test: a raw, unencoded ASGCT BCI (bit 20 / ENCODED_MASK not set) + // that incidentally has bit 30 set must NOT be reported as a raw pointer. + // Such values occur on non-HotSpot VMs (e.g. J9), which never call encode() + // and therefore never set ENCODED_MASK; only encode(..., rawPointer=true) + // (HotSpot-only, per its own assert) is allowed to set RAW_POINTER_MASK. + int rawAsgctBciWithBit30 = 1 << 30; + EXPECT_FALSE(FrameType::isRawPointer(rawAsgctBciWithBit30)) + << "isRawPointer() must require ENCODED_MASK (bit 20) before trusting bit 30, " + << "otherwise raw ASGCT BCIs that never went through encode() can false-positive"; +} diff --git a/ddprof-stresstest/src/chaos/README.md b/ddprof-stresstest/src/chaos/README.md index 5dfcabb707..2c72a4eba0 100644 --- a/ddprof-stresstest/src/chaos/README.md +++ b/ddprof-stresstest/src/chaos/README.md @@ -15,6 +15,7 @@ the runner script. | `classloader-churn` | class unload racing stack walk, `CodeCache`/`Symbols` invalidation | | `alloc-storm` | Java alloc engine + GOT-patched libc malloc/free | | `trace-context` | `setContext`/`clearContext` racing signals, span ID propagation | +| `vthread-context-cascade` | tracer-style context propagate/activate/restore across fan-out cascades of short-lived virtual threads, racing carrier-pin churn to trigger the `ContextStorageMode.THREAD` stale-carrier `DirectByteBuffer` use-after-free | ## Deferred diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java index 9b2cccc432..14d70be832 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java @@ -43,15 +43,15 @@ public static void main(String[] args) throws Exception { log("starting duration=" + parsed.duration + " antagonists=" + parsed.antagonists); List running = new ArrayList<>(); - for (String name : parsed.antagonists) { - Antagonist a = create(name); - a.start(); - running.add(a); - log("antagonist started: " + a.name()); - } - - long deadlineNanos = System.nanoTime() + parsed.duration.toNanos(); try { + for (String name : parsed.antagonists) { + Antagonist a = create(name); + a.start(); + running.add(a); + log("antagonist started: " + a.name()); + } + + long deadlineNanos = System.nanoTime() + parsed.duration.toNanos(); while (System.nanoTime() < deadlineNanos) { Thread.sleep(1_000L); } @@ -72,6 +72,8 @@ private static Antagonist create(String name) { return new AllocStormAntagonist(); case "vthread-churn": return new VirtualThreadChurnAntagonist(); + case "vthread-context-cascade": + return new VirtualThreadContextCascadeAntagonist(); case "classloader-churn": return new ClassLoaderChurnAntagonist(); case "trace-context": diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java new file mode 100644 index 0000000000..78de52e195 --- /dev/null +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java @@ -0,0 +1,545 @@ +/* + * Copyright 2026, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.datadoghq.profiler.chaos; + +import datadog.trace.api.Trace; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.concurrent.Executor; +import java.util.concurrent.Semaphore; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Simulates a tracer's context propagation across a cascade of short-lived virtual threads, and + * deliberately races the carrier-thread churn needed to expose a stale-carrier use-after-free. + * + *

Unlike a hand-rolled context simulation, every context write here is driven by the real + * Datadog tracer: {@link #runCascadeNode}/{@link #nestedOp}/{@link #staleRacerBefore}/{@link + * #staleRacerAfter} are {@link Trace}-annotated methods. Under a {@code dd-java-agent}- + * instrumented JVM the tracer intercepts each one, activating a real span on entry ({@code + * JavaProfiler.setContext}) and deactivating it on exit ({@code clearContext}) — exactly what a + * genuinely-instrumented application does. {@code dd-trace-api} is a {@code compileOnly} + * dependency (see {@code TraceContextAntagonist}): at runtime the patched {@code dd-java-agent} + * provides the (relocated) classes and intercepts the annotation, so this antagonist never touches + * {@code com.datadoghq.profiler.*} directly. + * + *

Each cascade node ({@link #runCascadeNode}) runs a handful of nested sub-operations, each + * activating a new child span and sleeping briefly ({@link #forceUnmount}) to force the virtual + * thread off its current carrier before the span's exit (and the next nested entry) write context + * again — the high context-change-rate dimension. It then fans out to child virtual threads while + * its own span is still active, handing the tracer's own async/virtual-thread instrumentation the + * job of propagating that span as parent context onto the next hop (the chain continues on a new + * thread/carrier). On exit, the span's close restores whatever was logically active underneath it. + * + *

A second, independent driver ({@link #pinningChurnLoop()}) continuously pins short-lived + * virtual threads to their carrier (a {@code synchronized} block held across a blocking sleep) + * across a rotating set of monitors, forcing the JDK's virtual-thread scheduler to spin up extra + * compensating carrier platform threads while the pins are held. + * + *

Targets: {@code com.datadoghq.profiler.OtelContextStorage}'s {@code ContextStorageMode.THREAD} + * fallback (the default on JDK 21+ without {@code --add-exports + * java.base/jdk.internal.misc=ALL-UNNAMED}). Under that mode the {@code DirectByteBuffer} conduit + * for a virtual thread's OTEL context (which backs both the span id fields written by {@code + * setContext} and any custom attributes) is cached in a plain {@code ThreadLocal} the first time + * the thread writes context, bound to whichever carrier happened to be mounted at that moment; the + * conduit wraps memory embedded directly inside that carrier's native {@code ProfiledThread} + * (see {@code threadLocalData.h}), which is {@code delete}d — a real {@code free()} — the moment + * the carrier's JVMTI {@code ThreadEnd} fires. The nested-op loop in {@link #runCascadeNode} races + * this cheaply: a nested span's entry writes, {@link #forceUnmount} forces a short real unmount, + * then the span's exit (or the next nested entry) writes again — if the virtual thread was + * remounted elsewhere, that second write reuses the (possibly now-stale) cached conduit. + * + *

Catching genuine memory reuse (not just misattribution) needs the original carrier + * to actually be torn down first. {@link #driverLoop} runs a duty cycle for the cascade/pin-churn + * side only (a short active burst followed by a quiet phase with no new work, long enough for the + * JDK's default {@code ForkJoinPool} scheduler — 30s keep-alive, see {@code + * java.lang.VirtualThread#createDefaultScheduler()} — to actually reap idle carriers). Stale- + * buffer racers ({@link #runStaleBufferRacer}) don't wait on that: every racer runs on a virtual + * thread bound to a purpose-built {@link #fastCarrierScheduler} — a {@link ThreadPoolExecutor} + * with a 200ms keep-alive — via the package-private {@code VirtualThread(Executor, String, int, + * Runnable)} constructor (JDK 21+; see {@code java.lang.VirtualThread}, requires {@code + * --add-opens java.base/java.lang=ALL-UNNAMED}). {@link #staleRacerLoop} keeps up to {@link + * #MAX_STALE_RACERS} of these racers in flight continuously, independent of the cascade/pin duty + * cycle above — sustained concurrent hammering rather than one batch per cycle, since more + * concurrent carrier churn means more chances for a freed {@code ProfiledThread} slot to actually + * get reused before a stale racer touches it. Each racer writes context once ({@link + * #staleRacerBefore}), sleeps a randomized interval straddling the scheduler's keep-alive ({@link + * #FAST_STALE_RACER_SLEEP_MIN_MILLIS}–{@link #FAST_STALE_RACER_SLEEP_MAX_MILLIS}), then writes + * again ({@link #staleRacerAfter}) — varying the wait means a run samples both "carrier likely + * still tearing down" and "carrier long gone, memory possibly already reused" timings, instead of + * only ever probing one fixed instant. + * + *

Reflectively detects {@code Thread.ofVirtual()} (Java 21+); gracefully no-ops the whole + * antagonist on older runtimes where it's absent. But on a JDK where it is present, the + * {@code VirtualThread} constructor above is required, not optional: it's this antagonist's only + * way to reach the stale-carrier UAF without a slow duty-cycle wait, so {@link #start} fails fast + * with an {@link IllegalStateException} (crashing the chaos harness process, which the CI runner + * already treats as a failure) rather than silently losing that coverage if a missing {@code + * --add-opens} flag or a JDK build that changed the constructor's signature makes it + * unreachable. Live virtual thread count is bounded by semaphore permit pools so cascades, + * pinning churn, and stale-buffer racers cannot runaway the JVM. + */ +public final class VirtualThreadContextCascadeAntagonist implements Antagonist { + + private static final Method OF_VIRTUAL = resolveOfVirtual(); + private static final Method BUILDER_START = resolveBuilderStart(); + + private static final int FAN_OUT = 3; + private static final int MAX_DEPTH = 4; + private static final int NESTED_OPS_PER_HOP = 3; + private static final int MAX_LIVE_VTHREADS = 512; + private static final int ROOT_BATCH = 16; + + // Carrier-pinning churn: rotating monitors held across a blocking sleep to force the + // scheduler to spin up (and later reap) compensating carrier platform threads. + private static final int PIN_LOCK_COUNT = 32; + private static final int PIN_BATCH = 24; + private static final int MAX_PIN_VTHREADS = 128; + private static final Object[] PIN_LOCKS = new Object[PIN_LOCK_COUNT]; + + static { + for (int i = 0; i < PIN_LOCK_COUNT; i++) { + PIN_LOCKS[i] = new Object(); + } + } + + // Duty cycle driving genuine carrier teardown: an active burst of cascade/pin-churn work, + // then a quiet phase long enough to clear the JDK's default 30s carrier keep-alive so idle + // carriers actually exit before the stale-buffer racers spawned in the burst wake back up. + private static final long ACTIVE_PHASE_MILLIS = 8_000L; + private static final long QUIET_PHASE_MILLIS = 40_000L; + + // Stale-buffer racers run continuously on their own driver, decoupled from the cascade/pin + // duty cycle above — that cycle only exists to let the JDK default scheduler's 30s keep-alive + // reap cascade/pin carriers, which is irrelevant to racers bound to fastCarrierScheduler. + private static final int STALE_RACER_BATCH = 64; + private static final int MAX_STALE_RACERS = 1024; + + // A short keep-alive forces genuine carrier-thread exit (and thus a real ProfiledThread + // free()) in well under a second instead of 30s. The racer's sleep is randomized across a + // range straddling this keep-alive so runs cover both "carrier likely still exiting" and + // "carrier long gone, memory possibly already reused" timings instead of one fixed instant. + private static final long FAST_SCHEDULER_KEEPALIVE_MILLIS = 200L; + private static final long FAST_STALE_RACER_SLEEP_MIN_MILLIS = 50L; + private static final long FAST_STALE_RACER_SLEEP_MAX_MILLIS = FAST_SCHEDULER_KEEPALIVE_MILLIS * 4; + private static final Constructor VTHREAD_CTOR = resolveVirtualThreadCtor(); + + private final Semaphore liveVthreads = new Semaphore(MAX_LIVE_VTHREADS); + private final Semaphore livePinVthreads = new Semaphore(MAX_PIN_VTHREADS); + private final Semaphore liveStaleRacers = new Semaphore(MAX_STALE_RACERS); + private final Executor fastCarrierScheduler = VTHREAD_CTOR != null ? newFastCarrierScheduler() : null; + private volatile boolean running; + private volatile boolean activePhase = true; + private Thread driver; + private Thread pinningDriver; + private Thread staleRacerDriver; + private final AtomicLong sink = new AtomicLong(); + + @Override + public String name() { + return "vthread-context-cascade"; + } + + @Override + public void start() { + if (OF_VIRTUAL != null && BUILDER_START != null && VTHREAD_CTOR == null) { + // We're on a VT-capable JDK, so the fast-carrier constructor should be reachable — + // its absence (missing --add-opens, or a JDK build that changed the constructor) + // would otherwise silently drop stale-racer coverage instead of failing the run. + throw new IllegalStateException( + "vthread-context-cascade: java.lang.VirtualThread(Executor, String, int, " + + "Runnable) is unreachable on a VT-capable JDK — pass --add-opens " + + "java.base/java.lang=ALL-UNNAMED, or this antagonist loses its core " + + "stale-carrier UAF coverage"); + } + running = true; + driver = new Thread(this::driverLoop, "chaos-vthread-cascade-driver"); + driver.setDaemon(true); + driver.start(); + pinningDriver = new Thread(this::pinningChurnLoop, "chaos-vthread-cascade-pin-driver"); + pinningDriver.setDaemon(true); + pinningDriver.start(); + staleRacerDriver = new Thread(this::staleRacerLoop, "chaos-vthread-cascade-racer-driver"); + staleRacerDriver.setDaemon(true); + staleRacerDriver.start(); + } + + @Override + public void stopGracefully(Duration timeout) { + running = false; + long deadlineNanos = System.nanoTime() + timeout.toNanos(); + try { + driver.join(remainingMillis(deadlineNanos)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + pinningDriver.join(remainingMillis(deadlineNanos)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + staleRacerDriver.join(remainingMillis(deadlineNanos)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + // Best-effort drain of in-flight cascades so the JVM doesn't exit mid-fan-out. + try { + liveVthreads.tryAcquire(MAX_LIVE_VTHREADS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + livePinVthreads.tryAcquire(MAX_PIN_VTHREADS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + // Stale-buffer racers sleep well past stopGracefully's own timeout budget, so this is + // best-effort only — outstanding racers are daemon threads and die with the JVM. + try { + liveStaleRacers.tryAcquire(MAX_STALE_RACERS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** Milliseconds remaining until {@code deadlineNanos}, clamped to zero (never negative). */ + private static long remainingMillis(long deadlineNanos) { + long remainingNanos = deadlineNanos - System.nanoTime(); + return remainingNanos <= 0L ? 0L : TimeUnit.NANOSECONDS.toMillis(remainingNanos); + } + + /** + * Independently pins short-lived virtual threads to their carrier and releases them again, + * forcing the scheduler to grow and shrink its compensating carrier pool while cascade nodes + * are racing carrier migration on the other driver. + */ + private void pinningChurnLoop() { + if (OF_VIRTUAL == null || BUILDER_START == null) { + return; + } + while (running) { + if (!activePhase) { + // Quiet phase: stay out of the way so idle carriers can actually be reaped. + try { + Thread.sleep(50L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + continue; + } + for (int i = 0; i < PIN_BATCH && running && activePhase; i++) { + if (!livePinVthreads.tryAcquire()) { + continue; + } + final Object lock = PIN_LOCKS[ThreadLocalRandom.current().nextInt(PIN_LOCK_COUNT)]; + try { + Object builder = OF_VIRTUAL.invoke(null); + BUILDER_START.invoke( + builder, + (Runnable) + () -> { + try { + synchronized (lock) { + // Sleeping while holding a monitor pins the + // carrier: the scheduler must compensate with an + // extra platform thread for other runnable + // virtual threads, then reap it once idle. + Thread.sleep(2L); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + livePinVthreads.release(); + } + }); + } catch (Throwable t) { + livePinVthreads.release(); + } + } + try { + Thread.sleep(1L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + private void driverLoop() { + if (OF_VIRTUAL == null || BUILDER_START == null) { + System.out.println("[chaos] vthread-context-cascade: skipping (Thread.ofVirtual not available)"); + return; + } + while (running) { + // Active phase: mint cascade work. + activePhase = true; + long activeDeadlineNanos = System.nanoTime() + Duration.ofMillis(ACTIVE_PHASE_MILLIS).toNanos(); + while (running && System.nanoTime() < activeDeadlineNanos) { + for (int i = 0; i < ROOT_BATCH && running; i++) { + // Mint one chain-of-operations: a fresh synthetic trace, its root span + // activated by the tracer the moment runCascadeNode is entered. + spawnCascadeNode(0); + } + try { + Thread.sleep(1L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + if (!running) { + return; + } + // Quiet phase: mint nothing new so short-lived virtual threads drain and carriers + // can actually sit idle past the scheduler's 30s keep-alive and get reaped. + activePhase = false; + try { + Thread.sleep(QUIET_PHASE_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + /** + * Independent driver that keeps the stale-buffer racer population saturated at {@link + * #MAX_STALE_RACERS} for as long as the antagonist runs, regardless of the cascade/pin duty + * cycle — the fast-carrier scheduler needs no quiet phase to reap its carriers, so there's no + * reason to gate racer spawning on one. This is the "many vthreads hammering" dimension: + * sustained concurrent pressure on the fast-carrier scheduler's pool, not periodic bursts. + */ + private void staleRacerLoop() { + while (running) { + boolean spawnedAny = spawnStaleBufferRacers(); + try { + // Once the pool is fully saturated, back off instead of re-polling 1000x/sec for + // permits that free up far slower (racers sleep up to FAST_STALE_RACER_SLEEP_MAX_MILLIS). + Thread.sleep(spawnedAny ? 1L : 20L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + /** Returns whether at least one racer was actually started this pass. */ + private boolean spawnStaleBufferRacers() { + boolean spawnedAny = false; + for (int i = 0; i < STALE_RACER_BATCH && running; i++) { + if (!liveStaleRacers.tryAcquire()) { + continue; + } + if (startFastCarrierVirtualThread(this::runStaleBufferRacer) == null) { + liveStaleRacers.release(); + } else { + spawnedAny = true; + } + } + return spawnedAny; + } + + /** + * Activates a span once, sleeps past the carrier's keep-alive, then activates another. Under + * {@code ContextStorageMode.THREAD} the second activation reuses whatever {@code + * DirectByteBuffer} was cached on the first — if the original carrier has since been reaped + * and its {@code ProfiledThread} memory reused, this is a genuine use-after-free. + * + *

The sleep is randomized per racer across [{@link #FAST_STALE_RACER_SLEEP_MIN_MILLIS}, + * {@link #FAST_STALE_RACER_SLEEP_MAX_MILLIS}] rather than fixed, so a sustained run samples a + * spread of timings relative to {@link #FAST_SCHEDULER_KEEPALIVE_MILLIS} instead of only ever + * probing the same instant. + */ + private void runStaleBufferRacer() { + try { + staleRacerBefore(); + long sleepMillis = + ThreadLocalRandom.current() + .nextLong( + FAST_STALE_RACER_SLEEP_MIN_MILLIS, + FAST_STALE_RACER_SLEEP_MAX_MILLIS + 1); + Thread.sleep(sleepMillis); + staleRacerAfter(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + liveStaleRacers.release(); + } + } + + @Trace(operationName = "chaos.stale.before", resourceName = "chaos.stale.before") + private void staleRacerBefore() { + // Entry/exit alone is the "write" — the tracer activates/deactivates a span around this + // no-op body, exactly like TraceContextAntagonist's inner()/outer(). + } + + @Trace(operationName = "chaos.stale.after", resourceName = "chaos.stale.after") + private void staleRacerAfter() {} + + /** Spawns one virtual thread to run a cascade node, respecting the live-thread budget. */ + private void spawnCascadeNode(int depth) { + if (!running || !liveVthreads.tryAcquire()) { + return; + } + try { + Object builder = OF_VIRTUAL.invoke(null); + BUILDER_START.invoke( + builder, + (Runnable) + () -> { + try { + runCascadeNode(depth); + } finally { + liveVthreads.release(); + } + }); + } catch (Throwable t) { + liveVthreads.release(); + } + } + + /** + * One hop of the cascade, run as its own {@link Trace}-annotated span on a fresh virtual + * thread. The tracer activates this span on entry and deactivates it on exit; while it's + * active, this hop fans out to child hops on new virtual threads, handing the tracer's own + * virtual-thread instrumentation the job of propagating it as parent context onto each child. + */ + @Trace(operationName = "chaos.cascade.hop", resourceName = "chaos.cascade.hop") + private void runCascadeNode(int depth) { + long r = ThreadLocalRandom.current().nextLong() ^ ((long) depth << 48); + for (int op = 0; op < NESTED_OPS_PER_HOP && running; op++) { + r = nestedOp(r); + } + sink.addAndGet(r); + + // Propagate the chain to the next hop before this thread's own span is torn down. + if (depth < MAX_DEPTH && running) { + for (int i = 0; i < FAN_OUT; i++) { + spawnCascadeNode(depth + 1); + } + } + } + + /** + * A nested sub-operation within a hop: the tracer activates a child span on entry, {@link + * #forceUnmount} sleeps briefly to force the virtual thread off its current carrier, then the + * span's exit (or the next nested entry) writes context again — racing carrier migration + * against the cached context conduit. + */ + @Trace(operationName = "chaos.cascade.op", resourceName = "chaos.cascade.op") + private long nestedOp(long seed) { + return forceUnmount(seed); + } + + /** + * Burns some CPU (a stand-in for real work) then blocks briefly, forcing the virtual thread + * to genuinely unmount. On resumption it may be remounted on a different carrier than the + * one it was on when the context before this call was written. + */ + private static long forceUnmount(long seed) { + long r = seed; + for (int i = 0; i < 2_000; i++) { + r = r * 6364136223846793005L + 1442695040888963407L; + } + try { + Thread.sleep(1L + (r & 1)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return r; + } + + /** + * Resolves {@code java.lang.VirtualThread}'s package-private {@code (Executor, String, int, + * Runnable)} constructor, the only way to bind a virtual thread to a custom scheduler on + * mainline OpenJDK 21+ (there is no public {@code Thread.ofVirtual().scheduler(Executor)} + * API). Requires {@code --add-opens java.base/java.lang=ALL-UNNAMED}; returns {@code null} + * (caller falls back to the default-scheduler path) if that flag is absent, the JVM is + * pre-21, or the constructor's signature changes on some future JDK build. + */ + private static Constructor resolveVirtualThreadCtor() { + try { + Class vthreadClass = Class.forName("java.lang.VirtualThread"); + Constructor ctor = + vthreadClass.getDeclaredConstructor( + Executor.class, String.class, int.class, Runnable.class); + ctor.setAccessible(true); + return ctor; + } catch (Throwable t) { + return null; + } + } + + /** + * A carrier pool with a short keep-alive instead of the JDK default scheduler's fixed 30s, so + * an idle carrier's OS thread actually exits (and its {@code ProfiledThread} is freed) in + * well under a second. + */ + private static Executor newFastCarrierScheduler() { + return new ThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + FAST_SCHEDULER_KEEPALIVE_MILLIS, + TimeUnit.MILLISECONDS, + new SynchronousQueue<>(), + r -> { + Thread carrier = new Thread(r, "chaos-vthread-cascade-fast-carrier"); + carrier.setDaemon(true); + return carrier; + }); + } + + /** + * Starts {@code task} on a virtual thread bound to {@link #fastCarrierScheduler}. Returns + * {@code null} (never starts anything) if {@link #VTHREAD_CTOR} wasn't resolved or the + * reflective construction fails; {@link #start} already fails fast when {@link #VTHREAD_CTOR} + * is unreachable on a VT-capable JDK, so callers here only need to handle the rarer + * construction-failure case (e.g. transient reflection errors). + * + *

{@code 0} for the characteristics argument mirrors {@code Thread.ofVirtual()}'s default + * (the only known bit, {@code Thread.NO_INHERIT_THREAD_LOCALS}, is unset). + */ + private Thread startFastCarrierVirtualThread(Runnable task) { + if (VTHREAD_CTOR == null) { + return null; + } + try { + Thread t = (Thread) VTHREAD_CTOR.newInstance(fastCarrierScheduler, null, 0, task); + t.start(); + return t; + } catch (Throwable t) { + return null; + } + } + + private static Method resolveOfVirtual() { + try { + return Thread.class.getMethod("ofVirtual"); + } catch (NoSuchMethodException e) { + return null; + } + } + + private static Method resolveBuilderStart() { + try { + Class builder = Class.forName("java.lang.Thread$Builder"); + return builder.getMethod("start", Runnable.class); + } catch (ClassNotFoundException | NoSuchMethodException e) { + return null; + } + } +} diff --git a/utils/run-chaos-harness.sh b/utils/run-chaos-harness.sh new file mode 100755 index 0000000000..45a35ba4ab --- /dev/null +++ b/utils/run-chaos-harness.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# +# Standalone chaos-harness runner. Builds/locates the ddprof jar, patches a +# dd-java-agent build with it, and runs the long-running chaos harness +# (ddprof-stresstest/src/chaos) with a chosen antagonist set and allocator. +# +# Usable both from CI (see .gitlab/reliability/chaos_check.sh, a thin wrapper +# around this script) and by hand for local repro — no CI-only assumptions +# (no fixed /opt or /var/lib paths, no CI-artifact env vars required). +# +# Usage: run-chaos-harness.sh [config] [allocator] [antagonists] +# config: profiler | profiler+tracer (default: profiler+tracer) +# allocator: gmalloc | tcmalloc | jemalloc (default: gmalloc) +# antagonists: comma-separated antagonist names; overrides the config default +# +# Env vars: +# CHAOS_JDK sdkman-style version (e.g. 21.0.3-tem) to require/download. +# If unset, uses whatever `java` is already on PATH. +# CHAOS_JDK_DIR where to install a downloaded JDK (default: under +# CHAOS_WORK_DIR; set to a persistent path to cache across runs). +# CHAOS_WORK_DIR scratch dir for the patched agent, downloaded jars, and +# hs_err.log (default: /.chaos-harness). +# CHAOS_REFRESH_AGENT=1 force re-download of dd-java-agent.jar even if cached. + +set +e + +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT="$( cd "${HERE}/.." >/dev/null 2>&1 && pwd )" + +RUNTIME=${1} +CONFIG=${2:-profiler+tracer} +ALLOCATOR=${3:-gmalloc} +ANTAGONISTS_OVERRIDE=${4:-} + +if [ -z "${RUNTIME}" ]; then + echo "usage: $0 [config] [allocator] [antagonists]" >&2 + exit 1 +fi + +WORK_DIR="${CHAOS_WORK_DIR:-${ROOT}/.chaos-harness}" +mkdir -p "${WORK_DIR}" + +echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}" + +# --- JDK resolution --------------------------------------------------------- +# Only fetch a JDK if the caller explicitly pinned one via CHAOS_JDK and it +# doesn't match what's already active; otherwise just use `java` as found. +if [ -n "${CHAOS_JDK:-}" ]; then + JDK_MAJOR="${CHAOS_JDK%%.*}" + ACTIVE_MAJOR=$(java -version 2>&1 | head -1 | grep -oE '"[0-9]+' | tr -d '"') + if [ "${ACTIVE_MAJOR}" != "${JDK_MAJOR}" ]; then + JDK_ARCH=$(uname -m | sed 's/x86_64/x64/') + JDK_INSTALL_DIR="${CHAOS_JDK_DIR:-${WORK_DIR}/jdk-${CHAOS_JDK}}" + if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then + TMP=$(mktemp -d) + DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse" + echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..." + if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then + echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2 + rm -rf "${TMP}" + exit 1 + fi + mkdir -p "${JDK_INSTALL_DIR}" + tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1 + rm -rf "${TMP}" + fi + export JAVA_HOME="${JDK_INSTALL_DIR}" + export PATH="${JAVA_HOME}/bin:${PATH}" + fi +fi + +if ! command -v java >/dev/null 2>&1; then + echo "FAIL:no java on PATH (set CHAOS_JDK to have one downloaded)" >&2 + exit 1 +fi +echo "Using: $(java -version 2>&1 | head -1)" + +# --- ddprof jar -------------------------------------------------------------- +DDPROF_JAR=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1) +if [ -z "${DDPROF_JAR}" ] || [ ! -f "${DDPROF_JAR}" ]; then + echo "FAIL:no local ddprof jar found — build one first: ./gradlew :ddprof-lib:jar" >&2 + exit 1 +fi +echo "Using local ddprof jar: ${DDPROF_JAR}" + +# --- dd-java-agent ------------------------------------------------------------ +DD_AGENT_JAR="${WORK_DIR}/dd-java-agent.jar" +if [ ! -f "${DD_AGENT_JAR}" ] || [ "${CHAOS_REFRESH_AGENT:-0}" = "1" ]; then + echo "Fetching dd-java-agent.jar..." + wget -q --timeout=120 --tries=3 -O "${DD_AGENT_JAR}" 'https://dtdg.co/latest-java-tracer' +fi +if [ ! -f "${DD_AGENT_JAR}" ]; then + echo "FAIL:dd-java-agent.jar download failed" >&2 + exit 1 +fi + +CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar" +if [ ! -f "${CHAOS_JAR}" ]; then + echo "chaos.jar not present — building inline" + ( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q ) +fi +if [ ! -f "${CHAOS_JAR}" ]; then + echo "FAIL:chaos.jar unavailable" >&2 + exit 1 +fi + +# Patch dd-java-agent.jar with the locally built ddprof contents so the +# agent's (relocated) profiler classes match the version under test. +PATCHED_AGENT="${WORK_DIR}/dd-java-agent-patched.jar" +DD_AGENT_JAR="${DD_AGENT_JAR}" DDPROF_JAR="${DDPROF_JAR}" OUTPUT_JAR="${PATCHED_AGENT}" \ + "${ROOT}/utils/patch-dd-java-agent.sh" +if [ ! -f "${PATCHED_AGENT}" ]; then + echo "FAIL:dd-java-agent patching failed" >&2 + exit 1 +fi + +case $CONFIG in + profiler) + ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" + # @Trace is a no-op without the tracer, so trace-context is excluded here. + DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,vthread-context-cascade,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + ;; + profiler+tracer) + ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" + DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,vthread-context-cascade,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + ;; + *) + echo "Unknown configuration: $CONFIG (valid: profiler, profiler+tracer)" >&2 + exit 1 + ;; +esac +ANTAGONISTS="${ANTAGONISTS_OVERRIDE:-${DEFAULT_ANTAGONISTS}}" + +case $ALLOCATOR in + gmalloc) + # Turn silent heap corruption (e.g. a stale-pointer UAF write into a freed + # chunk) into an immediate, attributable SIGABRT instead of a crash much + # later in an unrelated allocation. Only meaningful against glibc's own + # malloc, not the LD_PRELOAD-replaced allocators below. + export MALLOC_CHECK_=3 + export MALLOC_PERTURB_=$((RANDOM % 255 + 1)) + ;; + tcmalloc) + export LD_PRELOAD=$(find /usr/lib/ /usr/lib64/ /opt/homebrew/lib/ /usr/local/lib/ -name 'libtcmalloc_minimal.so.4' -o -name 'libtcmalloc.dylib' 2>/dev/null | head -1) + ;; + jemalloc) + export LD_PRELOAD=$(find /usr/lib/ /usr/lib64/ /opt/homebrew/lib/ /usr/local/lib/ -name 'libjemalloc.so' -o -name 'libjemalloc.dylib' 2>/dev/null | head -1) + ;; + *) + echo "Unknown allocator: $ALLOCATOR (valid: gmalloc, tcmalloc, jemalloc)" >&2 + exit 1 + ;; +esac +echo "LD_PRELOAD=${LD_PRELOAD:-}" + +HS_ERR="${CHAOS_ERROR_FILE:-${WORK_DIR}/hs_err.log}" +rm -f "${HS_ERR}" + +timeout "$((RUNTIME + 300))" \ +java -javaagent:${PATCHED_AGENT} \ + --add-opens java.base/java.lang=ALL-UNNAMED \ + ${ENABLEMENT} \ + -Ddd.profiling.upload.period=10 \ + -Ddd.profiling.start-force-first=true \ + -Ddd.profiling.ddprof.liveheap.enabled=true \ + -Ddd.profiling.ddprof.alloc.enabled=true \ + -Ddd.profiling.ddprof.wall.enabled=true \ + -Ddd.profiling.ddprof.nativemem.enabled=true \ + -Ddd.env=java-profiler-stability \ + -Ddd.service=java-profiler-chaos \ + -Xmx2g -Xms2g \ + -XX:MaxMetaspaceSize=384m \ + -XX:NativeMemoryTracking=summary \ + -XX:ErrorFile=${HS_ERR} \ + -jar ${CHAOS_JAR} \ + --duration ${RUNTIME}s \ + --antagonists ${ANTAGONISTS} + +RC=$? +echo "RC=$RC" + +if [ $RC -ne 0 ]; then + CRASH_MSG="Chaos harness crashed (RC=${RC})" + if [ -f "${HS_ERR}" ]; then + SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120) + FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120) + [ -n "${SIG}" ] && CRASH_MSG="${CRASH_MSG};${SIG}" + [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}" + echo "hs_err written to ${HS_ERR}" + fi + echo "FAIL:${CRASH_MSG}" >&2 + exit 1 +fi + +echo "Chaos run completed cleanly"