From 99699e0dc141409de1adbf1234e5fc56928e2ce3 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 8 Jul 2026 17:35:28 +0200 Subject: [PATCH 01/10] Add vthread-context-cascade chaos antagonist; default MALLOC_CHECK_/MALLOC_PERTURB_ for glibc test runs Antagonist races tracer-style context propagation against virtual-thread carrier churn to target the ContextStorageMode.THREAD stale-carrier use-after-free. Since such heap corruption manifests silently until an unrelated later allocation, also turn on glibc's own corruption checks by default for chaos runs and ddprof-test suites (skipped on musl and whenever a sanitizer/allocator already owns malloc via LD_PRELOAD). Co-Authored-By: Claude Sonnet 5 --- .gitlab/reliability/chaos_check.sh | 6 + .../datadoghq/profiler/ProfilerTestPlugin.kt | 13 + ddprof-stresstest/src/chaos/README.md | 1 + .../com/datadoghq/profiler/chaos/Main.java | 2 + ...VirtualThreadContextCascadeAntagonist.java | 470 ++++++++++++++++++ 5 files changed, 492 insertions(+) create mode 100644 ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java diff --git a/.gitlab/reliability/chaos_check.sh b/.gitlab/reliability/chaos_check.sh index 844a8522b5..483bd06c3a 100755 --- a/.gitlab/reliability/chaos_check.sh +++ b/.gitlab/reliability/chaos_check.sh @@ -119,6 +119,12 @@ esac case $ALLOCATOR in gmalloc) echo "Running with gmalloc" + # Turn silent heap corruption (e.g. a stale-pointer UAF write into a freed + # chunk) into an immediate, attributable SIGABRT instead of a crash much + # later in an unrelated allocation. Only meaningful against glibc's own + # malloc, not the LD_PRELOAD-replaced allocators below. + export MALLOC_CHECK_=3 + export MALLOC_PERTURB_=$((RANDOM % 255 + 1)) ;; tcmalloc) echo "Running with tcmalloc" diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt index f76da2c8a1..bae6943563 100644 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt @@ -3,6 +3,7 @@ package com.datadoghq.profiler import com.datadoghq.native.NativeBuildExtension import com.datadoghq.native.model.BuildConfiguration +import com.datadoghq.native.model.Platform import com.datadoghq.native.util.PlatformUtils import org.gradle.api.DefaultTask import org.gradle.api.GradleException @@ -204,6 +205,18 @@ class ProfilerTestPlugin : Plugin { // Environment variables (explicit for consistency across both paths) val envVars = buildMap { + // Turn silent glibc heap corruption (e.g. a use-after-free write into a + // freed chunk) into an immediate, attributable SIGABRT instead of a crash + // much later in an unrelated allocation. Only meaningful against glibc's + // own malloc: skip on musl (no MALLOC_CHECK_ support) and skip whenever a + // sanitizer/allocator already replaces malloc via LD_PRELOAD. + if (PlatformUtils.currentPlatform == Platform.LINUX && + !PlatformUtils.isMusl() && + !testEnv.containsKey("LD_PRELOAD") + ) { + put("MALLOC_CHECK_", "3") + put("MALLOC_PERTURB_", (1..255).random().toString()) + } putAll(testEnv) put("DDPROF_TEST_DISABLE_RATE_LIMIT", "1") put("CI", (project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false).toString()) diff --git a/ddprof-stresstest/src/chaos/README.md b/ddprof-stresstest/src/chaos/README.md index 5dfcabb707..2c72a4eba0 100644 --- a/ddprof-stresstest/src/chaos/README.md +++ b/ddprof-stresstest/src/chaos/README.md @@ -15,6 +15,7 @@ the runner script. | `classloader-churn` | class unload racing stack walk, `CodeCache`/`Symbols` invalidation | | `alloc-storm` | Java alloc engine + GOT-patched libc malloc/free | | `trace-context` | `setContext`/`clearContext` racing signals, span ID propagation | +| `vthread-context-cascade` | tracer-style context propagate/activate/restore across fan-out cascades of short-lived virtual threads, racing carrier-pin churn to trigger the `ContextStorageMode.THREAD` stale-carrier `DirectByteBuffer` use-after-free | ## Deferred diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java index 9b2cccc432..1594b0ad66 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java @@ -72,6 +72,8 @@ private static Antagonist create(String name) { return new AllocStormAntagonist(); case "vthread-churn": return new VirtualThreadChurnAntagonist(); + case "vthread-context-cascade": + return new VirtualThreadContextCascadeAntagonist(); case "classloader-churn": return new ClassLoaderChurnAntagonist(); case "trace-context": diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java new file mode 100644 index 0000000000..a98864e399 --- /dev/null +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java @@ -0,0 +1,470 @@ +/* + * Copyright 2026, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.datadoghq.profiler.chaos; + +import com.datadoghq.profiler.ContextSetter; +import com.datadoghq.profiler.JavaProfiler; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Simulates a tracer's context propagation across a cascade of short-lived virtual threads, and + * deliberately races the carrier-thread churn needed to expose a stale-carrier use-after-free. + * + *

Each "chain-of-operations" (a synthetic distributed trace) is minted once with a stable + * {@code traceId}/{@code localRootSpanId} and an initial set of custom context attributes — this + * is the {@link ContextFrame} that a real tracer would carry inside a {@code Continuation} or + * wrapped {@code Runnable} across an async boundary. Every hop (a new virtual thread) then + * mimics {@code AgentScope} semantics explicitly: + * + *

    + *
  1. on entry, activate the propagated frame — the thread's first action ({@code + * setContext} + replaying every custom attribute), pushing it onto a locally tracked + * context stack; + *
  2. run a handful of synchronous nested "sub-operations", each activating a new child frame + * (new span id, rewritten attributes — the high context-change-rate dimension) and pushing + * it, sleeping briefly to force the virtual thread off its current carrier before writing + * again, then unwind by popping and re-activating the frame underneath — i.e. restoring + * the previously active context, exactly as {@code Scope.close()} does; + *
  3. fan out to child virtual threads, handing them the current frame to propagate (the chain + * continues on a new thread/carrier); + *
  4. on exit, pop the frame it activated on entry and restore whatever was logically + * "underneath" it (empty, since this is a fresh virtual thread). + *
+ * + *

A second, independent driver ({@link #pinningChurnLoop()}) continuously pins short-lived + * virtual threads to their carrier (a {@code synchronized} block held across a blocking sleep) + * across a rotating set of monitors, forcing the JDK's virtual-thread scheduler to spin up extra + * compensating carrier platform threads while the pins are held. + * + *

Targets: {@code com.datadoghq.profiler.OtelContextStorage}'s {@code ContextStorageMode.THREAD} + * fallback (the default on JDK 21+ without {@code --add-exports + * java.base/jdk.internal.misc=ALL-UNNAMED}). Under that mode the {@code DirectByteBuffer} conduit + * for a virtual thread's OTEL context is cached in a plain {@code ThreadLocal} the first time the + * thread writes context, bound to whichever carrier happened to be mounted at that moment; the + * conduit wraps memory embedded directly inside that carrier's native {@code ProfiledThread} + * (see {@code threadLocalData.h}), which is {@code delete}d — a real {@code free()} — the moment + * the carrier's JVMTI {@code ThreadEnd} fires. The nested-op loop in {@link #runCascadeNode} races + * this cheaply: write, force a short real unmount via {@link #forceUnmount}, write again — if the + * virtual thread was remounted elsewhere, the second write lands on the stale buffer. + * + *

Catching genuine memory reuse (not just misattribution) needs the original carrier + * to actually be torn down first. The JDK's default virtual-thread scheduler is a + * {@code ForkJoinPool} built with {@code corePoolSize=0} and a 30s keep-alive (see + * {@code java.lang.VirtualThread#createDefaultScheduler()}) — every carrier, not just + * compensating ones, is reaped after 30s with nothing runnable. {@link #driverLoop} therefore + * runs a duty cycle: a short active burst (cascades, pin churn, and a batch of long-sleeping + * {@link #runStaleBufferRacer} threads) followed by a quiet phase with no new work, long enough + * for carriers to actually go idle and exit. Each racer writes context once, sleeps past the + * 30s keep-alive, then writes again on the far side of the quiet phase — by then its original + * carrier is very likely gone, and under sustained allocation churn from the rest of the harness + * its freed {@code ProfiledThread} memory may already belong to a different, live carrier. + * + *

Reflectively detects {@code Thread.ofVirtual()} (Java 21+); gracefully no-ops on older + * runtimes. Live virtual thread count is bounded by semaphore permit pools so cascades, pinning + * churn, and stale-buffer racers cannot runaway the JVM. + */ +public final class VirtualThreadContextCascadeAntagonist implements Antagonist { + + private static final Method OF_VIRTUAL = resolveOfVirtual(); + private static final Method BUILDER_START = resolveBuilderStart(); + + private static final String[] ATTR_NAMES = { + "http.route", "http.method", "http.status", "db.operation", "rpc.service", + "cache.hit", "queue.name", "tenant.id" + }; + + private static final int FAN_OUT = 3; + private static final int MAX_DEPTH = 4; + private static final int NESTED_OPS_PER_HOP = 3; + private static final int MAX_LIVE_VTHREADS = 512; + private static final int ROOT_BATCH = 16; + + // Carrier-pinning churn: rotating monitors held across a blocking sleep to force the + // scheduler to spin up (and later reap) compensating carrier platform threads. + private static final int PIN_LOCK_COUNT = 32; + private static final int PIN_BATCH = 24; + private static final int MAX_PIN_VTHREADS = 128; + private static final Object[] PIN_LOCKS = new Object[PIN_LOCK_COUNT]; + + static { + for (int i = 0; i < PIN_LOCK_COUNT; i++) { + PIN_LOCKS[i] = new Object(); + } + } + + // Duty cycle driving genuine carrier teardown: an active burst of cascade/pin-churn work, + // then a quiet phase long enough to clear the JDK's default 30s carrier keep-alive so idle + // carriers actually exit before the stale-buffer racers spawned in the burst wake back up. + private static final long ACTIVE_PHASE_MILLIS = 8_000L; + private static final long QUIET_PHASE_MILLIS = 40_000L; + private static final long STALE_RACER_SLEEP_MILLIS = QUIET_PHASE_MILLIS + 5_000L; + private static final int STALE_RACER_BATCH = 16; + private static final int MAX_STALE_RACERS = 256; + + /** No trace active — what a freshly minted virtual thread starts underneath. */ + private static final ContextFrame EMPTY = new ContextFrame(0, 0, 0, 0, new String[ATTR_NAMES.length]); + + private final Semaphore liveVthreads = new Semaphore(MAX_LIVE_VTHREADS); + private final Semaphore livePinVthreads = new Semaphore(MAX_PIN_VTHREADS); + private final Semaphore liveStaleRacers = new Semaphore(MAX_STALE_RACERS); + private volatile boolean running; + private volatile boolean activePhase = true; + private Thread driver; + private Thread pinningDriver; + private final AtomicLong sink = new AtomicLong(); + + /** An immutable snapshot of everything a tracer would propagate for one span activation. */ + private static final class ContextFrame { + final long localRootSpanId; + final long spanId; + final long traceIdHigh; + final long traceIdLow; + final String[] attrValues; // parallel to ATTR_NAMES; null entry means "unset" + + ContextFrame(long localRootSpanId, long spanId, long traceIdHigh, long traceIdLow, String[] attrValues) { + this.localRootSpanId = localRootSpanId; + this.spanId = spanId; + this.traceIdHigh = traceIdHigh; + this.traceIdLow = traceIdLow; + this.attrValues = attrValues; + } + + /** A child span within the same trace, with every custom attribute rewritten. */ + ContextFrame child(long seed, long salt) { + long childSpanId = spanId * 1103515245L + 12345L + salt; + String[] values = new String[ATTR_NAMES.length]; + long r = seed; + for (int i = 0; i < values.length; i++) { + r = r * 6364136223846793005L + 1442695040888963407L; + values[i] = ATTR_NAMES[i] + "-" + (r & 0xffff); + } + return new ContextFrame(localRootSpanId, childSpanId, traceIdHigh, traceIdLow, values); + } + } + + @Override + public String name() { + return "vthread-context-cascade"; + } + + @Override + public void start() { + running = true; + driver = new Thread(this::driverLoop, "chaos-vthread-cascade-driver"); + driver.setDaemon(true); + driver.start(); + pinningDriver = new Thread(this::pinningChurnLoop, "chaos-vthread-cascade-pin-driver"); + pinningDriver.setDaemon(true); + pinningDriver.start(); + } + + @Override + public void stopGracefully(Duration timeout) { + running = false; + try { + driver.join(timeout.toMillis()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + pinningDriver.join(timeout.toMillis()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + // Best-effort drain of in-flight cascades so the JVM doesn't exit mid-fan-out. + try { + liveVthreads.tryAcquire(MAX_LIVE_VTHREADS, timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + livePinVthreads.tryAcquire(MAX_PIN_VTHREADS, timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + // Stale-buffer racers sleep well past stopGracefully's own timeout budget, so this is + // best-effort only — outstanding racers are daemon threads and die with the JVM. + try { + liveStaleRacers.tryAcquire(MAX_STALE_RACERS, timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** + * Independently pins short-lived virtual threads to their carrier and releases them again, + * forcing the scheduler to grow and shrink its compensating carrier pool while cascade nodes + * are racing carrier migration on the other driver. + */ + private void pinningChurnLoop() { + if (OF_VIRTUAL == null || BUILDER_START == null) { + return; + } + while (running) { + if (!activePhase) { + // Quiet phase: stay out of the way so idle carriers can actually be reaped. + try { + Thread.sleep(50L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + continue; + } + for (int i = 0; i < PIN_BATCH && running && activePhase; i++) { + if (!livePinVthreads.tryAcquire()) { + continue; + } + final Object lock = PIN_LOCKS[ThreadLocalRandom.current().nextInt(PIN_LOCK_COUNT)]; + try { + Object builder = OF_VIRTUAL.invoke(null); + BUILDER_START.invoke( + builder, + (Runnable) + () -> { + try { + synchronized (lock) { + // Sleeping while holding a monitor pins the + // carrier: the scheduler must compensate with an + // extra platform thread for other runnable + // virtual threads, then reap it once idle. + Thread.sleep(2L); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + livePinVthreads.release(); + } + }); + } catch (Throwable t) { + livePinVthreads.release(); + } + } + try { + Thread.sleep(1L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + private void driverLoop() { + if (OF_VIRTUAL == null || BUILDER_START == null) { + System.out.println("[chaos] vthread-context-cascade: skipping (Thread.ofVirtual not available)"); + return; + } + JavaProfiler profiler; + try { + profiler = JavaProfiler.getInstance(); + } catch (Exception e) { + System.err.println("[chaos] vthread-context-cascade: failed to get profiler: " + e); + return; + } + while (running) { + // Active phase: mint cascade work and a batch of stale-buffer racers that will wake + // up on the far side of the upcoming quiet phase. + activePhase = true; + spawnStaleBufferRacers(profiler); + long activeDeadlineNanos = System.nanoTime() + Duration.ofMillis(ACTIVE_PHASE_MILLIS).toNanos(); + while (running && System.nanoTime() < activeDeadlineNanos) { + for (int i = 0; i < ROOT_BATCH && running; i++) { + // Mint one chain-of-operations: a fresh synthetic trace with its own + // root/span/trace ids and an initial set of custom context attributes. + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + long localRootSpanId = rnd.nextLong() & 0x7fffffffffffffffL; + ContextFrame root = new ContextFrame(localRootSpanId, localRootSpanId, 0, rnd.nextLong(), new String[ATTR_NAMES.length]) + .child(rnd.nextLong(), 0); + spawnCascadeNode(profiler, root, 0); + } + try { + Thread.sleep(1L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + if (!running) { + return; + } + // Quiet phase: mint nothing new so short-lived virtual threads drain and carriers + // can actually sit idle past the scheduler's 30s keep-alive and get reaped. + activePhase = false; + try { + Thread.sleep(QUIET_PHASE_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + /** + * Spawns a batch of stale-buffer racers timed to wake up just after the current quiet phase + * ends, when fresh work is remounting virtual threads onto carriers the scheduler has likely + * just (re)created. + */ + private void spawnStaleBufferRacers(JavaProfiler profiler) { + for (int i = 0; i < STALE_RACER_BATCH && running; i++) { + if (!liveStaleRacers.tryAcquire()) { + continue; + } + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + long localRootSpanId = rnd.nextLong() & 0x7fffffffffffffffL; + ContextFrame before = new ContextFrame(localRootSpanId, localRootSpanId, 0, rnd.nextLong(), new String[ATTR_NAMES.length]) + .child(rnd.nextLong(), 0); + ContextFrame after = before.child(rnd.nextLong(), 1L); + try { + Object builder = OF_VIRTUAL.invoke(null); + BUILDER_START.invoke(builder, (Runnable) () -> runStaleBufferRacer(profiler, before, after)); + } catch (Throwable t) { + liveStaleRacers.release(); + } + } + } + + /** + * Writes context once, sleeps past the scheduler's default 30s carrier keep-alive, then + * writes again. Under {@code ContextStorageMode.THREAD} the second write reuses whatever + * {@code DirectByteBuffer} was cached on the first — if the original carrier has since been + * reaped and its {@code ProfiledThread} memory reused, this is a genuine use-after-free. + */ + private void runStaleBufferRacer(JavaProfiler profiler, ContextFrame before, ContextFrame after) { + try { + ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList(ATTR_NAMES)); + activate(profiler, contextSetter, before); + Thread.sleep(STALE_RACER_SLEEP_MILLIS); + activate(profiler, contextSetter, after); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + liveStaleRacers.release(); + } + } + + /** Spawns one virtual thread to run a cascade node, respecting the live-thread budget. */ + private void spawnCascadeNode(JavaProfiler profiler, ContextFrame propagated, int depth) { + if (!running || !liveVthreads.tryAcquire()) { + return; + } + try { + Object builder = OF_VIRTUAL.invoke(null); + BUILDER_START.invoke( + builder, + (Runnable) + () -> { + try { + runCascadeNode(profiler, propagated, depth); + } finally { + liveVthreads.release(); + } + }); + } catch (Throwable t) { + liveVthreads.release(); + } + } + + private void runCascadeNode(JavaProfiler profiler, ContextFrame propagated, int depth) { + ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList(ATTR_NAMES)); + Deque stack = new ArrayDeque<>(); + + // First operation on this thread: manually activate the propagated context. + activate(profiler, contextSetter, propagated); + stack.push(propagated); + + long r = propagated.traceIdLow ^ ((long) depth << 48); + for (int op = 0; op < NESTED_OPS_PER_HOP && running; op++) { + r = r * 1103515245L + 12345L; + ContextFrame nested = stack.peek().child(r, op + 1L); + activate(profiler, contextSetter, nested); + stack.push(nested); + r = forceUnmount(r); + // Write again immediately on resumption: if the scheduler remounted this virtual + // thread on a different carrier during the sleep above, this activation races the + // stale-carrier DirectByteBuffer use-after-free under ContextStorageMode.THREAD. + activate(profiler, contextSetter, stack.peek()); + } + sink.addAndGet(r); + + // Unwind the nested sub-operations, restoring the previously active context at each step. + while (stack.size() > 1) { + stack.pop(); + activate(profiler, contextSetter, stack.peek()); + } + + // Propagate the chain to the next hop before this thread's own context is torn down. + if (depth < MAX_DEPTH && running) { + ContextFrame current = stack.peek(); + for (int i = 0; i < FAN_OUT; i++) { + spawnCascadeNode(profiler, current.child(r ^ ((long) i << 40), i + 1L), depth + 1); + } + } + + // Last operation on this thread: pop what we activated on entry and restore what was + // logically underneath it (nothing — this virtual thread never had an outer scope). + stack.pop(); + activate(profiler, contextSetter, EMPTY); + } + + private static void activate(JavaProfiler profiler, ContextSetter contextSetter, ContextFrame frame) { + profiler.setContext(frame.localRootSpanId, frame.spanId, frame.traceIdHigh, frame.traceIdLow); + for (int i = 0; i < ATTR_NAMES.length; i++) { + String value = frame.attrValues[i]; + if (value != null) { + contextSetter.setContextValue(i, value); + } else { + contextSetter.clearContextValue(i); + } + } + } + + /** + * Burns some CPU (a stand-in for real work) then blocks briefly, forcing the virtual thread + * to genuinely unmount. On resumption it may be remounted on a different carrier than the + * one it was on when the context before this call was written. + */ + private static long forceUnmount(long seed) { + long r = seed; + for (int i = 0; i < 2_000; i++) { + r = r * 6364136223846793005L + 1442695040888963407L; + } + try { + Thread.sleep(1L + (r & 1)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return r; + } + + private static Method resolveOfVirtual() { + try { + return Thread.class.getMethod("ofVirtual"); + } catch (NoSuchMethodException e) { + return null; + } + } + + private static Method resolveBuilderStart() { + try { + Class builder = Class.forName("java.lang.Thread$Builder"); + return builder.getMethod("start", Runnable.class); + } catch (ClassNotFoundException | NoSuchMethodException e) { + return null; + } + } +} From 0b03bba2fea52599bd93dff63db6db240f7dd339 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 8 Jul 2026 18:13:25 +0200 Subject: [PATCH 02/10] Exclude J9 from MALLOC_CHECK_ default test env J9 hits a pre-existing, unrelated glibc heap-corruption bug that MALLOC_CHECK_ now surfaces as a hard abort; gate it off until that's investigated separately. --- .../kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt index bae6943563..0b8f89c4c5 100644 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt @@ -208,11 +208,14 @@ class ProfilerTestPlugin : Plugin { // Turn silent glibc heap corruption (e.g. a use-after-free write into a // freed chunk) into an immediate, attributable SIGABRT instead of a crash // much later in an unrelated allocation. Only meaningful against glibc's - // own malloc: skip on musl (no MALLOC_CHECK_ support) and skip whenever a - // sanitizer/allocator already replaces malloc via LD_PRELOAD. + // own malloc: skip on musl (no MALLOC_CHECK_ support), skip whenever a + // sanitizer/allocator already replaces malloc via LD_PRELOAD, and skip on + // J9 (its own allocator usage trips a pre-existing, unrelated heap + // corruption that needs separate investigation). if (PlatformUtils.currentPlatform == Platform.LINUX && !PlatformUtils.isMusl() && - !testEnv.containsKey("LD_PRELOAD") + !testEnv.containsKey("LD_PRELOAD") && + !PlatformUtils.isTestJvmJ9() ) { put("MALLOC_CHECK_", "3") put("MALLOC_PERTURB_", (1..255).random().toString()) From 4dce8c62be269dc924b6debfa1bda51aa10af094 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 8 Jul 2026 19:33:36 +0200 Subject: [PATCH 03/10] Reference PROF-15360 in J9 MALLOC_CHECK_ exclusion comment --- .../main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt index 0b8f89c4c5..91254719db 100644 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt @@ -211,7 +211,7 @@ class ProfilerTestPlugin : Plugin { // own malloc: skip on musl (no MALLOC_CHECK_ support), skip whenever a // sanitizer/allocator already replaces malloc via LD_PRELOAD, and skip on // J9 (its own allocator usage trips a pre-existing, unrelated heap - // corruption that needs separate investigation). + // corruption that needs separate investigation — see PROF-15360). if (PlatformUtils.currentPlatform == Platform.LINUX && !PlatformUtils.isMusl() && !testEnv.containsKey("LD_PRELOAD") && From 4e3c9cadc1551742e6a1fc50410fad13dd47f38f Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 9 Jul 2026 09:53:51 +0200 Subject: [PATCH 04/10] Fix isRawPointer() false-positive on unencoded ASGCT BCIs (PROF-15364) isRawPointer() only checked RAW_POINTER_MASK, letting raw J9 ASGCT BCIs that never went through encode() misroute into JVMSupport::resolve(), which asserts false for non-HotSpot VMs. Require ENCODED_MASK too. --- ddprof-lib/src/main/cpp/frame.h | 6 +++++- ddprof-lib/src/test/cpp/frame_ut.cpp | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/frame.h b/ddprof-lib/src/main/cpp/frame.h index 574ca41a11..c1c30633c0 100644 --- a/ddprof-lib/src/main/cpp/frame.h +++ b/ddprof-lib/src/main/cpp/frame.h @@ -79,8 +79,12 @@ class FrameType { return (FrameTypeId)((bci >> TYPE_SHIFT) & FRAME_TYPE_MASK); } + // RAW_POINTER_MASK is only ever set by encode() (rawPointer=true), which + // unconditionally also sets ENCODED_MASK. Requiring ENCODED_MASK here + // prevents a raw, unencoded ASGCT BCI (never passed through encode()) + // from false-positiving just because it happens to have bit 30 set. static inline bool isRawPointer(int bci) { - return bci > 0 && (bci & RAW_POINTER_MASK) != 0; + return bci > 0 && (bci & ENCODED_MASK) != 0 && (bci & RAW_POINTER_MASK) != 0; } }; diff --git a/ddprof-lib/src/test/cpp/frame_ut.cpp b/ddprof-lib/src/test/cpp/frame_ut.cpp index c31d18f770..e1de762c58 100644 --- a/ddprof-lib/src/test/cpp/frame_ut.cpp +++ b/ddprof-lib/src/test/cpp/frame_ut.cpp @@ -154,3 +154,15 @@ TEST(FrameTypeIsRawPointerTest, FalseWithOnlyBit30SetOnNegative) { EXPECT_LT(negativeWithBit30, 0); EXPECT_FALSE(FrameType::isRawPointer(negativeWithBit30)); } + +TEST(FrameTypeIsRawPointerTest, FalseForRawAsgctBciWithBit30SetButNoEncodedMarker) { + // Regression test: a raw, unencoded ASGCT BCI (bit 20 / ENCODED_MASK not set) + // that incidentally has bit 30 set must NOT be reported as a raw pointer. + // Such values occur on non-HotSpot VMs (e.g. J9), which never call encode() + // and therefore never set ENCODED_MASK; only encode(..., rawPointer=true) + // (HotSpot-only, per its own assert) is allowed to set RAW_POINTER_MASK. + int rawAsgctBciWithBit30 = 1 << 30; + EXPECT_FALSE(FrameType::isRawPointer(rawAsgctBciWithBit30)) + << "isRawPointer() must require ENCODED_MASK (bit 20) before trusting bit 30, " + << "otherwise raw ASGCT BCIs that never went through encode() can false-positive"; +} From d17272e7f39dd9d1760ae10b1b4c9fa35240611c Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 9 Jul 2026 10:12:30 +0200 Subject: [PATCH 05/10] chaos: wire vthread-context-cascade into ANTAGONISTS lists It was registered in Main.java's antagonist factory but never added to either config's ANTAGONISTS string, so it silently never ran in CI. --- .gitlab/reliability/chaos_check.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/reliability/chaos_check.sh b/.gitlab/reliability/chaos_check.sh index 483bd06c3a..2ea2be7475 100755 --- a/.gitlab/reliability/chaos_check.sh +++ b/.gitlab/reliability/chaos_check.sh @@ -103,12 +103,12 @@ case $CONFIG in echo "Running with profiler only" ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" # @Trace is a no-op without the tracer, so trace-context is excluded here. - ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,vthread-context-cascade,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" ;; profiler+tracer) echo "Running with profiler and tracer" ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" - ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,vthread-context-cascade,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" ;; *) echo "Unknown configuration: $CONFIG" From e1dd0fb1c7b9f39a69937869c5ae7a2bf940711a Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 9 Jul 2026 10:16:55 +0200 Subject: [PATCH 06/10] ci(chaos): enable NativeMemoryTracking=summary for OOM diagnosis --- .gitlab/reliability/chaos_check.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/reliability/chaos_check.sh b/.gitlab/reliability/chaos_check.sh index 2ea2be7475..e73c70200f 100755 --- a/.gitlab/reliability/chaos_check.sh +++ b/.gitlab/reliability/chaos_check.sh @@ -156,6 +156,7 @@ java -javaagent:${PATCHED_AGENT} \ -Ddd.service=java-profiler-chaos \ -Xmx2g -Xms2g \ -XX:MaxMetaspaceSize=384m \ + -XX:NativeMemoryTracking=summary \ -XX:ErrorFile=${HERE}/../../hs_err.log \ -XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \ -jar ${CHAOS_JAR} \ From 3673735858b122e1257057ed0b3297eafc6686bd Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 9 Jul 2026 13:28:53 +0200 Subject: [PATCH 07/10] chaos: fix stale-racer wake timing and stopGracefully timeout budget Include ACTIVE_PHASE_MILLIS in STALE_RACER_SLEEP_MILLIS so racers wake as the next active phase begins instead of ~3s early. Bound stopGracefully's total wait to the caller's timeout via a shared deadline instead of giving each join/tryAcquire the full budget independently. --- ...VirtualThreadContextCascadeAntagonist.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java index a98864e399..22319558b1 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java @@ -112,7 +112,7 @@ public final class VirtualThreadContextCascadeAntagonist implements Antagonist { // carriers actually exit before the stale-buffer racers spawned in the burst wake back up. private static final long ACTIVE_PHASE_MILLIS = 8_000L; private static final long QUIET_PHASE_MILLIS = 40_000L; - private static final long STALE_RACER_SLEEP_MILLIS = QUIET_PHASE_MILLIS + 5_000L; + private static final long STALE_RACER_SLEEP_MILLIS = ACTIVE_PHASE_MILLIS + QUIET_PHASE_MILLIS + 5_000L; private static final int STALE_RACER_BATCH = 16; private static final int MAX_STALE_RACERS = 256; @@ -176,36 +176,43 @@ public void start() { @Override public void stopGracefully(Duration timeout) { running = false; + long deadlineNanos = System.nanoTime() + timeout.toNanos(); try { - driver.join(timeout.toMillis()); + driver.join(remainingMillis(deadlineNanos)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } try { - pinningDriver.join(timeout.toMillis()); + pinningDriver.join(remainingMillis(deadlineNanos)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } // Best-effort drain of in-flight cascades so the JVM doesn't exit mid-fan-out. try { - liveVthreads.tryAcquire(MAX_LIVE_VTHREADS, timeout.toMillis(), TimeUnit.MILLISECONDS); + liveVthreads.tryAcquire(MAX_LIVE_VTHREADS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } try { - livePinVthreads.tryAcquire(MAX_PIN_VTHREADS, timeout.toMillis(), TimeUnit.MILLISECONDS); + livePinVthreads.tryAcquire(MAX_PIN_VTHREADS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } // Stale-buffer racers sleep well past stopGracefully's own timeout budget, so this is // best-effort only — outstanding racers are daemon threads and die with the JVM. try { - liveStaleRacers.tryAcquire(MAX_STALE_RACERS, timeout.toMillis(), TimeUnit.MILLISECONDS); + liveStaleRacers.tryAcquire(MAX_STALE_RACERS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } + /** Milliseconds remaining until {@code deadlineNanos}, clamped to zero (never negative). */ + private static long remainingMillis(long deadlineNanos) { + long remainingNanos = deadlineNanos - System.nanoTime(); + return remainingNanos <= 0L ? 0L : TimeUnit.NANOSECONDS.toMillis(remainingNanos); + } + /** * Independently pins short-lived virtual threads to their carrier and releases them again, * forcing the scheduler to grow and shrink its compensating carrier pool while cascade nodes From 17c3771ff36afdb87e5c7eba21d44f872a44271c Mon Sep 17 00:00:00 2001 From: "jaroslav.bachorik" Date: Thu, 9 Jul 2026 18:10:20 +0000 Subject: [PATCH 08/10] chaos: fix vthread-context-cascade NoClassDefFoundError by driving context via @Trace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit driverLoop() called JavaProfiler.getInstance() directly, but JavaProfiler is a chaosCompileOnly dependency never bundled into chaos.jar and not visible to the app classloader (the agent's copy is shaded/relocated). The resulting NoClassDefFoundError is an Error, not an Exception, so it escaped the catch (Exception e) block and silently killed the driver thread — leaving activePhase stuck true and letting the sibling pinningChurnLoop run full-throttle for the rest of the test instead of its intended active/quiet duty cycle, which is the likely driver behind the OOM seen in reliability-chaos-aarch64 [profiler, gmalloc, 21.0.3-tem]. Rewrite the antagonist to only touch the tracer through @Trace-annotated methods (the same proven-safe dd-trace-api compileOnly pattern already used by TraceContextAntagonist), letting the real tracer and its virtual-thread instrumentation drive setContext/clearContext instead of calling com.datadoghq.profiler.* directly. Drops the custom-attribute/baggage simulation in favor of nested @Trace hops plus forceUnmount and stale-buffer racer threads that still race OtelContextStorage's thread-scoped fallback across carrier churn. Verified with a 90s live run (JDK 21.0.11, profiler+tracer, gmalloc): no NoClassDefFoundError, no exceptions, completed cleanly with RC=0. Environment: Datadog workspace Co-Authored-By: Claude Sonnet 5 --- ...VirtualThreadContextCascadeAntagonist.java | 217 ++++++------------ 1 file changed, 74 insertions(+), 143 deletions(-) diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java index 22319558b1..d4f2aa6309 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java @@ -9,13 +9,9 @@ */ package com.datadoghq.profiler.chaos; -import com.datadoghq.profiler.ContextSetter; -import com.datadoghq.profiler.JavaProfiler; +import datadog.trace.api.Trace; import java.lang.reflect.Method; import java.time.Duration; -import java.util.ArrayDeque; -import java.util.Arrays; -import java.util.Deque; import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @@ -25,26 +21,23 @@ * Simulates a tracer's context propagation across a cascade of short-lived virtual threads, and * deliberately races the carrier-thread churn needed to expose a stale-carrier use-after-free. * - *

Each "chain-of-operations" (a synthetic distributed trace) is minted once with a stable - * {@code traceId}/{@code localRootSpanId} and an initial set of custom context attributes — this - * is the {@link ContextFrame} that a real tracer would carry inside a {@code Continuation} or - * wrapped {@code Runnable} across an async boundary. Every hop (a new virtual thread) then - * mimics {@code AgentScope} semantics explicitly: + *

Unlike a hand-rolled context simulation, every context write here is driven by the real + * Datadog tracer: {@link #runCascadeNode}/{@link #nestedOp}/{@link #staleRacerBefore}/{@link + * #staleRacerAfter} are {@link Trace}-annotated methods. Under a {@code dd-java-agent}- + * instrumented JVM the tracer intercepts each one, activating a real span on entry ({@code + * JavaProfiler.setContext}) and deactivating it on exit ({@code clearContext}) — exactly what a + * genuinely-instrumented application does. {@code dd-trace-api} is a {@code compileOnly} + * dependency (see {@code TraceContextAntagonist}): at runtime the patched {@code dd-java-agent} + * provides the (relocated) classes and intercepts the annotation, so this antagonist never touches + * {@code com.datadoghq.profiler.*} directly. * - *

    - *
  1. on entry, activate the propagated frame — the thread's first action ({@code - * setContext} + replaying every custom attribute), pushing it onto a locally tracked - * context stack; - *
  2. run a handful of synchronous nested "sub-operations", each activating a new child frame - * (new span id, rewritten attributes — the high context-change-rate dimension) and pushing - * it, sleeping briefly to force the virtual thread off its current carrier before writing - * again, then unwind by popping and re-activating the frame underneath — i.e. restoring - * the previously active context, exactly as {@code Scope.close()} does; - *
  3. fan out to child virtual threads, handing them the current frame to propagate (the chain - * continues on a new thread/carrier); - *
  4. on exit, pop the frame it activated on entry and restore whatever was logically - * "underneath" it (empty, since this is a fresh virtual thread). - *
+ *

Each cascade node ({@link #runCascadeNode}) runs a handful of nested sub-operations, each + * activating a new child span and sleeping briefly ({@link #forceUnmount}) to force the virtual + * thread off its current carrier before the span's exit (and the next nested entry) write context + * again — the high context-change-rate dimension. It then fans out to child virtual threads while + * its own span is still active, handing the tracer's own async/virtual-thread instrumentation the + * job of propagating that span as parent context onto the next hop (the chain continues on a new + * thread/carrier). On exit, the span's close restores whatever was logically active underneath it. * *

A second, independent driver ({@link #pinningChurnLoop()}) continuously pins short-lived * virtual threads to their carrier (a {@code synchronized} block held across a blocking sleep) @@ -54,13 +47,15 @@ *

Targets: {@code com.datadoghq.profiler.OtelContextStorage}'s {@code ContextStorageMode.THREAD} * fallback (the default on JDK 21+ without {@code --add-exports * java.base/jdk.internal.misc=ALL-UNNAMED}). Under that mode the {@code DirectByteBuffer} conduit - * for a virtual thread's OTEL context is cached in a plain {@code ThreadLocal} the first time the - * thread writes context, bound to whichever carrier happened to be mounted at that moment; the + * for a virtual thread's OTEL context (which backs both the span id fields written by {@code + * setContext} and any custom attributes) is cached in a plain {@code ThreadLocal} the first time + * the thread writes context, bound to whichever carrier happened to be mounted at that moment; the * conduit wraps memory embedded directly inside that carrier's native {@code ProfiledThread} * (see {@code threadLocalData.h}), which is {@code delete}d — a real {@code free()} — the moment * the carrier's JVMTI {@code ThreadEnd} fires. The nested-op loop in {@link #runCascadeNode} races - * this cheaply: write, force a short real unmount via {@link #forceUnmount}, write again — if the - * virtual thread was remounted elsewhere, the second write lands on the stale buffer. + * this cheaply: a nested span's entry writes, {@link #forceUnmount} forces a short real unmount, + * then the span's exit (or the next nested entry) writes again — if the virtual thread was + * remounted elsewhere, that second write reuses the (possibly now-stale) cached conduit. * *

Catching genuine memory reuse (not just misattribution) needs the original carrier * to actually be torn down first. The JDK's default virtual-thread scheduler is a @@ -69,10 +64,11 @@ * compensating ones, is reaped after 30s with nothing runnable. {@link #driverLoop} therefore * runs a duty cycle: a short active burst (cascades, pin churn, and a batch of long-sleeping * {@link #runStaleBufferRacer} threads) followed by a quiet phase with no new work, long enough - * for carriers to actually go idle and exit. Each racer writes context once, sleeps past the - * 30s keep-alive, then writes again on the far side of the quiet phase — by then its original - * carrier is very likely gone, and under sustained allocation churn from the rest of the harness - * its freed {@code ProfiledThread} memory may already belong to a different, live carrier. + * for carriers to actually go idle and exit. Each racer writes context once ({@link + * #staleRacerBefore}), sleeps past the 30s keep-alive, then writes again ({@link + * #staleRacerAfter}) on the far side of the quiet phase — by then its original carrier is very + * likely gone, and under sustained allocation churn from the rest of the harness its freed {@code + * ProfiledThread} memory may already belong to a different, live carrier. * *

Reflectively detects {@code Thread.ofVirtual()} (Java 21+); gracefully no-ops on older * runtimes. Live virtual thread count is bounded by semaphore permit pools so cascades, pinning @@ -83,11 +79,6 @@ public final class VirtualThreadContextCascadeAntagonist implements Antagonist { private static final Method OF_VIRTUAL = resolveOfVirtual(); private static final Method BUILDER_START = resolveBuilderStart(); - private static final String[] ATTR_NAMES = { - "http.route", "http.method", "http.status", "db.operation", "rpc.service", - "cache.hit", "queue.name", "tenant.id" - }; - private static final int FAN_OUT = 3; private static final int MAX_DEPTH = 4; private static final int NESTED_OPS_PER_HOP = 3; @@ -116,9 +107,6 @@ public final class VirtualThreadContextCascadeAntagonist implements Antagonist { private static final int STALE_RACER_BATCH = 16; private static final int MAX_STALE_RACERS = 256; - /** No trace active — what a freshly minted virtual thread starts underneath. */ - private static final ContextFrame EMPTY = new ContextFrame(0, 0, 0, 0, new String[ATTR_NAMES.length]); - private final Semaphore liveVthreads = new Semaphore(MAX_LIVE_VTHREADS); private final Semaphore livePinVthreads = new Semaphore(MAX_PIN_VTHREADS); private final Semaphore liveStaleRacers = new Semaphore(MAX_STALE_RACERS); @@ -128,35 +116,6 @@ public final class VirtualThreadContextCascadeAntagonist implements Antagonist { private Thread pinningDriver; private final AtomicLong sink = new AtomicLong(); - /** An immutable snapshot of everything a tracer would propagate for one span activation. */ - private static final class ContextFrame { - final long localRootSpanId; - final long spanId; - final long traceIdHigh; - final long traceIdLow; - final String[] attrValues; // parallel to ATTR_NAMES; null entry means "unset" - - ContextFrame(long localRootSpanId, long spanId, long traceIdHigh, long traceIdLow, String[] attrValues) { - this.localRootSpanId = localRootSpanId; - this.spanId = spanId; - this.traceIdHigh = traceIdHigh; - this.traceIdLow = traceIdLow; - this.attrValues = attrValues; - } - - /** A child span within the same trace, with every custom attribute rewritten. */ - ContextFrame child(long seed, long salt) { - long childSpanId = spanId * 1103515245L + 12345L + salt; - String[] values = new String[ATTR_NAMES.length]; - long r = seed; - for (int i = 0; i < values.length; i++) { - r = r * 6364136223846793005L + 1442695040888963407L; - values[i] = ATTR_NAMES[i] + "-" + (r & 0xffff); - } - return new ContextFrame(localRootSpanId, childSpanId, traceIdHigh, traceIdLow, values); - } - } - @Override public String name() { return "vthread-context-cascade"; @@ -276,28 +235,17 @@ private void driverLoop() { System.out.println("[chaos] vthread-context-cascade: skipping (Thread.ofVirtual not available)"); return; } - JavaProfiler profiler; - try { - profiler = JavaProfiler.getInstance(); - } catch (Exception e) { - System.err.println("[chaos] vthread-context-cascade: failed to get profiler: " + e); - return; - } while (running) { // Active phase: mint cascade work and a batch of stale-buffer racers that will wake // up on the far side of the upcoming quiet phase. activePhase = true; - spawnStaleBufferRacers(profiler); + spawnStaleBufferRacers(); long activeDeadlineNanos = System.nanoTime() + Duration.ofMillis(ACTIVE_PHASE_MILLIS).toNanos(); while (running && System.nanoTime() < activeDeadlineNanos) { for (int i = 0; i < ROOT_BATCH && running; i++) { - // Mint one chain-of-operations: a fresh synthetic trace with its own - // root/span/trace ids and an initial set of custom context attributes. - ThreadLocalRandom rnd = ThreadLocalRandom.current(); - long localRootSpanId = rnd.nextLong() & 0x7fffffffffffffffL; - ContextFrame root = new ContextFrame(localRootSpanId, localRootSpanId, 0, rnd.nextLong(), new String[ATTR_NAMES.length]) - .child(rnd.nextLong(), 0); - spawnCascadeNode(profiler, root, 0); + // Mint one chain-of-operations: a fresh synthetic trace, its root span + // activated by the tracer the moment runCascadeNode is entered. + spawnCascadeNode(0); } try { Thread.sleep(1L); @@ -326,19 +274,14 @@ private void driverLoop() { * ends, when fresh work is remounting virtual threads onto carriers the scheduler has likely * just (re)created. */ - private void spawnStaleBufferRacers(JavaProfiler profiler) { + private void spawnStaleBufferRacers() { for (int i = 0; i < STALE_RACER_BATCH && running; i++) { if (!liveStaleRacers.tryAcquire()) { continue; } - ThreadLocalRandom rnd = ThreadLocalRandom.current(); - long localRootSpanId = rnd.nextLong() & 0x7fffffffffffffffL; - ContextFrame before = new ContextFrame(localRootSpanId, localRootSpanId, 0, rnd.nextLong(), new String[ATTR_NAMES.length]) - .child(rnd.nextLong(), 0); - ContextFrame after = before.child(rnd.nextLong(), 1L); try { Object builder = OF_VIRTUAL.invoke(null); - BUILDER_START.invoke(builder, (Runnable) () -> runStaleBufferRacer(profiler, before, after)); + BUILDER_START.invoke(builder, (Runnable) this::runStaleBufferRacer); } catch (Throwable t) { liveStaleRacers.release(); } @@ -346,17 +289,17 @@ private void spawnStaleBufferRacers(JavaProfiler profiler) { } /** - * Writes context once, sleeps past the scheduler's default 30s carrier keep-alive, then - * writes again. Under {@code ContextStorageMode.THREAD} the second write reuses whatever - * {@code DirectByteBuffer} was cached on the first — if the original carrier has since been - * reaped and its {@code ProfiledThread} memory reused, this is a genuine use-after-free. + * Activates a span once, sleeps past the scheduler's default 30s carrier keep-alive, then + * activates another. Under {@code ContextStorageMode.THREAD} the second activation reuses + * whatever {@code DirectByteBuffer} was cached on the first — if the original carrier has + * since been reaped and its {@code ProfiledThread} memory reused, this is a genuine + * use-after-free. */ - private void runStaleBufferRacer(JavaProfiler profiler, ContextFrame before, ContextFrame after) { + private void runStaleBufferRacer() { try { - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList(ATTR_NAMES)); - activate(profiler, contextSetter, before); + staleRacerBefore(); Thread.sleep(STALE_RACER_SLEEP_MILLIS); - activate(profiler, contextSetter, after); + staleRacerAfter(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } finally { @@ -364,8 +307,17 @@ private void runStaleBufferRacer(JavaProfiler profiler, ContextFrame before, Con } } + @Trace(operationName = "chaos.stale.before", resourceName = "chaos.stale.before") + private void staleRacerBefore() { + // Entry/exit alone is the "write" — the tracer activates/deactivates a span around this + // no-op body, exactly like TraceContextAntagonist's inner()/outer(). + } + + @Trace(operationName = "chaos.stale.after", resourceName = "chaos.stale.after") + private void staleRacerAfter() {} + /** Spawns one virtual thread to run a cascade node, respecting the live-thread budget. */ - private void spawnCascadeNode(JavaProfiler profiler, ContextFrame propagated, int depth) { + private void spawnCascadeNode(int depth) { if (!running || !liveVthreads.tryAcquire()) { return; } @@ -376,7 +328,7 @@ private void spawnCascadeNode(JavaProfiler profiler, ContextFrame propagated, in (Runnable) () -> { try { - runCascadeNode(profiler, propagated, depth); + runCascadeNode(depth); } finally { liveVthreads.release(); } @@ -386,58 +338,37 @@ private void spawnCascadeNode(JavaProfiler profiler, ContextFrame propagated, in } } - private void runCascadeNode(JavaProfiler profiler, ContextFrame propagated, int depth) { - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList(ATTR_NAMES)); - Deque stack = new ArrayDeque<>(); - - // First operation on this thread: manually activate the propagated context. - activate(profiler, contextSetter, propagated); - stack.push(propagated); - - long r = propagated.traceIdLow ^ ((long) depth << 48); + /** + * One hop of the cascade, run as its own {@link Trace}-annotated span on a fresh virtual + * thread. The tracer activates this span on entry and deactivates it on exit; while it's + * active, this hop fans out to child hops on new virtual threads, handing the tracer's own + * virtual-thread instrumentation the job of propagating it as parent context onto each child. + */ + @Trace(operationName = "chaos.cascade.hop", resourceName = "chaos.cascade.hop") + private void runCascadeNode(int depth) { + long r = ThreadLocalRandom.current().nextLong() ^ ((long) depth << 48); for (int op = 0; op < NESTED_OPS_PER_HOP && running; op++) { - r = r * 1103515245L + 12345L; - ContextFrame nested = stack.peek().child(r, op + 1L); - activate(profiler, contextSetter, nested); - stack.push(nested); - r = forceUnmount(r); - // Write again immediately on resumption: if the scheduler remounted this virtual - // thread on a different carrier during the sleep above, this activation races the - // stale-carrier DirectByteBuffer use-after-free under ContextStorageMode.THREAD. - activate(profiler, contextSetter, stack.peek()); + r = nestedOp(r); } sink.addAndGet(r); - // Unwind the nested sub-operations, restoring the previously active context at each step. - while (stack.size() > 1) { - stack.pop(); - activate(profiler, contextSetter, stack.peek()); - } - - // Propagate the chain to the next hop before this thread's own context is torn down. + // Propagate the chain to the next hop before this thread's own span is torn down. if (depth < MAX_DEPTH && running) { - ContextFrame current = stack.peek(); for (int i = 0; i < FAN_OUT; i++) { - spawnCascadeNode(profiler, current.child(r ^ ((long) i << 40), i + 1L), depth + 1); + spawnCascadeNode(depth + 1); } } - - // Last operation on this thread: pop what we activated on entry and restore what was - // logically underneath it (nothing — this virtual thread never had an outer scope). - stack.pop(); - activate(profiler, contextSetter, EMPTY); } - private static void activate(JavaProfiler profiler, ContextSetter contextSetter, ContextFrame frame) { - profiler.setContext(frame.localRootSpanId, frame.spanId, frame.traceIdHigh, frame.traceIdLow); - for (int i = 0; i < ATTR_NAMES.length; i++) { - String value = frame.attrValues[i]; - if (value != null) { - contextSetter.setContextValue(i, value); - } else { - contextSetter.clearContextValue(i); - } - } + /** + * A nested sub-operation within a hop: the tracer activates a child span on entry, {@link + * #forceUnmount} sleeps briefly to force the virtual thread off its current carrier, then the + * span's exit (or the next nested entry) writes context again — racing carrier migration + * against the cached context conduit. + */ + @Trace(operationName = "chaos.cascade.op", resourceName = "chaos.cascade.op") + private long nestedOp(long seed) { + return forceUnmount(seed); } /** From 0bfeed0f8e16f1bfb770f11934487b6f9a6b1155 Mon Sep 17 00:00:00 2001 From: "jaroslav.bachorik" Date: Thu, 9 Jul 2026 18:11:03 +0000 Subject: [PATCH 09/10] chaos: fail fast on missing RUNTIME argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without a RUNTIME arg, chaos_check.sh silently proceeds with an empty value, which becomes --duration s in the chaos harness invocation and fails 15+ seconds later — after the JDK/agent/jar setup — with a NumberFormatException buried in the harness output, reported generically as "FAIL:Chaos harness crashed (RC=1)". Check for a missing RUNTIME right after arg parsing and fail immediately with a clear usage message. Environment: Datadog workspace Co-Authored-By: Claude Sonnet 5 --- .gitlab/reliability/chaos_check.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab/reliability/chaos_check.sh b/.gitlab/reliability/chaos_check.sh index e73c70200f..d4c2b26dc2 100755 --- a/.gitlab/reliability/chaos_check.sh +++ b/.gitlab/reliability/chaos_check.sh @@ -9,6 +9,11 @@ RUNTIME=${1} CONFIG=${2:-profiler+tracer} ALLOCATOR=${3:-gmalloc} +if [ -z "${RUNTIME}" ]; then + echo "FAIL:missing RUNTIME argument (usage: $0 [config] [allocator])" >&2 + exit 1 +fi + echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}" CHAOS_JDK="${CHAOS_JDK:-21.0.3-tem}" From bf5c239d522d62d82c46f33045e652ee389ec579 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 10 Jul 2026 21:08:06 +0200 Subject: [PATCH 10/10] chaos: standalone harness script + debug-only stale-carrier UAF watchdog Extract chaos_check.sh's logic into utils/run-chaos-harness.sh for local repro. Add a #ifdef DEBUG-only watchdog in ProfiledThread's free path (threadLocalData.cpp) that detects writes into freed OTel-context memory, and beef up VirtualThreadContextCascadeAntagonist's stale-carrier racing (continuous decoupled driver, randomized race window, dead-code removal). Fixes from review: Main.java antagonist-start try/finally, watcher thread signal-mask inheritance, racer-loop backoff under saturation, redundant frame_ut.cpp assertions. Co-Authored-By: Claude Sonnet 5 --- .gitlab/reliability/chaos_check.sh | 190 +--------------- ddprof-lib/src/main/cpp/threadLocalData.cpp | 130 +++++++++++ ddprof-lib/src/test/cpp/frame_ut.cpp | 16 -- .../com/datadoghq/profiler/chaos/Main.java | 16 +- ...VirtualThreadContextCascadeAntagonist.java | 205 +++++++++++++++--- utils/run-chaos-harness.sh | 195 +++++++++++++++++ 6 files changed, 515 insertions(+), 237 deletions(-) create mode 100755 utils/run-chaos-harness.sh diff --git a/.gitlab/reliability/chaos_check.sh b/.gitlab/reliability/chaos_check.sh index d4c2b26dc2..6ac5c6605e 100755 --- a/.gitlab/reliability/chaos_check.sh +++ b/.gitlab/reliability/chaos_check.sh @@ -1,185 +1,17 @@ #!/usr/bin/env bash - -set +e # Disable exit on error +# +# CI entry point for the chaos harness. Thin wrapper around +# utils/run-chaos-harness.sh — this file only supplies the CI-specific +# caching paths (so repeated scheduled runs on the same runner reuse the +# downloaded JDK/agent jar) and the fixed hs_err.log location the pipeline's +# `artifacts:` block expects at the repo root. All the actual build/run logic +# lives in the standalone script, which is also runnable by hand. HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" ROOT="$( cd "${HERE}/../.." >/dev/null 2>&1 && pwd )" -RUNTIME=${1} -CONFIG=${2:-profiler+tracer} -ALLOCATOR=${3:-gmalloc} - -if [ -z "${RUNTIME}" ]; then - echo "FAIL:missing RUNTIME argument (usage: $0 [config] [allocator])" >&2 - exit 1 -fi - -echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}" - -CHAOS_JDK="${CHAOS_JDK:-21.0.3-tem}" -# CHAOS_JDK uses sdkman notation (-); extract major for Adoptium API. -JDK_MAJOR="${CHAOS_JDK%%.*}" -JDK_ARCH=$(uname -m | sed 's/x86_64/x64/') -JDK_INSTALL_DIR="/opt/jdk-${CHAOS_JDK}" - -if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then - TMP=$(mktemp -d) - DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse" - echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..." - if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then - echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2 - rm -rf "${TMP}" - exit 1 - fi - mkdir -p "${JDK_INSTALL_DIR}" - tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1 - rm -rf "${TMP}" -fi - -if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then - echo "FAIL:JDK ${CHAOS_JDK} not available after install" >&2 - exit 1 -fi -export JAVA_HOME="${JDK_INSTALL_DIR}" -export PATH="${JAVA_HOME}/bin:${PATH}" -ACTIVE_JDK=$(java -version 2>&1 | head -1) -# Check major version only — patch may differ from the Adoptium latest GA. -if ! echo "${ACTIVE_JDK}" | grep -qE "\"${JDK_MAJOR}\."; then - echo "FAIL:wrong JDK active (expected major ${JDK_MAJOR}, got: ${ACTIVE_JDK})" >&2 - exit 1 -fi - -# Resolve ddprof.jar: prefer local build artifact, fall back to Maven snapshot. -# Running mvn from /tmp avoids the empty pom.xml at the repo root. -DDPROF_JAR_LOCAL=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1) -if [ -n "${DDPROF_JAR_LOCAL}" ] && [ -f "${DDPROF_JAR_LOCAL}" ]; then - DDPROF_JAR="${DDPROF_JAR_LOCAL}" - echo "Using local ddprof jar: ${DDPROF_JAR}" -else - if [ -z "${CURRENT_VERSION:-}" ]; then - echo "FAIL:CURRENT_VERSION is empty and no local jar found (get-versions dotenv missing)" >&2 - exit 1 - fi - echo "Local ddprof jar not found — downloading ${CURRENT_VERSION} from Maven snapshots" - (cd /tmp && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ - -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \ - -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION}) - DDPROF_JAR="/root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar" -fi - -if [ ! -f "${DDPROF_JAR}" ]; then - echo "FAIL:ddprof jar unavailable" >&2 - exit 1 -fi - -mkdir -p /var/lib/datadog -wget -q --timeout=120 --tries=3 -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer' - -# chaos.jar is produced once per pipeline by the chaos:build job (stresstest -# stage) and pulled here as an artifact. Fall back to an inline build if the -# artifact is absent (e.g. local repro outside CI). -CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar" -if [ ! -f "${CHAOS_JAR}" ]; then - echo "chaos.jar artifact not present — building inline" - ( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q ) -fi - -if [ ! -f "${CHAOS_JAR}" ]; then - echo "FAIL:chaos.jar unavailable" >&2 - exit 1 -fi - -# Patch dd-java-agent.jar with the locally built ddprof contents so the agent's -# (relocated) profiler classes match the version under test. -DD_AGENT_JAR=/var/lib/datadog/dd-java-agent.jar \ -DDPROF_JAR=${DDPROF_JAR} \ -OUTPUT_JAR=/var/lib/datadog/dd-java-agent-patched.jar \ -"${ROOT}/utils/patch-dd-java-agent.sh" - -if [ ! -f /var/lib/datadog/dd-java-agent-patched.jar ]; then - echo "FAIL:dd-java-agent patching failed" >&2 - exit 1 -fi - -PATCHED_AGENT=/var/lib/datadog/dd-java-agent-patched.jar - -case $CONFIG in - profiler) - echo "Running with profiler only" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" - # @Trace is a no-op without the tracer, so trace-context is excluded here. - ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,vthread-context-cascade,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" - ;; - profiler+tracer) - echo "Running with profiler and tracer" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" - ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,vthread-context-cascade,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" - ;; - *) - echo "Unknown configuration: $CONFIG" - exit 1 - ;; -esac - -case $ALLOCATOR in - gmalloc) - echo "Running with gmalloc" - # Turn silent heap corruption (e.g. a stale-pointer UAF write into a freed - # chunk) into an immediate, attributable SIGABRT instead of a crash much - # later in an unrelated allocation. Only meaningful against glibc's own - # malloc, not the LD_PRELOAD-replaced allocators below. - export MALLOC_CHECK_=3 - export MALLOC_PERTURB_=$((RANDOM % 255 + 1)) - ;; - tcmalloc) - echo "Running with tcmalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4') - ;; - jemalloc) - echo "Running with jemalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so') - ;; - *) - echo "Unknown allocator: $ALLOCATOR" - echo "Valid values are: gmalloc, tcmalloc, jemalloc" - exit 1 - ;; -esac - -echo "LD_PRELOAD=$LD_PRELOAD" - -timeout "$((RUNTIME + 300))" \ -java -javaagent:${PATCHED_AGENT} \ - ${ENABLEMENT} \ - -Ddd.profiling.upload.period=10 \ - -Ddd.profiling.start-force-first=true \ - -Ddd.profiling.ddprof.liveheap.enabled=true \ - -Ddd.profiling.ddprof.alloc.enabled=true \ - -Ddd.profiling.ddprof.wall.enabled=true \ - -Ddd.profiling.ddprof.nativemem.enabled=true \ - -Ddd.env=java-profiler-stability \ - -Ddd.service=java-profiler-chaos \ - -Xmx2g -Xms2g \ - -XX:MaxMetaspaceSize=384m \ - -XX:NativeMemoryTracking=summary \ - -XX:ErrorFile=${HERE}/../../hs_err.log \ - -XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \ - -jar ${CHAOS_JAR} \ - --duration ${RUNTIME}s \ - --antagonists ${ANTAGONISTS} - -RC=$? -echo "RC=$RC" +export CHAOS_JDK_DIR="${CHAOS_JDK_DIR:-/opt/jdk-${CHAOS_JDK}}" +export CHAOS_WORK_DIR="${CHAOS_WORK_DIR:-/var/lib/datadog}" +export CHAOS_ERROR_FILE="${ROOT}/hs_err.log" -if [ $RC -ne 0 ]; then - CRASH_MSG="Chaos harness crashed (RC=${RC})" - HS_ERR="${HERE}/../../hs_err.log" - if [ -f "${HS_ERR}" ]; then - SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120) - FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120) - [ -n "${SIG}" ] && CRASH_MSG="${CRASH_MSG};${SIG}" - [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}" - fi - echo "FAIL:${CRASH_MSG}" >&2 - exit 1 -fi +exec "${ROOT}/utils/run-chaos-harness.sh" "$@" diff --git a/ddprof-lib/src/main/cpp/threadLocalData.cpp b/ddprof-lib/src/main/cpp/threadLocalData.cpp index 1843514bd2..7e5ebaae2f 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalData.cpp @@ -11,9 +11,125 @@ #include #include +#ifdef DEBUG +#include +#include +#include +#include +#include +#endif + pthread_key_t ProfiledThread::_tls_key; bool ProfiledThread::_tls_key_initialized = false; +#ifdef DEBUG +// Temporary chaos-testing diagnostic (jb/vt_churn): proves a stale-carrier UAF write actually +// lands on freed ProfiledThread memory, without relying on glibc's malloc-metadata checks +// (MALLOC_CHECK_/tcache safe-linking), which only catch corruption at chunk boundaries — the +// OTel record header a Java-side stale write targets sits well inside the chunk, not at its +// start. Snapshots the header right after free (MALLOC_PERTURB_ has already scribbled it if the +// gmalloc harness is active), then re-reads the same dangling address after a delay exceeding the +// stale-racer's max sleep window; a mismatch proves *something* wrote to memory this thread no +// longer owns. Reading freed memory is deliberately undefined behavior — this exists only to get +// a signal during the vthread-context-cascade UAF investigation and must not ship. +// +// Uses a fixed-size slot pool + one shared watcher thread instead of a thread per free: at this +// antagonist's churn rate (hundreds of frees/sec), thread-per-free piles up thousands of +// detached, sleeping threads and gets the harness process OOM-killed (observed as RC=137) — +// an artifact of the diagnostic, not the bug under investigation. A dropped sample (no free +// slot) just means fewer data points, not a wrong result. +namespace { +constexpr int kWatchSlotCount = 64; +constexpr long kWatchDelayMillis = 900; // past FAST_STALE_RACER_SLEEP_MAX_MILLIS (800ms) +enum SlotState : int { kFree = 0, kFilling = 1, kReady = 2 }; + +struct WatchSlot { + std::atomic state{kFree}; + const void *recordAddr; + int tid; + uint8_t snapshot[OTEL_HEADER_SIZE]; + std::chrono::steady_clock::time_point dueAt; +}; + +WatchSlot g_watchSlots[kWatchSlotCount]; +std::atomic g_watcherStarted{false}; + +// A byte pattern that could plausibly be a live, *activated* OtelThreadContextRecord (see +// otel_context.h). Requires valid==1 specifically rather than valid∈{0,1}: valid==0 is also the +// record's zero-initialized default, so it's satisfied by plain uninitialized/zeroed memory and +// proves nothing. valid==1 only happens once Java has actually written a real trace/span +// activation into the record. _reserved must be 0 per the OTEP spec, which coincidental +// allocator garbage (tcache/fastbin linkage, an unrelated object landing on the same address) +// only matches by chance. Combined with attrs_data_size being in range, this cuts the chance of +// a random-byte false positive to roughly 1 in 1.8e7 — reused-but-unrelated memory essentially +// never passes all three checks together. +bool looksLikeOtelRecord(const uint8_t *header) { + uint8_t valid = header[16 + 8]; + uint8_t reserved = header[16 + 8 + 1]; + uint16_t attrsSize; + memcpy(&attrsSize, header + 16 + 8 + 2, sizeof(attrsSize)); + return valid == 1 && reserved == 0 && attrsSize <= OTEL_MAX_ATTRS_DATA_SIZE; +} + +void watcherLoop() { + // Lazily spawned from inside freeKey()/release() while their SignalBlocker is still in scope, + // so this thread inherits SIGPROF/SIGVTALRM blocked from its creator. Unblock them here so a + // permanently-running debug thread doesn't carry that mask for the rest of the process. + sigset_t prof_signals; + sigemptyset(&prof_signals); + sigaddset(&prof_signals, SIGPROF); + sigaddset(&prof_signals, SIGVTALRM); + pthread_sigmask(SIG_UNBLOCK, &prof_signals, nullptr); + for (;;) { + usleep(50 * 1000); + auto now = std::chrono::steady_clock::now(); + for (WatchSlot &slot : g_watchSlots) { + if (slot.state.load(std::memory_order_acquire) != kReady) { + continue; + } + if (now < slot.dueAt) { + continue; + } + uint8_t after[OTEL_HEADER_SIZE]; + memcpy(after, slot.recordAddr, OTEL_HEADER_SIZE); + bool changed = memcmp(slot.snapshot, after, OTEL_HEADER_SIZE) != 0; + if (changed && looksLikeOtelRecord(after)) { + fprintf(stderr, + "[ddprof-debug] UAF WRITE DETECTED: freed ProfiledThread tid=%d otel-header " + "changed after free and looks like a live OTel record (addr=%p)\n", + slot.tid, slot.recordAddr); + fprintf(stderr, "[ddprof-debug] before:"); + for (uint8_t b : slot.snapshot) fprintf(stderr, " %02x", b); + fprintf(stderr, "\n[ddprof-debug] after: "); + for (uint8_t b : after) fprintf(stderr, " %02x", b); + fprintf(stderr, "\n"); + } + slot.state.store(kFree, std::memory_order_release); + } + } +} +} // namespace + +static void watchFreedContextMemory(const void *recordAddr, int tid) { + if (!g_watcherStarted.exchange(true, std::memory_order_acq_rel)) { + std::thread(watcherLoop).detach(); + } + for (WatchSlot &slot : g_watchSlots) { + int expected = kFree; + if (slot.state.compare_exchange_strong(expected, kFilling, std::memory_order_acq_rel)) { + slot.recordAddr = recordAddr; + slot.tid = tid; + memcpy(slot.snapshot, recordAddr, OTEL_HEADER_SIZE); + slot.dueAt = std::chrono::steady_clock::now() + std::chrono::milliseconds(kWatchDelayMillis); + slot.state.store(kReady, std::memory_order_release); + return; + } + } + // No free slot: drop this sample rather than blocking the freeing thread or spawning + // another thread. +} +#endif + void ProfiledThread::initTLSKey() { static pthread_once_t tls_initialized = PTHREAD_ONCE_INIT; pthread_once(&tls_initialized, doInitTLSKey); @@ -32,7 +148,14 @@ inline void ProfiledThread::freeKey(void *key) { ProfiledThread *tls_ref = (ProfiledThread *)(key); if (tls_ref != NULL) { SignalBlocker blocker; +#ifdef DEBUG + void *recordAddr = (void *)tls_ref->getOtelContextRecord(); + int tid = tls_ref->tid(); +#endif delete tls_ref; +#ifdef DEBUG + watchFreedContextMemory(recordAddr, tid); +#endif } } @@ -60,7 +183,14 @@ void ProfiledThread::release() { if (tls != NULL) { SignalBlocker blocker; pthread_setspecific(key, NULL); +#ifdef DEBUG + void *recordAddr = (void *)tls->getOtelContextRecord(); + int tid = tls->tid(); +#endif delete tls; +#ifdef DEBUG + watchFreedContextMemory(recordAddr, tid); +#endif } } diff --git a/ddprof-lib/src/test/cpp/frame_ut.cpp b/ddprof-lib/src/test/cpp/frame_ut.cpp index e1de762c58..a211eb02a3 100644 --- a/ddprof-lib/src/test/cpp/frame_ut.cpp +++ b/ddprof-lib/src/test/cpp/frame_ut.cpp @@ -56,13 +56,6 @@ TEST(FrameTypeEncodeTest, RawPointerBitNotSetByDefault) { EXPECT_EQ(encoded & (1 << 30), 0) << "rawPointer flag (bit 30) must not be set by default"; } -TEST(FrameTypeEncodeTest, EncodedValuesArePositive) { - for (int t = FRAME_INTERPRETED; t <= FRAME_TYPE_MAX; ++t) { - int encoded = FrameType::encode(t, 0); - EXPECT_GT(encoded, 0) << "encode() must return a positive value for type " << t; - } -} - // ---- decode ---------------------------------------------------------------- TEST(FrameTypeDecodeTest, DecodeZeroReturnsJitCompiled) { @@ -110,15 +103,6 @@ TEST(FrameTypeDecodeTest, RoundTripAllTypesNonZeroBci) { } } -TEST(FrameTypeDecodeTest, DecodedTypeIsInValidRange) { - for (int t = FRAME_INTERPRETED; t <= FRAME_TYPE_MAX; ++t) { - int encoded = FrameType::encode(t, 42); - FrameTypeId decoded = FrameType::decode(encoded); - EXPECT_GE(decoded, FRAME_INTERPRETED); - EXPECT_LE(decoded, FRAME_TYPE_MAX); - } -} - // ---- isRawPointer ---------------------------------------------------------- TEST(FrameTypeIsRawPointerTest, FalseForZero) { diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java index 1594b0ad66..14d70be832 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java @@ -43,15 +43,15 @@ public static void main(String[] args) throws Exception { log("starting duration=" + parsed.duration + " antagonists=" + parsed.antagonists); List running = new ArrayList<>(); - for (String name : parsed.antagonists) { - Antagonist a = create(name); - a.start(); - running.add(a); - log("antagonist started: " + a.name()); - } - - long deadlineNanos = System.nanoTime() + parsed.duration.toNanos(); try { + for (String name : parsed.antagonists) { + Antagonist a = create(name); + a.start(); + running.add(a); + log("antagonist started: " + a.name()); + } + + long deadlineNanos = System.nanoTime() + parsed.duration.toNanos(); while (System.nanoTime() < deadlineNanos) { Thread.sleep(1_000L); } diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java index d4f2aa6309..78de52e195 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java @@ -10,10 +10,14 @@ package com.datadoghq.profiler.chaos; import datadog.trace.api.Trace; +import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.time.Duration; +import java.util.concurrent.Executor; import java.util.concurrent.Semaphore; +import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -58,21 +62,34 @@ * remounted elsewhere, that second write reuses the (possibly now-stale) cached conduit. * *

Catching genuine memory reuse (not just misattribution) needs the original carrier - * to actually be torn down first. The JDK's default virtual-thread scheduler is a - * {@code ForkJoinPool} built with {@code corePoolSize=0} and a 30s keep-alive (see - * {@code java.lang.VirtualThread#createDefaultScheduler()}) — every carrier, not just - * compensating ones, is reaped after 30s with nothing runnable. {@link #driverLoop} therefore - * runs a duty cycle: a short active burst (cascades, pin churn, and a batch of long-sleeping - * {@link #runStaleBufferRacer} threads) followed by a quiet phase with no new work, long enough - * for carriers to actually go idle and exit. Each racer writes context once ({@link - * #staleRacerBefore}), sleeps past the 30s keep-alive, then writes again ({@link - * #staleRacerAfter}) on the far side of the quiet phase — by then its original carrier is very - * likely gone, and under sustained allocation churn from the rest of the harness its freed {@code - * ProfiledThread} memory may already belong to a different, live carrier. + * to actually be torn down first. {@link #driverLoop} runs a duty cycle for the cascade/pin-churn + * side only (a short active burst followed by a quiet phase with no new work, long enough for the + * JDK's default {@code ForkJoinPool} scheduler — 30s keep-alive, see {@code + * java.lang.VirtualThread#createDefaultScheduler()} — to actually reap idle carriers). Stale- + * buffer racers ({@link #runStaleBufferRacer}) don't wait on that: every racer runs on a virtual + * thread bound to a purpose-built {@link #fastCarrierScheduler} — a {@link ThreadPoolExecutor} + * with a 200ms keep-alive — via the package-private {@code VirtualThread(Executor, String, int, + * Runnable)} constructor (JDK 21+; see {@code java.lang.VirtualThread}, requires {@code + * --add-opens java.base/java.lang=ALL-UNNAMED}). {@link #staleRacerLoop} keeps up to {@link + * #MAX_STALE_RACERS} of these racers in flight continuously, independent of the cascade/pin duty + * cycle above — sustained concurrent hammering rather than one batch per cycle, since more + * concurrent carrier churn means more chances for a freed {@code ProfiledThread} slot to actually + * get reused before a stale racer touches it. Each racer writes context once ({@link + * #staleRacerBefore}), sleeps a randomized interval straddling the scheduler's keep-alive ({@link + * #FAST_STALE_RACER_SLEEP_MIN_MILLIS}–{@link #FAST_STALE_RACER_SLEEP_MAX_MILLIS}), then writes + * again ({@link #staleRacerAfter}) — varying the wait means a run samples both "carrier likely + * still tearing down" and "carrier long gone, memory possibly already reused" timings, instead of + * only ever probing one fixed instant. * - *

Reflectively detects {@code Thread.ofVirtual()} (Java 21+); gracefully no-ops on older - * runtimes. Live virtual thread count is bounded by semaphore permit pools so cascades, pinning - * churn, and stale-buffer racers cannot runaway the JVM. + *

Reflectively detects {@code Thread.ofVirtual()} (Java 21+); gracefully no-ops the whole + * antagonist on older runtimes where it's absent. But on a JDK where it is present, the + * {@code VirtualThread} constructor above is required, not optional: it's this antagonist's only + * way to reach the stale-carrier UAF without a slow duty-cycle wait, so {@link #start} fails fast + * with an {@link IllegalStateException} (crashing the chaos harness process, which the CI runner + * already treats as a failure) rather than silently losing that coverage if a missing {@code + * --add-opens} flag or a JDK build that changed the constructor's signature makes it + * unreachable. Live virtual thread count is bounded by semaphore permit pools so cascades, + * pinning churn, and stale-buffer racers cannot runaway the JVM. */ public final class VirtualThreadContextCascadeAntagonist implements Antagonist { @@ -103,17 +120,31 @@ public final class VirtualThreadContextCascadeAntagonist implements Antagonist { // carriers actually exit before the stale-buffer racers spawned in the burst wake back up. private static final long ACTIVE_PHASE_MILLIS = 8_000L; private static final long QUIET_PHASE_MILLIS = 40_000L; - private static final long STALE_RACER_SLEEP_MILLIS = ACTIVE_PHASE_MILLIS + QUIET_PHASE_MILLIS + 5_000L; - private static final int STALE_RACER_BATCH = 16; - private static final int MAX_STALE_RACERS = 256; + + // Stale-buffer racers run continuously on their own driver, decoupled from the cascade/pin + // duty cycle above — that cycle only exists to let the JDK default scheduler's 30s keep-alive + // reap cascade/pin carriers, which is irrelevant to racers bound to fastCarrierScheduler. + private static final int STALE_RACER_BATCH = 64; + private static final int MAX_STALE_RACERS = 1024; + + // A short keep-alive forces genuine carrier-thread exit (and thus a real ProfiledThread + // free()) in well under a second instead of 30s. The racer's sleep is randomized across a + // range straddling this keep-alive so runs cover both "carrier likely still exiting" and + // "carrier long gone, memory possibly already reused" timings instead of one fixed instant. + private static final long FAST_SCHEDULER_KEEPALIVE_MILLIS = 200L; + private static final long FAST_STALE_RACER_SLEEP_MIN_MILLIS = 50L; + private static final long FAST_STALE_RACER_SLEEP_MAX_MILLIS = FAST_SCHEDULER_KEEPALIVE_MILLIS * 4; + private static final Constructor VTHREAD_CTOR = resolveVirtualThreadCtor(); private final Semaphore liveVthreads = new Semaphore(MAX_LIVE_VTHREADS); private final Semaphore livePinVthreads = new Semaphore(MAX_PIN_VTHREADS); private final Semaphore liveStaleRacers = new Semaphore(MAX_STALE_RACERS); + private final Executor fastCarrierScheduler = VTHREAD_CTOR != null ? newFastCarrierScheduler() : null; private volatile boolean running; private volatile boolean activePhase = true; private Thread driver; private Thread pinningDriver; + private Thread staleRacerDriver; private final AtomicLong sink = new AtomicLong(); @Override @@ -123,6 +154,16 @@ public String name() { @Override public void start() { + if (OF_VIRTUAL != null && BUILDER_START != null && VTHREAD_CTOR == null) { + // We're on a VT-capable JDK, so the fast-carrier constructor should be reachable — + // its absence (missing --add-opens, or a JDK build that changed the constructor) + // would otherwise silently drop stale-racer coverage instead of failing the run. + throw new IllegalStateException( + "vthread-context-cascade: java.lang.VirtualThread(Executor, String, int, " + + "Runnable) is unreachable on a VT-capable JDK — pass --add-opens " + + "java.base/java.lang=ALL-UNNAMED, or this antagonist loses its core " + + "stale-carrier UAF coverage"); + } running = true; driver = new Thread(this::driverLoop, "chaos-vthread-cascade-driver"); driver.setDaemon(true); @@ -130,6 +171,9 @@ public void start() { pinningDriver = new Thread(this::pinningChurnLoop, "chaos-vthread-cascade-pin-driver"); pinningDriver.setDaemon(true); pinningDriver.start(); + staleRacerDriver = new Thread(this::staleRacerLoop, "chaos-vthread-cascade-racer-driver"); + staleRacerDriver.setDaemon(true); + staleRacerDriver.start(); } @Override @@ -146,6 +190,11 @@ public void stopGracefully(Duration timeout) { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + try { + staleRacerDriver.join(remainingMillis(deadlineNanos)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } // Best-effort drain of in-flight cascades so the JVM doesn't exit mid-fan-out. try { liveVthreads.tryAcquire(MAX_LIVE_VTHREADS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); @@ -236,10 +285,8 @@ private void driverLoop() { return; } while (running) { - // Active phase: mint cascade work and a batch of stale-buffer racers that will wake - // up on the far side of the upcoming quiet phase. + // Active phase: mint cascade work. activePhase = true; - spawnStaleBufferRacers(); long activeDeadlineNanos = System.nanoTime() + Duration.ofMillis(ACTIVE_PHASE_MILLIS).toNanos(); while (running && System.nanoTime() < activeDeadlineNanos) { for (int i = 0; i < ROOT_BATCH && running; i++) { @@ -270,35 +317,62 @@ private void driverLoop() { } /** - * Spawns a batch of stale-buffer racers timed to wake up just after the current quiet phase - * ends, when fresh work is remounting virtual threads onto carriers the scheduler has likely - * just (re)created. + * Independent driver that keeps the stale-buffer racer population saturated at {@link + * #MAX_STALE_RACERS} for as long as the antagonist runs, regardless of the cascade/pin duty + * cycle — the fast-carrier scheduler needs no quiet phase to reap its carriers, so there's no + * reason to gate racer spawning on one. This is the "many vthreads hammering" dimension: + * sustained concurrent pressure on the fast-carrier scheduler's pool, not periodic bursts. */ - private void spawnStaleBufferRacers() { + private void staleRacerLoop() { + while (running) { + boolean spawnedAny = spawnStaleBufferRacers(); + try { + // Once the pool is fully saturated, back off instead of re-polling 1000x/sec for + // permits that free up far slower (racers sleep up to FAST_STALE_RACER_SLEEP_MAX_MILLIS). + Thread.sleep(spawnedAny ? 1L : 20L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + /** Returns whether at least one racer was actually started this pass. */ + private boolean spawnStaleBufferRacers() { + boolean spawnedAny = false; for (int i = 0; i < STALE_RACER_BATCH && running; i++) { if (!liveStaleRacers.tryAcquire()) { continue; } - try { - Object builder = OF_VIRTUAL.invoke(null); - BUILDER_START.invoke(builder, (Runnable) this::runStaleBufferRacer); - } catch (Throwable t) { + if (startFastCarrierVirtualThread(this::runStaleBufferRacer) == null) { liveStaleRacers.release(); + } else { + spawnedAny = true; } } + return spawnedAny; } /** - * Activates a span once, sleeps past the scheduler's default 30s carrier keep-alive, then - * activates another. Under {@code ContextStorageMode.THREAD} the second activation reuses - * whatever {@code DirectByteBuffer} was cached on the first — if the original carrier has - * since been reaped and its {@code ProfiledThread} memory reused, this is a genuine - * use-after-free. + * Activates a span once, sleeps past the carrier's keep-alive, then activates another. Under + * {@code ContextStorageMode.THREAD} the second activation reuses whatever {@code + * DirectByteBuffer} was cached on the first — if the original carrier has since been reaped + * and its {@code ProfiledThread} memory reused, this is a genuine use-after-free. + * + *

The sleep is randomized per racer across [{@link #FAST_STALE_RACER_SLEEP_MIN_MILLIS}, + * {@link #FAST_STALE_RACER_SLEEP_MAX_MILLIS}] rather than fixed, so a sustained run samples a + * spread of timings relative to {@link #FAST_SCHEDULER_KEEPALIVE_MILLIS} instead of only ever + * probing the same instant. */ private void runStaleBufferRacer() { try { staleRacerBefore(); - Thread.sleep(STALE_RACER_SLEEP_MILLIS); + long sleepMillis = + ThreadLocalRandom.current() + .nextLong( + FAST_STALE_RACER_SLEEP_MIN_MILLIS, + FAST_STALE_RACER_SLEEP_MAX_MILLIS + 1); + Thread.sleep(sleepMillis); staleRacerAfter(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -389,6 +463,69 @@ private static long forceUnmount(long seed) { return r; } + /** + * Resolves {@code java.lang.VirtualThread}'s package-private {@code (Executor, String, int, + * Runnable)} constructor, the only way to bind a virtual thread to a custom scheduler on + * mainline OpenJDK 21+ (there is no public {@code Thread.ofVirtual().scheduler(Executor)} + * API). Requires {@code --add-opens java.base/java.lang=ALL-UNNAMED}; returns {@code null} + * (caller falls back to the default-scheduler path) if that flag is absent, the JVM is + * pre-21, or the constructor's signature changes on some future JDK build. + */ + private static Constructor resolveVirtualThreadCtor() { + try { + Class vthreadClass = Class.forName("java.lang.VirtualThread"); + Constructor ctor = + vthreadClass.getDeclaredConstructor( + Executor.class, String.class, int.class, Runnable.class); + ctor.setAccessible(true); + return ctor; + } catch (Throwable t) { + return null; + } + } + + /** + * A carrier pool with a short keep-alive instead of the JDK default scheduler's fixed 30s, so + * an idle carrier's OS thread actually exits (and its {@code ProfiledThread} is freed) in + * well under a second. + */ + private static Executor newFastCarrierScheduler() { + return new ThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + FAST_SCHEDULER_KEEPALIVE_MILLIS, + TimeUnit.MILLISECONDS, + new SynchronousQueue<>(), + r -> { + Thread carrier = new Thread(r, "chaos-vthread-cascade-fast-carrier"); + carrier.setDaemon(true); + return carrier; + }); + } + + /** + * Starts {@code task} on a virtual thread bound to {@link #fastCarrierScheduler}. Returns + * {@code null} (never starts anything) if {@link #VTHREAD_CTOR} wasn't resolved or the + * reflective construction fails; {@link #start} already fails fast when {@link #VTHREAD_CTOR} + * is unreachable on a VT-capable JDK, so callers here only need to handle the rarer + * construction-failure case (e.g. transient reflection errors). + * + *

{@code 0} for the characteristics argument mirrors {@code Thread.ofVirtual()}'s default + * (the only known bit, {@code Thread.NO_INHERIT_THREAD_LOCALS}, is unset). + */ + private Thread startFastCarrierVirtualThread(Runnable task) { + if (VTHREAD_CTOR == null) { + return null; + } + try { + Thread t = (Thread) VTHREAD_CTOR.newInstance(fastCarrierScheduler, null, 0, task); + t.start(); + return t; + } catch (Throwable t) { + return null; + } + } + private static Method resolveOfVirtual() { try { return Thread.class.getMethod("ofVirtual"); diff --git a/utils/run-chaos-harness.sh b/utils/run-chaos-harness.sh new file mode 100755 index 0000000000..45a35ba4ab --- /dev/null +++ b/utils/run-chaos-harness.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# +# Standalone chaos-harness runner. Builds/locates the ddprof jar, patches a +# dd-java-agent build with it, and runs the long-running chaos harness +# (ddprof-stresstest/src/chaos) with a chosen antagonist set and allocator. +# +# Usable both from CI (see .gitlab/reliability/chaos_check.sh, a thin wrapper +# around this script) and by hand for local repro — no CI-only assumptions +# (no fixed /opt or /var/lib paths, no CI-artifact env vars required). +# +# Usage: run-chaos-harness.sh [config] [allocator] [antagonists] +# config: profiler | profiler+tracer (default: profiler+tracer) +# allocator: gmalloc | tcmalloc | jemalloc (default: gmalloc) +# antagonists: comma-separated antagonist names; overrides the config default +# +# Env vars: +# CHAOS_JDK sdkman-style version (e.g. 21.0.3-tem) to require/download. +# If unset, uses whatever `java` is already on PATH. +# CHAOS_JDK_DIR where to install a downloaded JDK (default: under +# CHAOS_WORK_DIR; set to a persistent path to cache across runs). +# CHAOS_WORK_DIR scratch dir for the patched agent, downloaded jars, and +# hs_err.log (default: /.chaos-harness). +# CHAOS_REFRESH_AGENT=1 force re-download of dd-java-agent.jar even if cached. + +set +e + +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT="$( cd "${HERE}/.." >/dev/null 2>&1 && pwd )" + +RUNTIME=${1} +CONFIG=${2:-profiler+tracer} +ALLOCATOR=${3:-gmalloc} +ANTAGONISTS_OVERRIDE=${4:-} + +if [ -z "${RUNTIME}" ]; then + echo "usage: $0 [config] [allocator] [antagonists]" >&2 + exit 1 +fi + +WORK_DIR="${CHAOS_WORK_DIR:-${ROOT}/.chaos-harness}" +mkdir -p "${WORK_DIR}" + +echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}" + +# --- JDK resolution --------------------------------------------------------- +# Only fetch a JDK if the caller explicitly pinned one via CHAOS_JDK and it +# doesn't match what's already active; otherwise just use `java` as found. +if [ -n "${CHAOS_JDK:-}" ]; then + JDK_MAJOR="${CHAOS_JDK%%.*}" + ACTIVE_MAJOR=$(java -version 2>&1 | head -1 | grep -oE '"[0-9]+' | tr -d '"') + if [ "${ACTIVE_MAJOR}" != "${JDK_MAJOR}" ]; then + JDK_ARCH=$(uname -m | sed 's/x86_64/x64/') + JDK_INSTALL_DIR="${CHAOS_JDK_DIR:-${WORK_DIR}/jdk-${CHAOS_JDK}}" + if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then + TMP=$(mktemp -d) + DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse" + echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..." + if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then + echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2 + rm -rf "${TMP}" + exit 1 + fi + mkdir -p "${JDK_INSTALL_DIR}" + tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1 + rm -rf "${TMP}" + fi + export JAVA_HOME="${JDK_INSTALL_DIR}" + export PATH="${JAVA_HOME}/bin:${PATH}" + fi +fi + +if ! command -v java >/dev/null 2>&1; then + echo "FAIL:no java on PATH (set CHAOS_JDK to have one downloaded)" >&2 + exit 1 +fi +echo "Using: $(java -version 2>&1 | head -1)" + +# --- ddprof jar -------------------------------------------------------------- +DDPROF_JAR=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1) +if [ -z "${DDPROF_JAR}" ] || [ ! -f "${DDPROF_JAR}" ]; then + echo "FAIL:no local ddprof jar found — build one first: ./gradlew :ddprof-lib:jar" >&2 + exit 1 +fi +echo "Using local ddprof jar: ${DDPROF_JAR}" + +# --- dd-java-agent ------------------------------------------------------------ +DD_AGENT_JAR="${WORK_DIR}/dd-java-agent.jar" +if [ ! -f "${DD_AGENT_JAR}" ] || [ "${CHAOS_REFRESH_AGENT:-0}" = "1" ]; then + echo "Fetching dd-java-agent.jar..." + wget -q --timeout=120 --tries=3 -O "${DD_AGENT_JAR}" 'https://dtdg.co/latest-java-tracer' +fi +if [ ! -f "${DD_AGENT_JAR}" ]; then + echo "FAIL:dd-java-agent.jar download failed" >&2 + exit 1 +fi + +CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar" +if [ ! -f "${CHAOS_JAR}" ]; then + echo "chaos.jar not present — building inline" + ( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q ) +fi +if [ ! -f "${CHAOS_JAR}" ]; then + echo "FAIL:chaos.jar unavailable" >&2 + exit 1 +fi + +# Patch dd-java-agent.jar with the locally built ddprof contents so the +# agent's (relocated) profiler classes match the version under test. +PATCHED_AGENT="${WORK_DIR}/dd-java-agent-patched.jar" +DD_AGENT_JAR="${DD_AGENT_JAR}" DDPROF_JAR="${DDPROF_JAR}" OUTPUT_JAR="${PATCHED_AGENT}" \ + "${ROOT}/utils/patch-dd-java-agent.sh" +if [ ! -f "${PATCHED_AGENT}" ]; then + echo "FAIL:dd-java-agent patching failed" >&2 + exit 1 +fi + +case $CONFIG in + profiler) + ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" + # @Trace is a no-op without the tracer, so trace-context is excluded here. + DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,vthread-context-cascade,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + ;; + profiler+tracer) + ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" + DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,vthread-context-cascade,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + ;; + *) + echo "Unknown configuration: $CONFIG (valid: profiler, profiler+tracer)" >&2 + exit 1 + ;; +esac +ANTAGONISTS="${ANTAGONISTS_OVERRIDE:-${DEFAULT_ANTAGONISTS}}" + +case $ALLOCATOR in + gmalloc) + # Turn silent heap corruption (e.g. a stale-pointer UAF write into a freed + # chunk) into an immediate, attributable SIGABRT instead of a crash much + # later in an unrelated allocation. Only meaningful against glibc's own + # malloc, not the LD_PRELOAD-replaced allocators below. + export MALLOC_CHECK_=3 + export MALLOC_PERTURB_=$((RANDOM % 255 + 1)) + ;; + tcmalloc) + export LD_PRELOAD=$(find /usr/lib/ /usr/lib64/ /opt/homebrew/lib/ /usr/local/lib/ -name 'libtcmalloc_minimal.so.4' -o -name 'libtcmalloc.dylib' 2>/dev/null | head -1) + ;; + jemalloc) + export LD_PRELOAD=$(find /usr/lib/ /usr/lib64/ /opt/homebrew/lib/ /usr/local/lib/ -name 'libjemalloc.so' -o -name 'libjemalloc.dylib' 2>/dev/null | head -1) + ;; + *) + echo "Unknown allocator: $ALLOCATOR (valid: gmalloc, tcmalloc, jemalloc)" >&2 + exit 1 + ;; +esac +echo "LD_PRELOAD=${LD_PRELOAD:-}" + +HS_ERR="${CHAOS_ERROR_FILE:-${WORK_DIR}/hs_err.log}" +rm -f "${HS_ERR}" + +timeout "$((RUNTIME + 300))" \ +java -javaagent:${PATCHED_AGENT} \ + --add-opens java.base/java.lang=ALL-UNNAMED \ + ${ENABLEMENT} \ + -Ddd.profiling.upload.period=10 \ + -Ddd.profiling.start-force-first=true \ + -Ddd.profiling.ddprof.liveheap.enabled=true \ + -Ddd.profiling.ddprof.alloc.enabled=true \ + -Ddd.profiling.ddprof.wall.enabled=true \ + -Ddd.profiling.ddprof.nativemem.enabled=true \ + -Ddd.env=java-profiler-stability \ + -Ddd.service=java-profiler-chaos \ + -Xmx2g -Xms2g \ + -XX:MaxMetaspaceSize=384m \ + -XX:NativeMemoryTracking=summary \ + -XX:ErrorFile=${HS_ERR} \ + -jar ${CHAOS_JAR} \ + --duration ${RUNTIME}s \ + --antagonists ${ANTAGONISTS} + +RC=$? +echo "RC=$RC" + +if [ $RC -ne 0 ]; then + CRASH_MSG="Chaos harness crashed (RC=${RC})" + if [ -f "${HS_ERR}" ]; then + SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120) + FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120) + [ -n "${SIG}" ] && CRASH_MSG="${CRASH_MSG};${SIG}" + [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}" + echo "hs_err written to ${HS_ERR}" + fi + echo "FAIL:${CRASH_MSG}" >&2 + exit 1 +fi + +echo "Chaos run completed cleanly"