Add processor context for hotspot crash tracking

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java

@@ -561,6 +561,18 @@ private RequestBody makeErrorTrackingRequestBody(@Nonnull CrashLog payload, bool
                       "os.version")); // this has been restructured under OsInfo so taking raw here
           writer.endObject();
         }
+        // experimental
+        if (payload.experimental != null && payload.experimental.ucontext != null) {
+          writer.name("experimental");
+          writer.beginObject();
+          writer.name("ucontext");
+          writer.beginObject();
+          for (Map.Entry<String, String> entry : payload.experimental.ucontext.entrySet()) {
+            writer.name(entry.getKey()).value(entry.getValue());
+          }
+          writer.endObject();
+          writer.endObject();
+        }
         writer.endObject();
       }
       return RequestBody.create(APPLICATION_JSON, buf.readByteString());

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java

@@ -39,6 +39,8 @@ public final class CrashLog {
   @Json(name = "sig_info")
   public final SigInfo sigInfo;
 
+  public final Experimental experimental;
+
   public CrashLog(
       String uuid,
       boolean incomplete,
@@ -49,6 +51,30 @@ public CrashLog(
       ProcInfo procInfo,
       SigInfo sigInfo,
       String dataSchemaVersion) {
+    this(
+        uuid,
+        incomplete,
+        timestamp,
+        error,
+        metadata,
+        osInfo,
+        procInfo,
+        sigInfo,
+        dataSchemaVersion,
+        null);
+  }
+
+  public CrashLog(
+      String uuid,
+      boolean incomplete,
+      String timestamp,
+      ErrorData error,
+      Metadata metadata,
+      OSInfo osInfo,
+      ProcInfo procInfo,
+      SigInfo sigInfo,
+      String dataSchemaVersion,
+      Experimental experimental) {
     this.uuid = uuid != null ? uuid : RandomUtils.randomUUID().toString();
     this.incomplete = incomplete;
     this.timestamp = timestamp;
@@ -58,6 +84,7 @@ public CrashLog(
     this.procInfo = procInfo;
     this.sigInfo = sigInfo;
     this.dataSchemaVersion = dataSchemaVersion;
+    this.experimental = experimental;
   }
 
   public String toJson() {
@@ -85,7 +112,8 @@ public boolean equals(Object o) {
         && Objects.equals(osInfo, crashLog.osInfo)
         && Objects.equals(procInfo, crashLog.procInfo)
         && Objects.equals(sigInfo, crashLog.sigInfo)
-        && Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion);
+        && Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion)
+        && Objects.equals(experimental, crashLog.experimental);
   }
 
   @Override
@@ -100,7 +128,8 @@ public int hashCode() {
         procInfo,
         sigInfo,
         version,
-        dataSchemaVersion);
+        dataSchemaVersion,
+        experimental);
   }
 
   public boolean equalsForTest(Object o) {
@@ -119,6 +148,7 @@ public boolean equalsForTest(Object o) {
         && Objects.equals(error, crashLog.error)
         && Objects.equals(procInfo, crashLog.procInfo)
         && Objects.equals(sigInfo, crashLog.sigInfo)
-        && Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion);
+        && Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion)
+        && Objects.equals(experimental, crashLog.experimental);
   }
 }

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/Experimental.java

@@ -0,0 +1,24 @@
+package datadog.crashtracking.dto;
+
+import java.util.Map;
+import java.util.Objects;
+
+public final class Experimental {
+  public final Map<String, String> ucontext;
+
+  public Experimental(Map<String, String> ucontext) {
+    this.ucontext = ucontext;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!(o instanceof Experimental)) return false;
+    Experimental that = (Experimental) o;
+    return Objects.equals(ucontext, that.ucontext);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(ucontext);
+  }
+}

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java

@@ -7,6 +7,7 @@
 import datadog.crashtracking.buildid.BuildInfo;
 import datadog.crashtracking.dto.CrashLog;
 import datadog.crashtracking.dto.ErrorData;
+import datadog.crashtracking.dto.Experimental;
 import datadog.crashtracking.dto.Metadata;
 import datadog.crashtracking.dto.OSInfo;
 import datadog.crashtracking.dto.ProcInfo;
@@ -21,11 +22,25 @@
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+/**
+ * Parser for HotSpot JVM fatal error logs ({@code hs_err_pidNNN.log}).
+ *
+ * <p>The log is parsed using a linear state machine that mirrors the deterministic section order
+ * emitted by {@code VMError::report()} in HotSpot. The section order is fixed for a given platform
+ * but differs across OS/CPU combinations.
+ *
+ * <p>If an early sentinel line is absent (e.g. {@code "Native frames:"} is missing because the JVM
+ * crashed before producing a stack), the state machine will not advance past {@code THREAD} state
+ * and subsequent sections such as {@code siginfo} and registers will be silently skipped. The
+ * resulting {@link datadog.crashtracking.dto.CrashLog} will be marked {@code incomplete}.
+ */
 public final class HotspotCrashLogParser {
   private static final DateTimeFormatter ZONED_DATE_TIME_FORMATTER =
       DateTimeFormatter.ofPattern("EEE MMM ppd HH:mm:ss yyyy zzz", Locale.getDefault());
@@ -45,6 +60,7 @@ enum State {
     SUMMARY,
     THREAD,
     STACKTRACE,
+    REGISTERS,
     SEEK_DYNAMIC_LIBRARIES,
     DYNAMIC_LIBRARIES,
     DONE
@@ -65,6 +81,20 @@ public HotspotCrashLogParser() {
           "siginfo:\\s+si_signo:\\s+(\\d+)\\s+\\((\\w+)\\),\\s+si_code:\\s+(\\d+)\\s+\\(([^)]+)\\),\\s+si_addr:\\s+(0x[0-9a-fA-F]+)");
   private static final Pattern DYNAMIC_LIBS_PATH_PARSER =
       Pattern.compile("^(?:0x)?[0-9a-fA-F]+(?:-[0-9a-fA-F]+)?\\s+(?:[^\\s/\\[]+\\s+)*(.*)$");
+  // Matches register entries like:
+  // * RAX=0x..., R8 =0x..., TRAPNO=0x... (x86-64)
+  // * R0=0x..., R30=0x... (Linux aarch64)
+  // * x0=0x..., fp=0x..., lr=0x..., sp=0x..., pc=0x... (macOS aarch64)
+  // Note that register formatting varies by platform, the JVM crash handler can emit one or four
+  // per line.
+  private static final Pattern REGISTER_ENTRY_PARSER =
+      Pattern.compile("([A-Za-z][A-Za-z0-9]*)\\s*=\\s*(0x[0-9a-fA-F]+)");
+  // Used for the REGISTERS-state exit condition only: the register name must start the line
+  // (after optional whitespace). This prevents lines like "Top of Stack: (sp=0x...)" and
+  // "Instructions: (pc=0x...)" from being mistaken for register entries by REGISTER_ENTRY_PARSER's
+  // find(), which would otherwise match the lowercase "sp"/"pc" tokens embedded in those lines.
+  private static final Pattern REGISTER_LINE_START =
+      Pattern.compile("^\\s*[A-Za-z][A-Za-z0-9]*\\s*=\\s*0x");
 
   private StackFrame parseLine(String line) {
     if (line == null || line.isEmpty()) {
@@ -84,10 +114,10 @@ private StackFrame parseLine(String line) {
     switch (firstChar) {
       case 'J':
         {
-          // J 36572 c2 datadog.trace.util.AgentTaskScheduler$PeriodicTask.run()V (25 bytes) @
-          // 0x00007f2fd0198488 [0x00007f2fd0198420+0x0000000000000068]
-          // J 3896 c2 java.nio.ByteBuffer.allocate(I)Ljava/nio/ByteBuffer; java.base@21.0.1 (20
-          // bytes) @ 0x0000000112ad51e8 [0x0000000112ad4fc0+0x0000000000000228]
+          // spotless:off
+          // J 36572 c2 datadog.trace.util.AgentTaskScheduler$PeriodicTask.run()V (25 bytes) @ 0x00007f2fd0198488 [0x00007f2fd0198420+0x0000000000000068]
+          // J 3896 c2 java.nio.ByteBuffer.allocate(I)Ljava/nio/ByteBuffer; java.base@21.0.1 (20 bytes) @ 0x0000000112ad51e8 [0x0000000112ad4fc0+0x0000000000000228]
+          // spotless:on
           String[] parts = SPACE_SPLITTER.split(line);
           if (parts.length > 3) {
             functionName = parts[3];
@@ -221,6 +251,7 @@ public CrashLog parse(String uuid, String crashLog) {
     String datetime = null;
     boolean incomplete = false;
     String oomMessage = null;
+    Map<String, String> registers = null;
 
     String[] lines = NEWLINE_SPLITTER.split(crashLog);
     outer:
@@ -275,8 +306,9 @@ public CrashLog parse(String uuid, String crashLog) {
           break;
         case STACKTRACE:
           if (line.startsWith("siginfo:")) {
-            // siginfo: si_signo: 11 (SIGSEGV), si_code: 1 (SEGV_MAPERR), si_addr:
-            // 0x0000000000000070
+            // spotless:off
+            // siginfo: si_signo: 11 (SIGSEGV), si_code: 1 (SEGV_MAPERR), si_addr: 0x0000000000000070
+            // spotless:on
             final Matcher siginfoMatcher = SIGINFO_PARSER.matcher(line);
             if (siginfoMatcher.matches()) {
               Integer number = safelyParseInt(siginfoMatcher.group(1));
@@ -286,6 +318,9 @@ public CrashLog parse(String uuid, String crashLog) {
               String address = siginfoMatcher.group(5);
               sigInfo = new SigInfo(number, name, siCode, sigAction, address);
             }
+          } else if (line.startsWith("Registers:")) {
+            registers = new LinkedHashMap<>();
+            state = State.REGISTERS;
           } else if (line.contains("P R O C E S S")) {
             state = State.SEEK_DYNAMIC_LIBRARIES;
           } else {
@@ -296,6 +331,17 @@ public CrashLog parse(String uuid, String crashLog) {
             }
           }
           break;
+        case REGISTERS:
+          if (!line.isEmpty() && !REGISTER_LINE_START.matcher(line).find()) {
+            // non-empty line that does not start with a register entry signals end of section
+            state = State.STACKTRACE;
+          } else {
+            final Matcher m = REGISTER_ENTRY_PARSER.matcher(line);
+            while (m.find()) {
+              registers.put(m.group(1), m.group(2));
+            }
+          }
+          break;
         case SEEK_DYNAMIC_LIBRARIES:
           if (line.startsWith("Dynamic libraries:")) {
             state = State.DYNAMIC_LIBRARIES;
@@ -382,8 +428,19 @@ public CrashLog parse(String uuid, String crashLog) {
     Metadata metadata = new Metadata("dd-trace-java", VersionInfo.VERSION, "java", null);
     Integer parsedPid = safelyParseInt(pid);
     ProcInfo procInfo = parsedPid != null ? new ProcInfo(parsedPid) : null;
+    Experimental experimental =
+        (registers != null && !registers.isEmpty()) ? new Experimental(registers) : null;
     return new CrashLog(
-        uuid, incomplete, datetime, error, metadata, OSInfo.current(), procInfo, sigInfo, "1.0");
+        uuid,
+        incomplete,
+        datetime,
+        error,
+        metadata,
+        OSInfo.current(),
+        procInfo,
+        sigInfo,
+        "1.0",
+        experimental);
   }
 
   static String dateTimeToISO(String datetime) {

dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/parsers/HotspotCrashLogParserTest.java

@@ -52,6 +52,42 @@ public void testIncompleteParsing() throws Exception {
     assertEquals(0, crashLog.error.stack.frames.length);
   }
 
+  /** macOS aarch64 uses lowercase register names: x0-x28, fp, lr, sp, pc, cpsr */
+  @Test
+  public void testRegisterParsingMacosAarch64() throws Exception {
+    CrashLog crashLog =
+        new HotspotCrashLogParser()
+            .parse(
+                UUID.randomUUID().toString(), readFileAsString("sample-crash-macos-aarch64.txt"));
+
+    assertNotNull(crashLog.experimental, "experimental field should be populated");
+    assertNotNull(crashLog.experimental.ucontext, "ucontext should be populated");
+    assertEquals("0x0000000000000c55", crashLog.experimental.ucontext.get("x0"));
+    assertEquals("0x0000000000000000", crashLog.experimental.ucontext.get("x2"));
+    assertEquals("0x000000016feee210", crashLog.experimental.ucontext.get("fp"));
+    assertEquals("0x0000000116d0c970", crashLog.experimental.ucontext.get("lr"));
+    assertEquals("0x000000016feee0f0", crashLog.experimental.ucontext.get("sp"));
+    assertEquals("0x000000010f8ac794", crashLog.experimental.ucontext.get("pc"));
+    assertEquals("0x0000000060001000", crashLog.experimental.ucontext.get("cpsr"));
+  }
+
+  /** Linux aarch64 uses uppercase register names: R0-R30 */
+  @Test
+  public void testRegisterParsingLinuxAarch64() throws Exception {
+    CrashLog crashLog =
+        new HotspotCrashLogParser()
+            .parse(
+                UUID.randomUUID().toString(), readFileAsString("sample-crash-linux-aarch64.txt"));
+
+    assertNotNull(crashLog.experimental, "experimental field should be populated");
+    assertNotNull(crashLog.experimental.ucontext, "ucontext should be populated");
+    assertEquals("0x0000000000000000", crashLog.experimental.ucontext.get("R0"));
+    assertEquals("0x0000000000000001", crashLog.experimental.ucontext.get("R1"));
+    assertEquals("0x0000ffff9efa168c", crashLog.experimental.ucontext.get("R30"));
+    // "Register to memory mapping:" section must NOT be included
+    assertEquals(31, crashLog.experimental.ucontext.size(), "R0-R30 = 31 registers");
+  }
+
   private String readFileAsString(String resource) throws IOException {
     try (InputStream stream = getClass().getClassLoader().getResourceAsStream(resource)) {
       return new BufferedReader(

dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-2.txt

@@ -1 +1 @@
-{"timestamp":"2024-09-20T13:19:06Z","ddsource":"crashtracker","error":{"is_crash":true,"type":"UNKNOWN","message":"Process terminated by signal UNKNOWN","source_type":"Crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"__pthread_clockjoin_ex+0x255","path":"libpthread.so.0","relative_address":"0x9cd5"}]}}}
+{"timestamp":"2024-09-20T13:19:06Z","ddsource":"crashtracker","error":{"is_crash":true,"type":"UNKNOWN","message":"Process terminated by signal UNKNOWN","source_type":"Crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"__pthread_clockjoin_ex+0x255","path":"libpthread.so.0","relative_address":"0x9cd5"}]}},"experimental":{"ucontext":{"RAX":"0x00000000000000ca","RBX":"0x00000000000000ca","RCX":"0x00007f011ab1ccd7","RDX":"0x000000000008ca23","RSP":"0x00007ffeabf89710","RBP":"0x00007ffeabf897b8","RSI":"0x0000000000000000","RDI":"0x00007f01192129d0","R8":"0x0000000000000000","R9":"0x00007f0119212700","R10":"0x0000000000000000","R11":"0x0000000000000246","R12":"0x000000000008ca23","R13":"0x00007f01192129d0","R14":"0x00007ffeabf89840","R15":"0x00007f0119212700","RIP":"0x00007f011ab1ccd5","EFLAGS":"0x0000000000000246","CSGSFS":"0x002b000000000033","ERR":"0x0000000000000000","TRAPNO":"0x0000000000000000"}}}