From 1f9e1c582ca57cf658e85ab5daf3206e4e10841c Mon Sep 17 00:00:00 2001
From: ASDAlexander77 <duzhar@googlemail.com>
Date: Mon, 14 Sep 2026 21:54:43 +0100
Subject: [PATCH] Stop debug tslang --emit=exe/dll ending in a breakpoint at
 exit

A debug tslang turns on the CRT leak report at exit (_CRTDBG_LEAK_CHECK_DF),
which is emitted as a _CRT_WARN report. Building an exe or dll constructs
llvm::InitLLVM (exe.cpp), whose PrintStackTraceOnErrorSignal installs LLVM's
AvoidMessageBoxHook with _CrtSetReportHook. That hook answers every CRT report
with "retry", and the report macro turns a retry into _CrtDbgBreak - so every
debug --emit=exe and --emit=dll ended with an unhandled breakpoint
(0x80000003, exit code 3) after the output had been written. The heap was not
corrupted: periodic _CrtCheckMemory during the build stayed clean. The JIT
installs the same hook but exits through TerminateProcess, and --emit=obj never
installs it.

Install a _CrtSetReportHook2 hook for _CRT_WARN that writes the message to the
debugger output (where it went before) and does not ask to break. Report hooks
from _CrtSetReportHook2 run before the _CrtSetReportHook one (ucrt
dbgrptt.cpp), so asserts and errors still reach LLVM's hook.

Fixes test-compile-gc-shared-auto and test-jit-gc-defaultlib-collector on debug
builds (CI builds Release, where the leak check does not exist).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 tslang/tslang/tslang.cpp | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/tslang/tslang/tslang.cpp b/tslang/tslang/tslang.cpp
index c76b9be5f..6418e45da 100644
--- a/tslang/tslang/tslang.cpp
+++ b/tslang/tslang/tslang.cpp
@@ -263,9 +263,39 @@ bool prepareDefaultLib(CompileOptions &compileOptions)
     return true;
 }
 
+#if _MSC_VER && _DEBUG
+// The leak report at exit (_CRTDBG_LEAK_CHECK_DF below) goes out as _CRT_WARN. Building an exe or
+// dll runs llvm::InitLLVM (exe.cpp), which installs LLVM's AvoidMessageBoxHook: it answers every CRT
+// report with "retry", and a retry makes the report macro call _CrtDbgBreak - so a debug tslang ended
+// every --emit=exe/dll with an unhandled breakpoint (0x80000003) after writing its output. Hooks
+// installed with _CrtSetReportHook2 run before that one: a warning is written where it would have
+// gone anyway (the debugger output) and never asks to break. Asserts and errors still reach LLVM's hook.
+static int ReportWarningWithoutBreak(int reportType, char *message, int *returnValue)
+{
+    if (reportType != _CRT_WARN)
+    {
+        return FALSE;
+    }
+
+    if (message)
+    {
+        OutputDebugStringA(message);
+    }
+
+    if (returnValue)
+    {
+        *returnValue = 0;
+    }
+
+    return TRUE;
+}
+#endif
+
 int main(int argc, char **argv)
 {
 #if _MSC_VER && _DEBUG
+    _CrtSetReportHook2(_CRT_RPTHOOK_INSTALL, ReportWarningWithoutBreak);
+
     // Get current flag
     int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG );
     tmpFlag |= _CRTDBG_LEAK_CHECK_DF;