[wasm] Fix JIT helper calls modeled with the wrong return arity

src/coreclr/jit/compiler.h

@@ -3359,11 +3359,16 @@ class Compiler
         unsigned helper, var_types type, GenTree* arg1 = nullptr, GenTree* arg2 = nullptr, GenTree* arg3 = nullptr, GenTree* arg4 = nullptr);
 
 #ifdef TARGET_WASM
-    // On wasm these helpers return void* (InitHelpers.InitClass/InitInstantiatedClass). Model them as
-    // value-returning so the call_indirect signature matches the compiled managed helper; the value is unused.
+    // On wasm the emitted call signature is derived from the modeled return type, so helpers whose
+    // result we discard must still be modeled as value-returning. InitClass/InitInstantiatedClass
+    // return void*, CastHelpers.Unbox returns a byref.
     static constexpr var_types HelperInitClassRetType = TYP_I_IMPL;
+    // Only for unbox sites that discard the result. CastHelpers.Unbox returns a byref on every
+    // target, so a site that consumes the result must model TYP_BYREF directly.
+    static constexpr var_types HelperUnboxDiscardedRetType = TYP_BYREF;
 #else
-    static constexpr var_types HelperInitClassRetType = TYP_VOID;
+    static constexpr var_types HelperInitClassRetType      = TYP_VOID;
+    static constexpr var_types HelperUnboxDiscardedRetType = TYP_VOID;
 #endif // TARGET_WASM
 
     GenTreeCall* gtNewVirtualFunctionLookupHelperCallNode(

src/coreclr/jit/importer.cpp

@@ -10700,7 +10700,9 @@ void Compiler::impImportBlockCode(BasicBlock* block)
                     { // compDonotInline()
                         return;
                     }
-                    op1 = gtNewHelperCallNode(helper, TYP_VOID, op2, op1);
+                    // The helper result is unused; the byref is formed from clone + TARGET_POINTER_SIZE
+                    // below. Discard it so the enclosing COLON/QMARK remain void.
+                    op1 = gtUnusedValNode(gtNewHelperCallNode(helper, HelperUnboxDiscardedRetType, op2, op1));
 
                     op1 = new (this, GT_COLON) GenTreeColon(TYP_VOID, gtNewNothingNode(), op1);
                     op1 = gtNewQmarkNode(TYP_VOID, condBox, op1->AsColon());

src/coreclr/jit/morph.cpp

@@ -64,7 +64,7 @@ PhaseStatus Compiler::fgMorphInit()
             {
                 // confirm that the argument is a GC pointer (for debugging (GC stress))
                 GenTree* op = gtNewLclvNode(i, TYP_REF);
-                op          = gtNewHelperCallNode(CORINFO_HELP_CHECK_OBJ, TYP_VOID, op);
+                op          = gtUnusedValNode(gtNewHelperCallNode(CORINFO_HELP_CHECK_OBJ, TYP_REF, op));
 
                 fgNewStmtAtBeg(fgFirstBB, op);
                 madeChanges = true;

src/coreclr/jit/objectalloc.cpp

@@ -2837,7 +2837,11 @@ void ObjectAllocator::RewriteUses()
                         //
                         JITDUMP("Rewriting to invoke box type test helper%s\n", isForEffect ? " for side effect" : "");
 
+                        // Unbox_TypeTest returns void, unlike Unbox. Note isForEffect above uses the
+                        // original type.
                         call->gtCallMethHnd = m_compiler->eeFindHelper(CORINFO_HELP_UNBOX_TYPETEST);
+                        call->gtType        = TYP_VOID;
+                        call->gtReturnType  = TYP_VOID;
                         GenTree* const mt   = m_compiler->gtNewMethodTableLookup(lcl, /* onStack */ true);
                         call->gtArgs.Remove(secondArg);
                         call->gtArgs.PushBack(m_compiler, NewCallArg::Primitive(mt));

src/tests/JIT/Regression/JitBlue/Runtime_131377/Runtime_131377.cs

@@ -0,0 +1,55 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+// Regression test for a WebAssembly R2R (crossgen) codegen bug in object stack
+// allocation. When a non-escaping box is stack allocated, ObjectAllocator
+// rewrites the CORINFO_HELP_UNBOX call to CORINFO_HELP_UNBOX_TYPETEST. Unbox
+// returns a byref while Unbox_TypeTest returns void, but the call node kept its
+// byref type, so wasm emitted a value-returning call_indirect for a void helper
+// and the engine trapped with "function signature mismatch".
+//
+// The mismatched call_indirect is guarded by an inline method table check that
+// short-circuits the helper whenever the unbox type matches, so a same-type
+// unbox compiles the bad instruction but never executes it. Driving a
+// non-matching type through the same unbox site is what makes the slow path --
+// and the trap -- actually run.
+//
+// Reproduces only under crossgen wasm R2R (TargetOS=browser), where the unfixed
+// JIT aborts the module before InvalidCastException can be thrown. Passes
+// trivially on all other targets.
+
+using System;
+using System.Runtime.CompilerServices;
+using Xunit;
+
+public class Runtime_131377
+{
+    [Fact]
+    public static void TestEntryPoint()
+    {
+        // Matching type: the box is stack allocated and the inline method table
+        // check succeeds, so the type-test helper is skipped.
+        Unbox(null);
+
+        // Non-matching type: the inline check fails, so the type-test helper
+        // runs and reaches the call_indirect that trapped under the unfixed JIT.
+        Assert.Throws<InvalidCastException>(() => Unbox("not a guid"));
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void Unbox(object o)
+    {
+        if (o is null)
+        {
+            // Non-escaping box, so this is stack allocated.
+            o = new Guid();
+        }
+
+        Consume((Guid)o);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void Consume<T>(T _)
+    {
+    }
+}

src/tests/JIT/Regression/JitBlue/Runtime_131377/Runtime_131377.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <!-- Object stack allocation only runs with optimizations enabled, and the
+         rewrite it performs is what introduces the bad call signature. -->
+    <Optimize>True</Optimize>
+    <JitOptimizationSensitive>true</JitOptimizationSensitive>
+    <!-- The bug is wasm R2R (crossgen) codegen. On non-browser legs this runs as
+         ordinary JIT/interp and passes trivially (expected). On the browser leg,
+         AlwaysUseCrossGen2 forces crossgen (exports RunCrossGen2=1) so the test is
+         actually compiled to wasm R2R and run under node; otherwise a plain
+         browser leg runs it as normal JIT and silently false-passes without ever
+         exercising the fix. CrossGen2OutputFormat=wasm is set automatically for
+         TargetOS=browser (src/tests/Directory.Build.props). -->
+    <AlwaysUseCrossGen2 Condition="'$(TargetOS)' == 'browser'">true</AlwaysUseCrossGen2>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+  </ItemGroup>
+</Project>