From 4ba5a7824cb34c3b53ea4bd40b7ebb654710ce96 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 24 Apr 2026 17:48:41 +0200 Subject: [PATCH 1/4] JIT: support SIMD32/64 assertions via heap-allocated payload Extend the O2K_CONST_VEC assertion storage to handle TYP_SIMD32/64 via a heap-allocated simd_t* (m_bigSimdVal) in the existing union. SIMD8/12/16 continue to use the inline simd16_t. The active SIMD byte size is tracked in m_simdSize, unioned with m_encodedIconFlags to keep struct size unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/assertionprop.cpp | 16 +++-- src/coreclr/jit/compiler.h | 100 ++++++++++++++++++++++++------ 2 files changed, 87 insertions(+), 29 deletions(-) diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index e042d5f09bd8f5..702a534a284961 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -1145,8 +1145,9 @@ AssertionIndex Compiler::optCreateAssertion(GenTree* op1, GenTree* op2, bool equ #if defined(FEATURE_HW_INTRINSICS) case GT_CNS_VEC: { - // For now, only support SIMD constants up to 16 bytes (SIMD8/12/16). - if (!op1->TypeIs(TYP_SIMD8, TYP_SIMD12, TYP_SIMD16) || (op1->TypeGet() != op2->TypeGet())) + // Support all SIMD constants. SIMD8/12/16 are stored inline in the assertion; + // SIMD32/64 are heap-allocated. + if (!varTypeIsSIMD(op1) || (op1->TypeGet() != op2->TypeGet())) { return NO_ASSERTION_INDEX; } @@ -1159,11 +1160,8 @@ AssertionIndex Compiler::optCreateAssertion(GenTree* op1, GenTree* op2, bool equ return NO_ASSERTION_INDEX; } - simd16_t simdVal = {}; - memcpy(&simdVal, &op2->AsVecCon()->gtSimdVal, genTypeSize(op2->TypeGet())); - AssertionDsc dsc = - AssertionDsc::CreateConstLclVarAssertion(this, lclNum, op1VN, simdVal, op2VN, equals); + AssertionDsc::CreateConstLclVarAssertion(this, lclNum, op1VN, op2->AsVecCon(), op2VN, equals); return optAddAssertion(dsc); } #endif // FEATURE_HW_INTRINSICS @@ -3263,15 +3261,15 @@ GenTree* Compiler::optConstantAssertionProp(const AssertionDsc& curAssertion, case O2K_CONST_VEC: { // The assertion was created from a LCL_VAR == CNS_VEC where types matched. - // For now, only support SIMD constants up to 16 bytes (SIMD8/12/16). - if (!tree->TypeIs(TYP_SIMD8, TYP_SIMD12, TYP_SIMD16) || !tree->TypeIs(lvaGetDesc(lclNum)->TypeGet())) + if (!varTypeIsSIMD(tree) || !tree->TypeIs(lvaGetDesc(lclNum)->TypeGet())) { return nullptr; } + assert(genTypeSize(tree->TypeGet()) == curAssertion.GetOp2().GetSimdSize()); // We can't bash a LCL_VAR into a GenTreeVecCon (different node size), so allocate a fresh node. GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); - memcpy(&vecCon->gtSimdVal, &curAssertion.GetOp2().GetSimdConstant(), genTypeSize(tree->TypeGet())); + memcpy(&vecCon->gtSimdVal, curAssertion.GetOp2().GetSimdConstant(), genTypeSize(tree->TypeGet())); newTree = vecCon; break; } diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 4498ff45dd4087..ac38c18741a489 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8041,15 +8041,19 @@ class Compiler INDEBUG(const Compiler* m_compiler); optOp2Kind m_kind; bool m_checkedBoundIsNeverNegative; // only meaningful for O2K_CHECKED_BOUND_ADD_CNS kind - uint16_t m_encodedIconFlags; // encoded icon gtFlags + union + { + uint16_t m_encodedIconFlags; // encoded icon gtFlags; only meaningful for O2K_CONST_INT. + uint8_t m_simdSize; // SIMD constant size in bytes; only meaningful for O2K_CONST_VEC. + }; ValueNum m_vn; union { unsigned m_lclNum; double m_dconVal; IntegralRange m_range; - simd16_t m_simdVal; // for O2K_CONST_VEC (TYP_SIMD8/12/16 only). TODO-CQ: support wider SIMD via heap - // allocation. + simd16_t m_simdVal; // for O2K_CONST_VEC, inline storage for TYP_SIMD8/12/16. + simd_t* m_bigSimdVal; // for O2K_CONST_VEC, heap-allocated storage for TYP_SIMD32/64. struct { ssize_t m_iconVal; @@ -8082,10 +8086,22 @@ class Compiler return m_dconVal; } - const simd16_t& GetSimdConstant() const + // Returns a pointer to the SIMD constant payload. The valid byte length is GetSimdSize(). + // For TYP_SIMD8/12/16 the storage is inline; for TYP_SIMD32/64 it is heap-allocated. + const void* GetSimdConstant() const { assert(KindIs(O2K_CONST_VEC)); - return m_simdVal; + if (m_simdSize <= sizeof(simd16_t)) + { + return &m_simdVal; + } + return m_bigSimdVal; + } + + unsigned GetSimdSize() const + { + assert(KindIs(O2K_CONST_VEC)); + return m_simdSize; } ssize_t GetIntConstant() const @@ -8441,7 +8457,12 @@ class Compiler case O2K_CONST_VEC: // memcmp the full stored payload; GenTreeVecCon zero-inits gtSimdVal before populating. - return (memcmp(&GetOp2().m_simdVal, &that.GetOp2().m_simdVal, sizeof(simd16_t)) == 0); + if (GetOp2().GetSimdSize() != that.GetOp2().GetSimdSize()) + { + return false; + } + return (memcmp(GetOp2().GetSimdConstant(), that.GetOp2().GetSimdConstant(), + GetOp2().GetSimdSize()) == 0); case O2K_ZEROOBJ: return true; @@ -8482,14 +8503,14 @@ class Compiler // Create a generic "lclNum ==/!= constant" or "vn ==/!= constant" assertion template - static AssertionDsc CreateConstLclVarAssertion(const Compiler* comp, - unsigned lclNum, - ValueNum vn, - const T& cns, - ValueNum cnsVN, - bool equals, - GenTreeFlags iconFlags = GTF_EMPTY, - FieldSeq* fldSeq = nullptr) + static AssertionDsc CreateConstLclVarAssertion(Compiler* comp, + unsigned lclNum, + ValueNum vn, + const T& cns, + ValueNum cnsVN, + bool equals, + GenTreeFlags iconFlags = GTF_EMPTY, + FieldSeq* fldSeq = nullptr) { AssertionDsc dsc = CreateEmptyAssertion(comp); dsc.m_assertionKind = equals ? OAK_EQUAL : OAK_NOT_EQUAL; @@ -8534,11 +8555,50 @@ class Compiler assert(iconFlags == GTF_EMPTY); // no flags expected for double constants assert(fldSeq == nullptr); // no fieldSeq expected for double constants } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { - dsc.m_op2.m_kind = O2K_CONST_VEC; - dsc.m_op2.m_simdVal = {}; - memcpy(&dsc.m_op2.m_simdVal, &cns, sizeof(simd16_t)); + dsc.m_op2.m_kind = O2K_CONST_VEC; + + // genTypeSize() lives in compiler.hpp and isn't visible here, so map TYP_SIMDxx to byte size inline. + unsigned simdSize; + switch (cns->TypeGet()) + { + case TYP_SIMD8: + simdSize = 8; + break; + case TYP_SIMD12: + simdSize = 12; + break; + case TYP_SIMD16: + simdSize = 16; + break; +#if defined(TARGET_XARCH) + case TYP_SIMD32: + simdSize = 32; + break; + case TYP_SIMD64: + simdSize = 64; + break; +#endif // TARGET_XARCH + default: + unreached(); + } + dsc.m_op2.m_simdSize = static_cast(simdSize); + + if (simdSize <= sizeof(simd16_t)) + { + dsc.m_op2.m_simdVal = {}; + memcpy(&dsc.m_op2.m_simdVal, &cns->gtSimdVal, simdSize); + } + else + { + // Heap-allocate storage for SIMD32/SIMD64 constants. The AssertionProp allocator uses the + // compiler arena, so the lifetime matches the assertion table. + simd_t* heapVal = new (comp, CMK_AssertionProp) simd_t(); + memset(heapVal, 0, sizeof(simd_t)); + memcpy(heapVal, &cns->gtSimdVal, simdSize); + dsc.m_op2.m_bigSimdVal = heapVal; + } assert(iconFlags == GTF_EMPTY); // no flags expected for vector constants assert(fldSeq == nullptr); // no fieldSeq expected for vector constants } @@ -8558,7 +8618,7 @@ class Compiler } // Create "lclNum != null" assertion - static AssertionDsc CreateLclNonNullAssertion(const Compiler* comp, unsigned lclNum) + static AssertionDsc CreateLclNonNullAssertion(Compiler* comp, unsigned lclNum) { assert(comp->optLocalAssertionProp); return CreateConstLclVarAssertion(comp, lclNum, ValueNumStore::NoVN, 0, ValueNumStore::VNForNull(), @@ -8566,7 +8626,7 @@ class Compiler } // Create "vn != null" assertion - static AssertionDsc CreateVNNonNullAssertion(const Compiler* comp, ValueNum vn) + static AssertionDsc CreateVNNonNullAssertion(Compiler* comp, ValueNum vn) { assert(!comp->optLocalAssertionProp); return CreateConstLclVarAssertion(comp, BAD_VAR_NUM, vn, 0, ValueNumStore::VNForNull(), From 97158686fdefcc361a86b6d50b6ee40624704630 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 24 Apr 2026 17:59:17 +0200 Subject: [PATCH 2/4] JIT: enable assertion creation for Vector256/Vector512 op_Equality/Inequality Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/assertionprop.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index 702a534a284961..ce1eb57bf0c1da 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -1869,6 +1869,8 @@ AssertionInfo Compiler::optAssertionGenJtrue(GenTree* tree) { #if defined(TARGET_XARCH) case NI_Vector128_op_Equality: + case NI_Vector256_op_Equality: + case NI_Vector512_op_Equality: #elif defined(TARGET_ARM64) case NI_Vector64_op_Equality: case NI_Vector128_op_Equality: @@ -1876,6 +1878,8 @@ AssertionInfo Compiler::optAssertionGenJtrue(GenTree* tree) break; #if defined(TARGET_XARCH) case NI_Vector128_op_Inequality: + case NI_Vector256_op_Inequality: + case NI_Vector512_op_Inequality: #elif defined(TARGET_ARM64) case NI_Vector64_op_Inequality: case NI_Vector128_op_Inequality: @@ -1900,7 +1904,7 @@ AssertionInfo Compiler::optAssertionGenJtrue(GenTree* tree) return NO_ASSERTION_INDEX; } - assert(op1->TypeIs(TYP_SIMD8, TYP_SIMD12, TYP_SIMD16)); + assert(varTypeIsSIMD(op1)); assert(op1->TypeIs(op2->TypeGet())); } else From 6615f82d04d6752d9357a701d7c65c1f742cdc20 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 24 Apr 2026 18:02:50 +0200 Subject: [PATCH 3/4] JIT: simplify SIMD size lookup using genTypeSize Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/compiler.h | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index ac38c18741a489..9eab71ce27da15 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8559,31 +8559,9 @@ class Compiler { dsc.m_op2.m_kind = O2K_CONST_VEC; - // genTypeSize() lives in compiler.hpp and isn't visible here, so map TYP_SIMDxx to byte size inline. - unsigned simdSize; - switch (cns->TypeGet()) - { - case TYP_SIMD8: - simdSize = 8; - break; - case TYP_SIMD12: - simdSize = 12; - break; - case TYP_SIMD16: - simdSize = 16; - break; -#if defined(TARGET_XARCH) - case TYP_SIMD32: - simdSize = 32; - break; - case TYP_SIMD64: - simdSize = 64; - break; -#endif // TARGET_XARCH - default: - unreached(); - } - dsc.m_op2.m_simdSize = static_cast(simdSize); + assert(varTypeIsSIMD(cns)); + const unsigned simdSize = genTypeSize(cns->TypeGet()); + dsc.m_op2.m_simdSize = static_cast(simdSize); if (simdSize <= sizeof(simd16_t)) { From 0327f8a85f7f409add9a35d0b91678aa555736a4 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 24 Apr 2026 13:43:48 -0700 Subject: [PATCH 4/4] Fix build failures --- src/coreclr/jit/compiler.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 9eab71ce27da15..c5341a95bfc0e8 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8046,7 +8046,7 @@ class Compiler uint16_t m_encodedIconFlags; // encoded icon gtFlags; only meaningful for O2K_CONST_INT. uint8_t m_simdSize; // SIMD constant size in bytes; only meaningful for O2K_CONST_VEC. }; - ValueNum m_vn; + ValueNum m_vn; union { unsigned m_lclNum; @@ -8555,6 +8555,7 @@ class Compiler assert(iconFlags == GTF_EMPTY); // no flags expected for double constants assert(fldSeq == nullptr); // no fieldSeq expected for double constants } +#if defined(FEATURE_SIMD) else if constexpr (std::is_same_v) { dsc.m_op2.m_kind = O2K_CONST_VEC; @@ -8580,6 +8581,7 @@ class Compiler assert(iconFlags == GTF_EMPTY); // no flags expected for vector constants assert(fldSeq == nullptr); // no fieldSeq expected for vector constants } +#endif // FEATURE_SIMD else if constexpr (std::is_same_v) { dsc.m_op2.m_kind = static_cast(cns);