diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index e042d5f09bd8f5..ce1eb57bf0c1da 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -1145,8 +1145,9 @@ AssertionIndex Compiler::optCreateAssertion(GenTree* op1, GenTree* op2, bool equ #if defined(FEATURE_HW_INTRINSICS) case GT_CNS_VEC: { - // For now, only support SIMD constants up to 16 bytes (SIMD8/12/16). - if (!op1->TypeIs(TYP_SIMD8, TYP_SIMD12, TYP_SIMD16) || (op1->TypeGet() != op2->TypeGet())) + // Support all SIMD constants. SIMD8/12/16 are stored inline in the assertion; + // SIMD32/64 are heap-allocated. + if (!varTypeIsSIMD(op1) || (op1->TypeGet() != op2->TypeGet())) { return NO_ASSERTION_INDEX; } @@ -1159,11 +1160,8 @@ AssertionIndex Compiler::optCreateAssertion(GenTree* op1, GenTree* op2, bool equ return NO_ASSERTION_INDEX; } - simd16_t simdVal = {}; - memcpy(&simdVal, &op2->AsVecCon()->gtSimdVal, genTypeSize(op2->TypeGet())); - AssertionDsc dsc = - AssertionDsc::CreateConstLclVarAssertion(this, lclNum, op1VN, simdVal, op2VN, equals); + AssertionDsc::CreateConstLclVarAssertion(this, lclNum, op1VN, op2->AsVecCon(), op2VN, equals); return optAddAssertion(dsc); } #endif // FEATURE_HW_INTRINSICS @@ -1871,6 +1869,8 @@ AssertionInfo Compiler::optAssertionGenJtrue(GenTree* tree) { #if defined(TARGET_XARCH) case NI_Vector128_op_Equality: + case NI_Vector256_op_Equality: + case NI_Vector512_op_Equality: #elif defined(TARGET_ARM64) case NI_Vector64_op_Equality: case NI_Vector128_op_Equality: @@ -1878,6 +1878,8 @@ AssertionInfo Compiler::optAssertionGenJtrue(GenTree* tree) break; #if defined(TARGET_XARCH) case NI_Vector128_op_Inequality: + case NI_Vector256_op_Inequality: + case NI_Vector512_op_Inequality: #elif defined(TARGET_ARM64) case NI_Vector64_op_Inequality: case NI_Vector128_op_Inequality: @@ -1902,7 +1904,7 @@ AssertionInfo Compiler::optAssertionGenJtrue(GenTree* tree) return NO_ASSERTION_INDEX; } - assert(op1->TypeIs(TYP_SIMD8, TYP_SIMD12, TYP_SIMD16)); + assert(varTypeIsSIMD(op1)); assert(op1->TypeIs(op2->TypeGet())); } else @@ -3263,15 +3265,15 @@ GenTree* Compiler::optConstantAssertionProp(const AssertionDsc& curAssertion, case O2K_CONST_VEC: { // The assertion was created from a LCL_VAR == CNS_VEC where types matched. - // For now, only support SIMD constants up to 16 bytes (SIMD8/12/16). - if (!tree->TypeIs(TYP_SIMD8, TYP_SIMD12, TYP_SIMD16) || !tree->TypeIs(lvaGetDesc(lclNum)->TypeGet())) + if (!varTypeIsSIMD(tree) || !tree->TypeIs(lvaGetDesc(lclNum)->TypeGet())) { return nullptr; } + assert(genTypeSize(tree->TypeGet()) == curAssertion.GetOp2().GetSimdSize()); // We can't bash a LCL_VAR into a GenTreeVecCon (different node size), so allocate a fresh node. GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); - memcpy(&vecCon->gtSimdVal, &curAssertion.GetOp2().GetSimdConstant(), genTypeSize(tree->TypeGet())); + memcpy(&vecCon->gtSimdVal, curAssertion.GetOp2().GetSimdConstant(), genTypeSize(tree->TypeGet())); newTree = vecCon; break; } diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 4498ff45dd4087..c5341a95bfc0e8 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8041,15 +8041,19 @@ class Compiler INDEBUG(const Compiler* m_compiler); optOp2Kind m_kind; bool m_checkedBoundIsNeverNegative; // only meaningful for O2K_CHECKED_BOUND_ADD_CNS kind - uint16_t m_encodedIconFlags; // encoded icon gtFlags - ValueNum m_vn; + union + { + uint16_t m_encodedIconFlags; // encoded icon gtFlags; only meaningful for O2K_CONST_INT. + uint8_t m_simdSize; // SIMD constant size in bytes; only meaningful for O2K_CONST_VEC. + }; + ValueNum m_vn; union { unsigned m_lclNum; double m_dconVal; IntegralRange m_range; - simd16_t m_simdVal; // for O2K_CONST_VEC (TYP_SIMD8/12/16 only). TODO-CQ: support wider SIMD via heap - // allocation. + simd16_t m_simdVal; // for O2K_CONST_VEC, inline storage for TYP_SIMD8/12/16. + simd_t* m_bigSimdVal; // for O2K_CONST_VEC, heap-allocated storage for TYP_SIMD32/64. struct { ssize_t m_iconVal; @@ -8082,10 +8086,22 @@ class Compiler return m_dconVal; } - const simd16_t& GetSimdConstant() const + // Returns a pointer to the SIMD constant payload. The valid byte length is GetSimdSize(). + // For TYP_SIMD8/12/16 the storage is inline; for TYP_SIMD32/64 it is heap-allocated. + const void* GetSimdConstant() const + { + assert(KindIs(O2K_CONST_VEC)); + if (m_simdSize <= sizeof(simd16_t)) + { + return &m_simdVal; + } + return m_bigSimdVal; + } + + unsigned GetSimdSize() const { assert(KindIs(O2K_CONST_VEC)); - return m_simdVal; + return m_simdSize; } ssize_t GetIntConstant() const @@ -8441,7 +8457,12 @@ class Compiler case O2K_CONST_VEC: // memcmp the full stored payload; GenTreeVecCon zero-inits gtSimdVal before populating. - return (memcmp(&GetOp2().m_simdVal, &that.GetOp2().m_simdVal, sizeof(simd16_t)) == 0); + if (GetOp2().GetSimdSize() != that.GetOp2().GetSimdSize()) + { + return false; + } + return (memcmp(GetOp2().GetSimdConstant(), that.GetOp2().GetSimdConstant(), + GetOp2().GetSimdSize()) == 0); case O2K_ZEROOBJ: return true; @@ -8482,14 +8503,14 @@ class Compiler // Create a generic "lclNum ==/!= constant" or "vn ==/!= constant" assertion template - static AssertionDsc CreateConstLclVarAssertion(const Compiler* comp, - unsigned lclNum, - ValueNum vn, - const T& cns, - ValueNum cnsVN, - bool equals, - GenTreeFlags iconFlags = GTF_EMPTY, - FieldSeq* fldSeq = nullptr) + static AssertionDsc CreateConstLclVarAssertion(Compiler* comp, + unsigned lclNum, + ValueNum vn, + const T& cns, + ValueNum cnsVN, + bool equals, + GenTreeFlags iconFlags = GTF_EMPTY, + FieldSeq* fldSeq = nullptr) { AssertionDsc dsc = CreateEmptyAssertion(comp); dsc.m_assertionKind = equals ? OAK_EQUAL : OAK_NOT_EQUAL; @@ -8534,14 +8555,33 @@ class Compiler assert(iconFlags == GTF_EMPTY); // no flags expected for double constants assert(fldSeq == nullptr); // no fieldSeq expected for double constants } - else if constexpr (std::is_same_v) +#if defined(FEATURE_SIMD) + else if constexpr (std::is_same_v) { - dsc.m_op2.m_kind = O2K_CONST_VEC; - dsc.m_op2.m_simdVal = {}; - memcpy(&dsc.m_op2.m_simdVal, &cns, sizeof(simd16_t)); + dsc.m_op2.m_kind = O2K_CONST_VEC; + + assert(varTypeIsSIMD(cns)); + const unsigned simdSize = genTypeSize(cns->TypeGet()); + dsc.m_op2.m_simdSize = static_cast(simdSize); + + if (simdSize <= sizeof(simd16_t)) + { + dsc.m_op2.m_simdVal = {}; + memcpy(&dsc.m_op2.m_simdVal, &cns->gtSimdVal, simdSize); + } + else + { + // Heap-allocate storage for SIMD32/SIMD64 constants. The AssertionProp allocator uses the + // compiler arena, so the lifetime matches the assertion table. + simd_t* heapVal = new (comp, CMK_AssertionProp) simd_t(); + memset(heapVal, 0, sizeof(simd_t)); + memcpy(heapVal, &cns->gtSimdVal, simdSize); + dsc.m_op2.m_bigSimdVal = heapVal; + } assert(iconFlags == GTF_EMPTY); // no flags expected for vector constants assert(fldSeq == nullptr); // no fieldSeq expected for vector constants } +#endif // FEATURE_SIMD else if constexpr (std::is_same_v) { dsc.m_op2.m_kind = static_cast(cns); @@ -8558,7 +8598,7 @@ class Compiler } // Create "lclNum != null" assertion - static AssertionDsc CreateLclNonNullAssertion(const Compiler* comp, unsigned lclNum) + static AssertionDsc CreateLclNonNullAssertion(Compiler* comp, unsigned lclNum) { assert(comp->optLocalAssertionProp); return CreateConstLclVarAssertion(comp, lclNum, ValueNumStore::NoVN, 0, ValueNumStore::VNForNull(), @@ -8566,7 +8606,7 @@ class Compiler } // Create "vn != null" assertion - static AssertionDsc CreateVNNonNullAssertion(const Compiler* comp, ValueNum vn) + static AssertionDsc CreateVNNonNullAssertion(Compiler* comp, ValueNum vn) { assert(!comp->optLocalAssertionProp); return CreateConstLclVarAssertion(comp, BAD_VAR_NUM, vn, 0, ValueNumStore::VNForNull(),