[Autobackout][FuncReg]Revert of change: e7db2ba

 Fold bitcasts to rdregion/wrregion by changing region parameters

This transformation reorders rdregion->bitcast to bitcast->rdregion and
bitcast->wrregion to wrregion->bitcast, so they can be baled later.

Author: skachkov-intel

IGC/VectorCompiler/lib/GenXCodeGen/GenXLegalization.cpp

@@ -377,7 +377,7 @@ class GenXLegalization : public FunctionPass {
     Fixed4 = nullptr;
     TwiceWidth = nullptr;
   }
-  unsigned adjustTwiceWidthOrFixed4(const Bale &B);
+  unsigned getExecSizeAllowedBits(Instruction *Inst);
   bool checkIfLongLongSupportNeeded(Instruction *Inst) const;
   void verifyLSCFence(const Instruction *Inst);
   void verifyLSCAtomic(const Instruction *Inst);
@@ -575,30 +575,84 @@ bool GenXLegalization::runOnFunction(Function &F) {
   return true;
 }
 
-unsigned GenXLegalization::adjustTwiceWidthOrFixed4(const Bale &B) {
-  auto Main = B.getMainInst();
-  if (!Main)
-    return 0x3f;
-  // Spot whether we have a FIXED operand and/or a TWICEWIDTH operand.
-  if (GenXIntrinsic::isGenXIntrinsic(Main->Inst)) {
-    GenXIntrinsicInfo II(GenXIntrinsic::getAnyIntrinsicID(Main->Inst));
+/***********************************************************************
+ * getExecSizeAllowedBits : get bitmap of allowed execution sizes
+ *
+ * Enter:   Inst = main instruction of bale
+ *
+ * Return:  bit N set if execution size 1<<N is allowed
+ *
+ * Most instructions have a minimum width of 1. But some instructions,
+ * such as dp4 and lrp, have a minimum width of 4, and legalization cannot
+ * allow such an instruction to be split to a smaller width.
+ *
+ * This also sets up fields in GenXLegalization: Fixed4 is set to a use
+ * that is a FIXED4 operand, and TwiceWidth is set to a use that is a
+ * TWICEWIDTH operand.
+ */
+unsigned GenXLegalization::getExecSizeAllowedBits(Instruction *Inst) {
+
+  switch (Inst->getOpcode()) {
+  default:
+    break;
+  case BinaryOperator::SDiv:
+  case BinaryOperator::UDiv:
+  case BinaryOperator::SRem:
+  case BinaryOperator::URem:
+    // If integer division IS supported.
+    //   Set maximum SIMD width to 16:
+    //      Recent HW does not support SIMD16/SIMD32 division, however,
+    //      finalizer splits such SIMD16 operations and we piggy-back
+    //      on this behavior.
+    // If integer division IS NOT supported.
+    //   The expectation is for GenXEmulate pass to replace such operations
+    //   with emulation routines (which has no restriction on SIMD width)
+    return ST->hasIntDivRem32() ? 0x1f : 0x3f;
+  }
+
+  unsigned ID = GenXIntrinsic::getAnyIntrinsicID(Inst);
+  switch (ID) {
+    case GenXIntrinsic::genx_ssmad:
+    case GenXIntrinsic::genx_sumad:
+    case GenXIntrinsic::genx_usmad:
+    case GenXIntrinsic::genx_uumad:
+    case GenXIntrinsic::genx_ssmad_sat:
+    case GenXIntrinsic::genx_sumad_sat:
+    case GenXIntrinsic::genx_usmad_sat:
+    case GenXIntrinsic::genx_uumad_sat:
+    case Intrinsic::fma:
+      // Do not emit simd32 mad for pre-ICLLP.
+      return ST->isICLLPplus() ? 0x3f : 0x1f;
+    default:
+      break;
+  }
+
+  if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
+    // We have a call instruction, so we can assume it is an intrinsic since
+    // otherwise processInst would not have got as far as calling us as
+    // a non-intrinsic call forces isSplittable() to be false.
+    auto CalledF = CI->getCalledFunction();
+    IGC_ASSERT(CalledF);
+    GenXIntrinsicInfo II(GenXIntrinsic::getAnyIntrinsicID(CalledF));
+    // While we have the intrinsic info, we also spot whether we have a FIXED4
+    // operand and/or a TWICEWIDTH operand.
     for (auto i = II.begin(), e = II.end(); i != e; ++i) {
       auto ArgInfo = *i;
-      if (!ArgInfo.isArgOrRet())
-       continue;
-      switch (ArgInfo.getRestriction()) {
-      case GenXIntrinsicInfo::FIXED4:
-        Fixed4 = &Main->Inst->getOperandUse(ArgInfo.getArgIdx());
-        break;
-      case GenXIntrinsicInfo::TWICEWIDTH:
-        TwiceWidth = &Main->Inst->getOperandUse(ArgInfo.getArgIdx());
-        break;
+      if (ArgInfo.isArgOrRet()) {
+        switch (ArgInfo.getRestriction()) {
+        case GenXIntrinsicInfo::FIXED4:
+          Fixed4 = &CI->getOperandUse(ArgInfo.getArgIdx());
+          break;
+        case GenXIntrinsicInfo::TWICEWIDTH:
+          TwiceWidth = &CI->getOperandUse(ArgInfo.getArgIdx());
+          break;
+        }
       }
     }
+    return II.getExecSizeAllowedBits();
   }
-  return genx::getExecSizeAllowedBits(Main->Inst, ST);
+  return 0x3f;
 }
-
 /***********************************************************************
  * checkIfLongLongSupportNeeded: checks if an instruction requires
  *  target to support 64-bit integer operations
@@ -1328,7 +1382,9 @@ unsigned GenXLegalization::determineWidth(unsigned WholeWidth,
                                           unsigned StartIdx) {
   // Prepare to keep track of whether an instruction with a minimum width
   // (e.g. dp4) would be split too small, and whether we need to unbale.
-  unsigned ExecSizeAllowedBits = adjustTwiceWidthOrFixed4(B);
+  unsigned ExecSizeAllowedBits = 0x3f;
+  if (auto Main = B.getMainInst())
+    ExecSizeAllowedBits = getExecSizeAllowedBits(Main->Inst);
   unsigned MainInstMinWidth =
       1 << countTrailingZeros(ExecSizeAllowedBits, ZB_Undefined);
   // Determine the vector width that we need to split into.
@@ -2548,6 +2604,20 @@ static Value *createBitCastIfNeeded(Value *V, Type *NewTy,
   return Inst;
 }
 
+// Get type that represents OldTy as vector of NewScalarType.
+static Type *getNewVectorType(Type *OldTy, IntegerType *NewScalarType) {
+  IGC_ASSERT(OldTy->isIntOrIntVectorTy());
+  unsigned OldElemSize = OldTy->getScalarSizeInBits();
+  IGC_ASSERT(OldElemSize > 0);
+  auto *VTy = dyn_cast<IGCLLVM::FixedVectorType>(OldTy);
+  unsigned OldNumElems = VTy ? VTy->getNumElements() : 1;
+  unsigned NewElemSize = NewScalarType->getBitWidth();
+  if (OldElemSize * OldNumElems % NewElemSize)
+    return nullptr;
+  return IGCLLVM::FixedVectorType::get(NewScalarType,
+                         OldElemSize * OldNumElems / NewElemSize);
+}
+
 /***********************************************************************
  * transformMoveType : transform move bale to new integer type.
  *
@@ -2601,13 +2671,16 @@ Instruction *GenXLegalization::transformMoveType(Bale *B, IntegerType *FromTy,
     return nullptr;
   Region DstRgn = Wr ? makeRegionFromBaleInfo(Wr, BaleInfo()) : Region(Rd);
 
+  // Check that dst and src regions can be changed on new type.
+  if (SrcRgn.Indirect || DstRgn.Indirect || DstRgn.Mask)
+    return nullptr;
   // If destination region is not contiguous, changing element type can be achived
   // by conversion to 2D region, but we try to avoid it because such dst operands
   // are not supported in HW and require additional code for emulation.
   if (DstRgn.Stride != 1)
     return nullptr;
-  Type *NewSrcTy = genx::changeVectorType(Src->getType(), ToTy),
-       *NewDstTy = genx::changeVectorType(Dst->getType(), ToTy);
+  Type *NewSrcTy = getNewVectorType(Src->getType(), ToTy),
+       *NewDstTy = getNewVectorType(Dst->getType(), ToTy);
   if (!NewSrcTy || !NewDstTy)
     return nullptr;
 

IGC/VectorCompiler/lib/GenXCodeGen/GenXPostLegalization.cpp

@@ -138,7 +138,7 @@ bool GenXPostLegalization::runOnFunction(Function &F)
   // Run the vector decomposer for this function.
   Modified |= VD.run(DT);
   // Cleanup region reads and writes.
-  Modified |= simplifyRegionInsts(&F, DL, ST);
+  Modified |= simplifyRegionInsts(&F, DL);
   // Cleanup redundant global loads.
   Modified |= cleanupLoads(&F);
   // Cleanup constant loads.

IGC/VectorCompiler/lib/GenXCodeGen/GenXRegionUtils.cpp

@@ -24,10 +24,10 @@ SPDX-License-Identifier: MIT
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/Support/Debug.h"
 #include <unordered_map>
+#include "Probe/Assertion.h"
 
 #include "llvmWrapper/IR/DerivedTypes.h"
 #include "llvmWrapper/Support/TypeSize.h"
@@ -922,66 +922,14 @@ static Instruction* simplifyConstIndirectRegion(Instruction* Inst) {
   return Inst;
 }
 
-// fold bitcast with wrregion:
-//                                  ==>  %oldval.cast = bitcast(%oldval)
-// %2 = bitcast(%1)                      %3 = wrregion(%oldval.cast, %1, ...)
-// %3 = wrregion(%oldval, %2, ...)       %2 = bitcast(%3)
-// so it can be baled later.
-static Value *simplifyBitCastWithRegionWrite(Instruction *WrR,
-                                             const DataLayout &DL,
-                                             const GenXSubtarget &ST) {
-  using namespace GenXIntrinsic::GenXRegion;
-  IGC_ASSERT(GenXIntrinsic::isWrRegion(WrR));
-  Value *NewVal = WrR->getOperand(NewValueOperandNum);
-  auto *BCI = dyn_cast<BitCastInst>(NewVal);
-  if (!BCI)
-    return nullptr;
-  if (WrR->hasOneUse() && GenXIntrinsic::isWritePredefReg(WrR->user_back()))
-    return nullptr;
-  auto *NewScalarTy = BCI->getSrcTy()->getScalarType();
-  // Do not change register category to predicate.
-  if (NewScalarTy->isIntegerTy(1))
-    return nullptr;
-  auto *OldVal = WrR->getOperand(OldValueOperandNum);
-  if (GenXIntrinsic::isReadWritePredefReg(OldVal))
-    return nullptr;
-  auto *NewVecTy = genx::changeVectorType(OldVal->getType(), NewScalarTy);
-  if (!NewVecTy)
-    return nullptr;
-  Region R = makeRegionFromBaleInfo(WrR, BaleInfo());
-  if (!R.changeElementType(NewScalarTy, &DL))
-    return nullptr;
-  // Transformation is not profitable for 2D regions or if it will require
-  // legalization.
-  if (R.is2D() || R.NumElements > llvm::PowerOf2Floor(
-                                      genx::getExecSizeAllowedBits(WrR, &ST)))
-    return nullptr;
-  IRBuilder<> IRB(WrR);
-  auto *OldValCast =
-      IRB.CreateBitCast(OldVal, NewVecTy, OldVal->getName() + ".cast");
-  auto *NewWrR = R.createWrRegion(OldValCast, BCI->getOperand(0),
-                                  WrR->getName(), WrR, WrR->getDebugLoc());
-  auto *NewBCI = IRB.CreateBitCast(NewWrR, WrR->getType(), BCI->getName());
-  return NewBCI;
-}
+static Value *simplifyRegionWrite(Instruction *Inst) {
+  IGC_ASSERT(GenXIntrinsic::isWrRegion(Inst));
+  Value *NewVal = Inst->getOperand(GenXIntrinsic::GenXRegion::NewValueOperandNum);
 
-static Value *simplifyRegionWrite(Instruction *WrR) {
-  using namespace GenXIntrinsic::GenXRegion;
-  IGC_ASSERT(GenXIntrinsic::isWrRegion(WrR));
-  Value *NewVal = WrR->getOperand(NewValueOperandNum);
-
-  // Replace C with B if R - whole region
-  // C = wrregion(A, B, R)
-  if (std::none_of(
-          WrR->user_begin(), WrR->user_end(),
-          [](auto *U) { return GenXIntrinsic::isWritePredefReg(U); }) &&
-      makeRegionFromBaleInfo(WrR, BaleInfo()).isWhole(WrR->getType()) &&
-      NewVal->getType() == WrR->getType())
-    return NewVal;
   // Replace C with A
   // C = wrregion(A, undef, R)
   if (isa<UndefValue>(NewVal))
-    return WrR->getOperand(OldValueOperandNum);
+    return Inst->getOperand(GenXIntrinsic::GenXRegion::OldValueOperandNum);
 
   // When A and undef have the same type, replace C with A
   // B = rdregion(A, R)
@@ -993,68 +941,29 @@ static Value *simplifyRegionWrite(Instruction *WrR) {
   // C = wrregion(A, B, R)
   //
   if (GenXIntrinsic::isRdRegion(NewVal)) {
-    Instruction *RdR = cast<Instruction>(NewVal);
-    Region InnerR = makeRegionFromBaleInfo(RdR, BaleInfo());
-    Region OuterR = makeRegionFromBaleInfo(WrR, BaleInfo());
+    Instruction *B = cast<Instruction>(NewVal);
+    Region InnerR = makeRegionFromBaleInfo(B, BaleInfo());
+    Region OuterR = makeRegionFromBaleInfo(Inst, BaleInfo());
     if (OuterR != InnerR)
       return nullptr;
 
-    auto OldValRdR = RdR->getOperand(OldValueOperandNum);
-    if (GenXIntrinsic::isReadPredefReg(OldValRdR))
+    auto OldValB = B->getOperand(GenXIntrinsic::GenXRegion::OldValueOperandNum);
+    if (GenXIntrinsic::isReadPredefReg(OldValB))
       return nullptr;
-    auto OldValWrR = WrR->getOperand(OldValueOperandNum);
-    if ((isa<UndefValue>(OldValWrR) &&
-         OldValRdR->getType() == OldValWrR->getType()) ||
-        OldValRdR == OldValWrR)
-      return OldValRdR;
+    auto OldValC = Inst->getOperand(GenXIntrinsic::GenXRegion::OldValueOperandNum);
+    if ((isa<UndefValue>(OldValC) &&
+         OldValB->getType() == OldValC->getType()) ||
+        OldValB == OldValC)
+      return OldValB;
   }
-  return nullptr;
-}
 
-// fold bitcast with rdregion:
-// %2 = rdregion(%1, ...)  ==>  %3 = bitcast(%1)
-// %3 = bitcast(%2)             %2 = rdregion(%3, ...)
-// so it can be baled later.
-static Value *simplifyBitCastFromRegionRead(BitCastInst *BCI,
-                                            const DataLayout &DL,
-                                            const GenXSubtarget &ST) {
-  using namespace GenXIntrinsic::GenXRegion;
-  Instruction *RdR = dyn_cast<Instruction>(BCI->getOperand(0));
-  if (!RdR || !GenXIntrinsic::isRdRegion(RdR) || !RdR->hasOneUse())
-    return nullptr;
-  auto *OldVal = RdR->getOperand(OldValueOperandNum);
-  if (GenXIntrinsic::isReadPredefReg(OldVal))
-    return nullptr;
-  auto *NewScalarTy = BCI->getDestTy()->getScalarType();
-  // Do not change register category to predicate.
-  if (NewScalarTy->isIntegerTy(1))
-    return nullptr;
-  auto *NewVecTy = genx::changeVectorType(OldVal->getType(), NewScalarTy);
-  if (!NewVecTy)
-    return nullptr;
-  Region R = makeRegionFromBaleInfo(RdR, BaleInfo());
-  if (!R.changeElementType(NewScalarTy, &DL))
-    return nullptr;
-  // Transformation is not profitable for 2D regions or if it will require
-  // legalization.
-  if (R.is2D() || R.NumElements > llvm::PowerOf2Floor(
-                                      genx::getExecSizeAllowedBits(RdR, &ST)))
-    return nullptr;
-  auto *NewBCI =
-      IRBuilder<>(BCI).CreateBitCast(OldVal, NewVecTy, BCI->getName());
-  auto *NewRdR =
-      R.createRdRegion(NewBCI, RdR->getName(), BCI, RdR->getDebugLoc());
-  return NewRdR;
+  return nullptr;
 }
 
 static Value *simplifyRegionRead(Instruction *Inst) {
   IGC_ASSERT(GenXIntrinsic::isRdRegion(Inst));
   Value *Input = Inst->getOperand(GenXIntrinsic::GenXRegion::OldValueOperandNum);
-  if (!GenXIntrinsic::isReadPredefReg(Input) &&
-      makeRegionFromBaleInfo(Inst, BaleInfo()).isWhole(Input->getType()) &&
-      Input->getType() == Inst->getType())
-    return Input;
-  else if (isa<UndefValue>(Input))
+  if (isa<UndefValue>(Input))
     return UndefValue::get(Inst->getType());
   else if (auto C = dyn_cast<Constant>(Input)) {
     if (auto Splat = C->getSplatValue()) {
@@ -1081,8 +990,7 @@ static Value *simplifyRegionRead(Instruction *Inst) {
 }
 
 // Simplify a region read or write.
-Value *llvm::genx::simplifyRegionInst(Instruction *Inst, const DataLayout *DL,
-                                      const GenXSubtarget *ST) {
+Value *llvm::genx::simplifyRegionInst(Instruction *Inst, const DataLayout *DL) {
   if (Inst->use_empty())
     return nullptr;
 
@@ -1105,17 +1013,11 @@ Value *llvm::genx::simplifyRegionInst(Instruction *Inst, const DataLayout *DL,
   if (Constant *C = ConstantFoldGenX(Inst, *DL))
     return C;
 
-  if (auto *BCI = dyn_cast<BitCastInst>(Inst); BCI && DL && ST)
-    return simplifyBitCastFromRegionRead(BCI, *DL, *ST);
   ID = GenXIntrinsic::getGenXIntrinsicID(Inst);
   switch (ID) {
   case GenXIntrinsic::genx_wrregionf:
   case GenXIntrinsic::genx_wrregioni:
-    if (auto *Res = simplifyRegionWrite(Inst))
-      return Res;
-    if (DL && ST)
-      return simplifyBitCastWithRegionWrite(Inst, *DL, *ST);
-    break;
+    return simplifyRegionWrite(Inst);
   case GenXIntrinsic::genx_rdregionf:
   case GenXIntrinsic::genx_rdregioni:
     return simplifyRegionRead(Inst);
@@ -1125,13 +1027,12 @@ Value *llvm::genx::simplifyRegionInst(Instruction *Inst, const DataLayout *DL,
   return nullptr;
 }
 
-bool llvm::genx::simplifyRegionInsts(Function *F, const DataLayout *DL,
-                                     const GenXSubtarget *ST) {
+bool llvm::genx::simplifyRegionInsts(Function *F, const DataLayout *DL) {
   bool Changed = false;
   for (auto &BB : F->getBasicBlockList()) {
     for (auto I = BB.begin(); I != BB.end();) {
       Instruction *Inst = &*I++;
-      if (auto V = simplifyRegionInst(Inst, DL, ST)) {
+      if (auto V = simplifyRegionInst(Inst, DL)) {
         Inst->replaceAllUsesWith(V);
         Inst->eraseFromParent();
         Changed = true;

IGC/VectorCompiler/lib/GenXCodeGen/GenXRegionUtils.h

@@ -162,10 +162,8 @@ inline raw_ostream &operator<<(raw_ostream &OS, const RdWrRegionSequence &RWS) {
   return OS;
 }
 
-Value *simplifyRegionInst(Instruction *Inst, const DataLayout *DL = nullptr,
-                          const GenXSubtarget *ST = nullptr);
-bool simplifyRegionInsts(Function *F, const DataLayout *DL = nullptr,
-                         const GenXSubtarget *ST = nullptr);
+Value *simplifyRegionInst(Instruction *Inst, const DataLayout *DL);
+bool simplifyRegionInsts(Function *F, const DataLayout *DL);
 
 bool cleanupLoads(Function *F);