Add functions to enable and disable MidThread Preemption

JaroszPiotr · igcbot · commit cf0c745e9368 · 2022-04-08T15:19:23.000+02:00
This changes adds functions which allow to enable and disable
MidThread Preemption during the shader execution.
diff --git a/IGC/AdaptorCommon/RayTracing/RTBuilder.cpp b/IGC/AdaptorCommon/RayTracing/RTBuilder.cpp
@@ -597,6 +597,7 @@ Value* RTBuilder::CreateSyncStackPtrIntrinsic(
     return StackPtr;
 }
 
+
 RTBuilder::SWStackPtrVal* RTBuilder::getSWStackPointer(const Twine& Name)
 {
     auto* CI = this->CreateSWStackPtrIntrinsic(nullptr, false, Name);
diff --git a/IGC/AdaptorCommon/RayTracing/RTBuilder.h b/IGC/AdaptorCommon/RayTracing/RTBuilder.h
@@ -385,6 +385,8 @@ class RTBuilder : public IGCIRBuilder<>
     Value* CreateSWHotZonePtrIntrinsic(Value *Addr, Type *PtrTy, bool AddDecoration);
     Value* CreateAsyncStackPtrIntrinsic(Value *Addr, Type *PtrTy, bool AddDecoration);
     Value* CreateSyncStackPtrIntrinsic(Value* Addr, Type* PtrTy, bool AddDecoration);
+
+
     CallInst* CreateSWStackPtrIntrinsic(
         Value *Addr, bool AddDecoration, const Twine &Name = "");
     SWStackPtrVal* getSWStackPointer(
@@ -705,12 +707,6 @@ class RTBuilder : public IGCIRBuilder<>
         uint32_t dim,
         IGC::CallableShaderTypeMD ShaderTy);
 
-    Value* getTraceRayPayload(
-        Value* bvhLevel,
-        Value* traceRayCtrl,
-        bool isRayQuery,
-        const Twine& PayloadName = "");
-
     Value* emitStateRegID(uint32_t BitStart, uint32_t BitEnd);
     Value* getSliceID();
     Value* getSubsliceID();
@@ -727,6 +723,13 @@ class RTBuilder : public IGCIRBuilder<>
     const IGC::RayDispatchShaderContext& RtCtx() const;
 //printf
 public:
+
+    Value* getTraceRayPayload(
+        Value* bvhLevel,
+        Value* traceRayCtrl,
+        bool isRayQuery,
+        const Twine& PayloadName = "");
+
     void printTraceRay(const TraceRayAsyncHLIntrinsic* trace);
     void printDispatchRayIndex(const std::vector<Value*>& Indices);
 public:
diff --git a/IGC/Compiler/CISACodeGen/CISABuilder.cpp b/IGC/Compiler/CISACodeGen/CISABuilder.cpp
@@ -3006,6 +3006,61 @@ namespace IGC
         return RMEncoding::RoundToZero_int;
     }
 
+    CEncoder::PreemptionEncoding CEncoder::getEncoderPreemptionMode(EPreemptionMode preemptionMode)
+    {
+        switch (preemptionMode)
+        {
+        default:
+            break;
+        case PREEMPTION_ENABLED:
+            return PreemptionEncoding::PreemptionEnabled;
+        case PREEMPTION_DISABLED:
+            return PreemptionEncoding::PreemptionDisabled;
+        }
+
+        return PreemptionEncoding::PreemptionEnabled;
+    }
+
+    void CEncoder::SetPreemptionMode(EPreemptionMode actualPreemptionMode, EPreemptionMode newPreemptionMode)
+    {
+        if (actualPreemptionMode != newPreemptionMode)
+        {
+            PreemptionEncoding actualPreemptionMode_en = getEncoderPreemptionMode(actualPreemptionMode);
+            PreemptionEncoding newPreemptionMode_en = getEncoderPreemptionMode(newPreemptionMode);
+            SetPreemptionMode(actualPreemptionMode_en, newPreemptionMode_en);
+        }
+    }
+
+    void CEncoder::SetPreemptionMode(PreemptionEncoding actualPreemptionMode, PreemptionEncoding newPreemptionMode)
+    {
+        IGC_ASSERT_MESSAGE(
+            (actualPreemptionMode != newPreemptionMode),
+            "Only setting PreemptionMode if the new PreemptionMode is different from the current PreemptionMode!");
+
+        VISA_VectorOpnd* src0_Opnd = nullptr;
+        VISA_VectorOpnd* src1_Opnd = nullptr;
+        VISA_VectorOpnd* dst_Opnd = nullptr;
+        VISA_GenVar* cr0_var = nullptr;
+
+        uint preemptionMode = actualPreemptionMode ^ newPreemptionMode;
+
+        IGC_ASSERT(nullptr != vKernel);
+
+        V(vKernel->GetPredefinedVar(cr0_var, PREDEFINED_CR0));
+        V(vKernel->CreateVISASrcOperand(src0_Opnd, cr0_var, MODIFIER_NONE, 0, 1, 0, 0, 0));
+        V(vKernel->CreateVISAImmediate(src1_Opnd, &preemptionMode, ISA_TYPE_UD));
+        V(vKernel->CreateVISADstOperand(dst_Opnd, cr0_var, 1, 0, 0));
+        V(vKernel->AppendVISAArithmeticInst(
+            ISA_XOR,
+            nullptr,
+            false,
+            vISA_EMASK_M1_NM,
+            EXEC_SIZE_1,
+            dst_Opnd,
+            src0_Opnd,
+            src1_Opnd));
+    }
+
     VISA_LabelOpnd* CEncoder::GetLabel(uint label)
     {
         VISA_LabelOpnd* visaLabel = labelMap[label];
diff --git a/IGC/Compiler/CISACodeGen/CISABuilder.hpp b/IGC/Compiler/CISACodeGen/CISABuilder.hpp
@@ -471,6 +471,8 @@ namespace IGC
         void SetRoundingMode_FP(ERoundingMode actualRM, ERoundingMode newRM);
         void SetRoundingMode_FPCvtInt(ERoundingMode actualRM, ERoundingMode newRM);
 
+        void SetPreemptionMode(EPreemptionMode actualPreemptionMode, EPreemptionMode newPreemptionMode);
+
         static uint GetCISADataTypeSize(VISA_Type type) {return CVariable::GetCISADataTypeSize(type);}
         static e_alignment GetCISADataTypeAlignment(VISA_Type type) {return CVariable::GetCISADataTypeAlignment(type);}
 
@@ -670,6 +672,16 @@ namespace IGC
         RMEncoding getEncoderRoundingMode_FP(ERoundingMode FP_RM);
         RMEncoding getEncoderRoundingMode_FPCvtInt(ERoundingMode FCvtI_RM);
 
+        enum PreemptionEncoding
+        {
+            PreemptionDisabled = 0x000,
+            PreemptionEnabled = 0x800
+        };
+
+        PreemptionEncoding getEncoderPreemptionMode(EPreemptionMode preemptionMode);
+
+        void SetPreemptionMode(PreemptionEncoding actualPreemptionMode, PreemptionEncoding newPreemptionMode);
+
         unsigned GetRawOpndSplitOffset(VISA_Exec_Size fromExecSize,
             VISA_Exec_Size toExecSize,
             unsigned thePart, CVariable* var) const;
diff --git a/IGC/Compiler/CISACodeGen/ComputeShaderCodeGen.hpp b/IGC/Compiler/CISACodeGen/ComputeShaderCodeGen.hpp
@@ -37,6 +37,11 @@ namespace IGC
             m_disableMidThreadPreemption = true;
         }
 
+        bool        GetDisableMidthreadPreemption()
+        {
+            return m_disableMidThreadPreemption;
+        }
+
         bool        HasSLM() const { return m_hasSLM; }
         bool        HasFullDispatchMask() override;
 
diff --git a/IGC/Compiler/CISACodeGen/EmitVISAPass.cpp b/IGC/Compiler/CISACodeGen/EmitVISAPass.cpp
@@ -126,6 +126,7 @@ EmitPass::EmitPass(CShaderProgram::KernelShaderMap& shaders, SIMDMode mode, bool
     m_canAbortOnSpill(canAbortOnSpill),
     m_roundingMode_FP(ERoundingMode::ROUND_TO_NEAREST_EVEN),
     m_roundingMode_FPCvtInt(ERoundingMode::ROUND_TO_ZERO),
+    m_preemptionMode(EPreemptionMode::PREEMPTION_ENABLED),
     m_pSignature(pSignature),
     m_isDuplicate(false)
 {
@@ -1238,6 +1239,8 @@ bool EmitPass::runOnFunction(llvm::Function& F)
         (m_currShader->GetContext()->m_instrTypes.numLoopInsts == 0) &&
         (m_currShader->ProgramOutput()->m_InstructionCount < IGC_GET_FLAG_VALUE(MidThreadPreemptionDisableThreshold)))
     {
+        m_preemptionMode = PREEMPTION_DISABLED;
+
         if (m_currShader->GetShaderType() == ShaderType::COMPUTE_SHADER)
         {
             CComputeShader* csProgram = static_cast<CComputeShader*>(m_currShader);
@@ -16759,6 +16762,15 @@ void EmitPass::SetRoundingMode_FPCvtInt(ERoundingMode newRM_FPCvtInt)
     }
 }
 
+void EmitPass::SetPreemptionMode(EPreemptionMode newPreemptionMode)
+{
+    if (newPreemptionMode != m_preemptionMode)
+    {
+        m_encoder->SetPreemptionMode(m_preemptionMode, newPreemptionMode);
+        m_preemptionMode = newPreemptionMode;
+    }
+}
+
 // Return true if inst needs specific rounding mode; false otherwise.
 //
 // Currently, only gen intrinsic needs rounding mode other than the default.
@@ -22585,8 +22597,12 @@ void EmitPass::emitTraceRay(TraceRayIntrinsic* I, bool RayQueryEnable)
                 offsetof(RTStackFormat::TraceRayMessage::Header, rayQueryLocation) / sizeof(DWORD);
             static_assert(RayQueryDword == 4, "header change?");
 
-            CVariable* RayQueryVal =
-                m_currShader->ImmToVariable(RayQueryEnable ? 1 : 0, ISA_TYPE_UD);
+        uint64_t rayQueryHeader = 0x0;
+
+        rayQueryHeader |= RayQueryEnable ? 1 : 0;
+
+        CVariable* RayQueryVal =
+            m_currShader->ImmToVariable(rayQueryHeader, ISA_TYPE_UD);
 
             m_encoder->SetSimdSize(SIMDMode::SIMD1);
             m_encoder->SetNoMask();
@@ -22631,6 +22647,7 @@ void EmitPass::emitTraceRay(TraceRayIntrinsic* I, bool RayQueryEnable)
             m_currShader->m_SIMDSize >= SIMDMode::SIMD16 ? 1 : 0,
             true,
             RayQueryEnable);
+
         CVariable* pMessDesc =
             m_currShader->ImmToVariable(messageSpecificControl, ISA_TYPE_UD);
 
@@ -22666,7 +22683,9 @@ void EmitPass::emitTraceRay(TraceRayIntrinsic* I, bool RayQueryEnable)
             payload,
             extDescriptor,
             exDesc,
-            pMessDesc);
+            pMessDesc,
+            false);
+
         m_encoder->Push();
     }
 
@@ -22715,6 +22734,7 @@ void EmitPass::emitReadTraceRaySync(llvm::GenIntrinsicInst* I)
     m_encoder->Push();
 }
 
+
 void EmitPass::emitBTD(
     CVariable* GlobalBufferPtr,
     CVariable* StackID,
diff --git a/IGC/Compiler/CISACodeGen/EmitVISAPass.hpp b/IGC/Compiler/CISACodeGen/EmitVISAPass.hpp
@@ -471,6 +471,8 @@ class EmitPass : public llvm::FunctionPass
     void emitSyncStackID(llvm::GenIntrinsicInst* I);
     void emitTraceRay(llvm::TraceRayIntrinsic *I, bool RayQueryEnable);
     void emitReadTraceRaySync(llvm::GenIntrinsicInst* I);
+
+
     void emitBTD(
         CVariable* GlobalBufferPtr,
         CVariable* StackID,
@@ -806,6 +808,8 @@ class EmitPass : public llvm::FunctionPass
     ERoundingMode m_roundingMode_FP;
     ERoundingMode m_roundingMode_FPCvtInt;
 
+    EPreemptionMode m_preemptionMode;
+
     uint m_currentBlock = (uint) -1;
 
     bool m_currFuncHasSubroutine = false;
@@ -877,6 +881,9 @@ class EmitPass : public llvm::FunctionPass
     void SetRoundingMode_FPCvtInt(ERoundingMode RM_FPCvtInt);
     bool setRMExplicitly(llvm::Instruction* inst);
     void ResetRoundingMode(llvm::Instruction* inst);
+
+    void SetPreemptionMode(EPreemptionMode newPreemptionMode);
+
     // returns true if the instruction does not care about the rounding mode settings
     bool ignoreRoundingMode(llvm::Instruction* inst) const;
 
diff --git a/IGC/Compiler/CISACodeGen/helper.cpp b/IGC/Compiler/CISACodeGen/helper.cpp
@@ -223,6 +223,26 @@ namespace IGC
         return type;
     }
 
+    ///
+    /// returns the number of exit blocks iin the given function.
+    ///
+    unsigned getNumberOfExitBlocks(llvm::Function& function)
+    {
+        unsigned numberOfExitBlocks = 0;
+
+        for (llvm::BasicBlock& block : function.getBasicBlockList())
+        {
+            llvm::Instruction* terminator = block.getTerminator();
+
+            if (llvm::isa_and_nonnull<ReturnInst>(terminator))
+            {
+                ++numberOfExitBlocks;
+            }
+        }
+
+        return numberOfExitBlocks;
+    }
+
     ///
     /// returns constant buffer load offset
     ///
@@ -1413,6 +1433,40 @@ namespace IGC
         return false;
     }
 
+    bool isBarrierIntrinsic(const llvm::Instruction* I)
+    {
+        const GenIntrinsicInst* GII = dyn_cast<GenIntrinsicInst>(I);
+        if (!GII)
+            return false;
+
+        switch (GII->getIntrinsicID())
+        {
+        case GenISAIntrinsic::GenISA_threadgroupbarrier:
+        case GenISAIntrinsic::GenISA_threadgroupbarrier_signal:
+        case GenISAIntrinsic::GenISA_threadgroupbarrier_wait:
+        case GenISAIntrinsic::GenISA_threadgroupnamedbarriers_signal:
+        case GenISAIntrinsic::GenISA_threadgroupnamedbarriers_wait:
+        case GenISAIntrinsic::GenISA_wavebarrier:
+            return true;
+        default:
+            return false;
+        }
+    }
+
+    bool isUserFunctionCall(const llvm::Instruction* I)
+    {
+        const CallInst* callInst = dyn_cast<CallInst>(I);
+
+        // Return true if:
+        // 1. callInst->getCalledFunction() == nullptr, this means indirect function call
+        // OR
+        // 2. Called function is not an Intrinsic.
+        bool isUserFunction = (callInst != nullptr) &&
+            ((callInst->getCalledFunction() == nullptr) || !callInst->getCalledFunction()->isIntrinsic());
+
+        return isUserFunction;
+    }
+
     bool isURBWriteIntrinsic(const llvm::Instruction* I)
     {
         const GenIntrinsicInst* GII = dyn_cast<GenIntrinsicInst>(I);
diff --git a/IGC/Compiler/CISACodeGen/helper.h b/IGC/Compiler/CISACodeGen/helper.h
@@ -178,6 +178,10 @@ namespace IGC
     bool isSubGroupIntrinsicPVC(const llvm::Instruction* I);
     bool hasSubGroupIntrinsicPVC(llvm::Function& F);
 
+    bool isBarrierIntrinsic(const llvm::Instruction* I);
+
+    bool isUserFunctionCall(const llvm::Instruction* I);
+
     bool IsStatelessMemLoadIntrinsic(llvm::GenISAIntrinsic::ID id);
     bool IsStatelessMemStoreIntrinsic(llvm::GenISAIntrinsic::ID id);
     bool IsStatelessMemAtomicIntrinsic(llvm::GenIntrinsicInst& inst, llvm::GenISAIntrinsic::ID id);
@@ -198,6 +202,8 @@ namespace IGC
     BufferType DecodeBufferType(unsigned addrSpace);
     int getConstantBufferLoadOffset(llvm::LoadInst* ld);
 
+    unsigned getNumberOfExitBlocks(llvm::Function& function);
+
     bool isDummyBasicBlock(llvm::BasicBlock* BB);
 
     bool IsDirectIdx(unsigned addrSpace);
diff --git a/IGC/Compiler/CodeGenPublicEnums.h b/IGC/Compiler/CodeGenPublicEnums.h
@@ -264,6 +264,12 @@ namespace IGC
         ROUND_TO_ANY   // dont care
     };
 
+    enum EPreemptionMode
+    {
+        PREEMPTION_ENABLED,
+        PREEMPTION_DISABLED
+    };
+
 
 
     enum DISPATCH_SHADER_RAY_INFO_TYPE : unsigned int
diff --git a/IGC/GenISAIntrinsics/GenIntrinsicInst.h b/IGC/GenISAIntrinsics/GenIntrinsicInst.h
@@ -1309,6 +1309,7 @@ class AllocateRayQueryIntrinsic : public GenIntrinsicInst {
     Value* getFlags()  const { return getOperand(0); }
 };
 
+
 class RayQueryInstrisicBase : public GenIntrinsicInst
 {
 public:

Original file line number	Diff line number	Diff line change
`@@ -597,6 +597,7 @@ Value* RTBuilder::CreateSyncStackPtrIntrinsic(`
`597`	`597`	`return StackPtr;`
`598`	`598`	`}`
`599`	`599`
	`600`	`+`
`600`	`601`	`RTBuilder::SWStackPtrVal* RTBuilder::getSWStackPointer(const Twine& Name)`
`601`	`602`	`{`
`602`	`603`	`auto* CI = this->CreateSWStackPtrIntrinsic(nullptr, false, Name);`