Replace VC math intrinsics with standard LLVM ones

vmustya · igcbot · commit f31c64610a96 · 2023-12-26T20:29:25.000+01:00
Use standard LLVM instrinsics as follows:
* `@llvm.genx.cos` -&gt; `@llvm.cos`
* `@llvm.genx.exp` -&gt; `@llvm.exp2`
* `@llvm.genx.log` -&gt; `@llvm.log2`
* `@llvm.genx.sin` -&gt; `@llvm.sin`
* `@llvm.genx.pow` -&gt; `@llvm.pow`
diff --git a/IGC/VectorCompiler/lib/GenXCodeGen/GenXLowering.cpp b/IGC/VectorCompiler/lib/GenXCodeGen/GenXLowering.cpp
@@ -2028,16 +2028,6 @@ bool GenXLowering::processInst(Instruction *Inst) {
       return lowerMathIntrinsic(CI, GenXIntrinsic::genx_rndd);
     case Intrinsic::trunc:
       return lowerMathIntrinsic(CI, GenXIntrinsic::genx_rndz);
-    case Intrinsic::exp2:
-      return lowerFastMathIntrinsic(CI, GenXIntrinsic::genx_exp);
-    case Intrinsic::log2:
-      return lowerFastMathIntrinsic(CI, GenXIntrinsic::genx_log);
-    case Intrinsic::pow:
-      return lowerFastMathIntrinsic(CI, GenXIntrinsic::genx_pow);
-    case Intrinsic::sin:
-      return lowerFastMathIntrinsic(CI, GenXIntrinsic::genx_sin);
-    case Intrinsic::cos:
-      return lowerFastMathIntrinsic(CI, GenXIntrinsic::genx_cos);
     case Intrinsic::stacksave:
       return lowerStackSave(CI);
     case Intrinsic::stackrestore:
diff --git a/IGC/VectorCompiler/lib/GenXCodeGen/Utils/cisa_gen_intrinsics.json b/IGC/VectorCompiler/lib/GenXCodeGen/Utils/cisa_gen_intrinsics.json
@@ -21,6 +21,47 @@
       "src1": [ "GENERAL", "MODIFIER_ARITH", 2 ],
       "src2": [ "GENERAL", "MODIFIER_ARITH", 3 ]
     },
+    "cos": {
+      "opc": "ISA_COS",
+      "exec_size": [ "EXECSIZE" ],
+      "elementwise": [ "ELEMENTWISE" ],
+      "pred": [ "IMPLICITPRED" ],
+      "dst": [ "GENERAL", 0 ],
+      "src0": [ "GENERAL", 1 ]
+    },
+    "exp2": {
+      "opc": "ISA_EXP",
+      "exec_size": [ "EXECSIZE" ],
+      "elementwise": [ "ELEMENTWISE" ],
+      "pred": [ "IMPLICITPRED" ],
+      "dst": [ "GENERAL", 0 ],
+      "src0": [ "GENERAL", 1 ]
+    },
+    "log2": {
+      "opc": "ISA_LOG",
+      "exec_size": [ "EXECSIZE" ],
+      "elementwise": [ "ELEMENTWISE" ],
+      "pred": [ "IMPLICITPRED" ],
+      "dst": [ "GENERAL", 0 ],
+      "src0": [ "GENERAL", 1 ]
+    },
+    "pow": {
+      "opc": "ISA_POW",
+      "exec_size": [ "EXECSIZE" ],
+      "elementwise": [ "ELEMENTWISE" ],
+      "pred": [ "IMPLICITPRED" ],
+      "dst": [ "GENERAL", 0 ],
+      "src0": [ "GENERAL", 1 ],
+      "src1": [ "GENERAL", 2 ]
+    },
+    "sin": {
+      "opc": "ISA_SIN",
+      "exec_size": [ "EXECSIZE" ],
+      "elementwise": [ "ELEMENTWISE" ],
+      "pred": [ "IMPLICITPRED" ],
+      "dst": [ "GENERAL", 0 ],
+      "src0": [ "GENERAL", 1 ]
+    },
     // Internal vc intrinsics
     "vc::InternalIntrinsic::cast_from_bf16": {
       "opc": "ISA_MOV",
@@ -1645,15 +1686,6 @@
       "src0": [ "GENERAL", "MODIFIER_ARITH", 1 ],
       "src1": [ "GENERAL", "MODIFIER_ARITH", 2 ]
     },
-    "genx_pow": {
-      "opc": "ISA_POW",
-      "exec_size": [ "EXECSIZE" ],
-      "elementwise": [ "ELEMENTWISE" ],
-      "pred": [ "IMPLICITPRED" ],
-      "dst": [ "GENERAL", 0 ],
-      "src0": [ "GENERAL", 1 ],
-      "src1": [ "GENERAL", 2 ]
-    },
     "genx_add_addr": {
       "opc": "ISA_ADDR_ADD",
       "exec_size": [ "EXECSIZE" ],
@@ -1706,22 +1738,6 @@
       "dst": [ "GENERAL", 0 ],
       "src0": [ "GENERAL", 1 ]
     },
-    "genx_log": {
-      "opc": "ISA_LOG",
-      "exec_size": [ "EXECSIZE" ],
-      "elementwise": [ "ELEMENTWISE" ],
-      "pred": [ "IMPLICITPRED" ],
-      "dst": [ "GENERAL", 0 ],
-      "src0": [ "GENERAL", 1 ]
-    },
-    "genx_exp": {
-      "opc": "ISA_EXP",
-      "exec_size": [ "EXECSIZE" ],
-      "elementwise": [ "ELEMENTWISE" ],
-      "pred": [ "IMPLICITPRED" ],
-      "dst": [ "GENERAL", 0 ],
-      "src0": [ "GENERAL", 1 ]
-    },
     "genx_scatter_scaled": {
       "opc": "ISA_SCATTER_SCALED",
       "exec_size": [ "EXECSIZE_FROM_ARG", 1 ],
@@ -2260,22 +2276,6 @@
       "deltaV": [ "GENERAL", 7 ],
       "dst": [ "RAW", 0 ]
     },
-    "genx_sin": {
-      "opc": "ISA_SIN",
-      "exec_size": [ "EXECSIZE" ],
-      "elementwise": [ "ELEMENTWISE" ],
-      "pred": [ "IMPLICITPRED" ],
-      "dst": [ "GENERAL", 0 ],
-      "src0": [ "GENERAL", 1 ]
-    },
-    "genx_cos": {
-      "opc": "ISA_COS",
-      "exec_size": [ "EXECSIZE" ],
-      "elementwise": [ "ELEMENTWISE" ],
-      "pred": [ "IMPLICITPRED" ],
-      "dst": [ "GENERAL", 0 ],
-      "src0": [ "GENERAL", 1 ]
-    },
     "genx_ssavg": {
       "opc": "ISA_AVG",
       "exec_size": [ "EXECSIZE" ],
diff --git a/IGC/VectorCompiler/lib/GenXOpts/CMPacketize/GenXPacketize.cpp b/IGC/VectorCompiler/lib/GenXOpts/CMPacketize/GenXPacketize.cpp
@@ -825,19 +825,20 @@ Value *GenXPacketize::packetizeLLVMIntrinsic(Instruction *pInst) {
   }
 
   // override certain intrinsics
-  Value *pNewCall;
   switch (id) {
+  default:
+    break;
   case Intrinsic::log2:
-    pNewCall = B->VLOG2PS(packetizedArgs[0]);
+    if (!pInst->hasApproxFunc())
+      return B->VLOG2PS(packetizedArgs[0]);
     break;
   case Intrinsic::exp2:
-    pNewCall = B->VEXP2PS(packetizedArgs[0]);
+    if (!pInst->hasApproxFunc())
+      return B->VEXP2PS(packetizedArgs[0]);
     break;
-  default: {
-    Function *newF = getVectorIntrinsic(M, id, vectorArgTys);
-    pNewCall = CallInst::Create(newF, packetizedArgs, "", pCall);
-  }
   }
+  Function *newF = getVectorIntrinsic(M, id, vectorArgTys);
+  auto *pNewCall = CallInst::Create(newF, packetizedArgs, "", pCall);
   return pNewCall;
 }
 
diff --git a/IGC/VectorCompiler/lib/GenXOpts/CMTrans/GenXImportOCLBiF.cpp b/IGC/VectorCompiler/lib/GenXOpts/CMTrans/GenXImportOCLBiF.cpp
@@ -87,21 +87,21 @@ BIConvert::BIConvert() {
   OneMap["__builtin_IB_frnd_pi"] = GenXIntrinsic::genx_rndu;
   OneMap["__builtin_IB_frnd_ni"] = GenXIntrinsic::genx_rndd;
   OneMap["__builtin_IB_frnd_zi"] = GenXIntrinsic::genx_rndz;
-  OneMap["__builtin_IB_native_cosf"] = GenXIntrinsic::genx_cos;
-  OneMap["__builtin_IB_native_cosh"] = GenXIntrinsic::genx_cos;
-  OneMap["__builtin_IB_native_sinf"] = GenXIntrinsic::genx_sin;
-  OneMap["__builtin_IB_native_sinh"] = GenXIntrinsic::genx_sin;
-  OneMap["__builtin_IB_native_exp2f"] = GenXIntrinsic::genx_exp;
-  OneMap["__builtin_IB_native_exp2h"] = GenXIntrinsic::genx_exp;
-  OneMap["__builtin_IB_native_log2f"] = GenXIntrinsic::genx_log;
-  OneMap["__builtin_IB_native_log2h"] = GenXIntrinsic::genx_log;
-  OneMap["__builtin_IB_native_sqrtf"] = GenXIntrinsic::genx_sqrt;
-  OneMap["__builtin_IB_native_sqrth"] = GenXIntrinsic::genx_sqrt;
-  OneMap["__builtin_IB_native_sqrtd"] = GenXIntrinsic::genx_sqrt;
+  OneMap["__builtin_IB_native_cosf"] = Intrinsic::cos;
+  OneMap["__builtin_IB_native_cosh"] = Intrinsic::cos;
+  OneMap["__builtin_IB_native_sinf"] = Intrinsic::sin;
+  OneMap["__builtin_IB_native_sinh"] = Intrinsic::sin;
+  OneMap["__builtin_IB_native_exp2f"] = Intrinsic::exp2;
+  OneMap["__builtin_IB_native_exp2h"] = Intrinsic::exp2;
+  OneMap["__builtin_IB_native_log2f"] = Intrinsic::log2;
+  OneMap["__builtin_IB_native_log2h"] = Intrinsic::log2;
+  OneMap["__builtin_IB_native_sqrtf"] = Intrinsic::sqrt;
+  OneMap["__builtin_IB_native_sqrth"] = Intrinsic::sqrt;
+  OneMap["__builtin_IB_native_sqrtd"] = Intrinsic::sqrt;
   OneMap["__builtin_IB_popcount_1u32"] = GenXIntrinsic::genx_cbit;
   OneMap["__builtin_IB_popcount_1u16"] = GenXIntrinsic::genx_cbit;
   OneMap["__builtin_IB_popcount_1u8"] = GenXIntrinsic::genx_cbit;
-  OneMap["__builtin_IB_native_powrf"] = GenXIntrinsic::genx_pow;
+  OneMap["__builtin_IB_native_powrf"] = Intrinsic::pow;
   OneMap["__builtin_IB_fma"] = Intrinsic::fma;
   OneMap["__builtin_IB_fmah"] = Intrinsic::fma;
   OneMap["__builtin_IB_bfrev"] = GenXIntrinsic::genx_bfrev;
@@ -185,7 +185,6 @@ static Function *getOneMapIntrinsicDeclaration(CallInst &CI, const unsigned IID,
     return vc::getGenXDeclarationForIdFromArgs(
         CI.getType(), CI.args(), static_cast<GenXIntrinsic::ID>(IID), M);
 
-  IGC_ASSERT_MESSAGE(IID == Intrinsic::fma, "Expected fma intrinsic");
   return Intrinsic::getDeclaration(&M, static_cast<Intrinsic::ID>(IID),
                                    {CI.getType()});
 }
@@ -237,6 +236,8 @@ void BIConvert::runOnModule(Module &M) {
           const SmallVector<llvm::Value *, 3> Args{InstCall->args()};
           Instruction *const IntrinCall =
               CallInst::Create(IntrinFunc, Args, InstCall->getName(), InstCall);
+          if (!GenXIntrinsic::isGenXIntrinsic(IID) && IID != Intrinsic::fma)
+            IntrinCall->setHasApproxFunc(true);
           IntrinCall->setDebugLoc(InstCall->getDebugLoc());
           InstCall->replaceAllUsesWith(IntrinCall);
           ListDelete.push_back(InstCall);
diff --git a/IGC/VectorCompiler/lib/GenXOpts/CMTrans/GenXTranslateIntrinsics.cpp b/IGC/VectorCompiler/lib/GenXOpts/CMTrans/GenXTranslateIntrinsics.cpp
@@ -48,7 +48,8 @@ class GenXTranslateIntrinsics final
 
 private:
   Constant *translateCacheControls(Constant *L1, Constant *L3) const;
-
+  Value *translateMath(CallInst &I, Intrinsic::ID IID,
+                       bool HasApproxFunc = true) const;
   Value *translateBFloat16Convert(CallInst &I) const;
   Value *translateTFloat32Convert(CallInst &I) const;
   Value *translateStochasticRounding(CallInst &I) const;
@@ -84,6 +85,21 @@ void GenXTranslateIntrinsics::visitCallInst(CallInst &I) const {
   switch (IID) {
   default:
     return;
+  case GenXIntrinsic::genx_cos:
+    NewI = translateMath(I, Intrinsic::cos);
+    break;
+  case GenXIntrinsic::genx_exp:
+    NewI = translateMath(I, Intrinsic::exp2);
+    break;
+  case GenXIntrinsic::genx_log:
+    NewI = translateMath(I, Intrinsic::log2);
+    break;
+  case GenXIntrinsic::genx_sin:
+    NewI = translateMath(I, Intrinsic::sin);
+    break;
+  case GenXIntrinsic::genx_pow:
+    NewI = translateMath(I, Intrinsic::pow);
+    break;
   case GenXIntrinsic::genx_bf_cvt:
     NewI = translateBFloat16Convert(I);
     break;
@@ -139,6 +155,19 @@ void GenXTranslateIntrinsics::visitCallInst(CallInst &I) const {
   return;
 }
 
+Value *GenXTranslateIntrinsics::translateMath(CallInst &I, Intrinsic::ID IID,
+                                              bool HasApproxFunc) const {
+  LLVM_DEBUG(dbgs() << "Translate: " << I << "\n");
+  IRBuilder<> Builder(&I);
+
+  SmallVector<Value *, 4> Args(I.args());
+  auto *NewI = Builder.CreateIntrinsic(IID, {I.getType()}, Args);
+  NewI->setHasApproxFunc(HasApproxFunc);
+
+  LLVM_DEBUG(dbgs() << "Created: " << *NewI << "\n");
+  return NewI;
+}
+
 Value *GenXTranslateIntrinsics::translateBFloat16Convert(CallInst &I) const {
   IGC_ASSERT_EXIT(GenXIntrinsic::getGenXIntrinsicID(&I) ==
                   GenXIntrinsic::genx_bf_cvt);
diff --git a/IGC/VectorCompiler/test/CisaBuilder/math_f16.ll b/IGC/VectorCompiler/test/CisaBuilder/math_f16.ll
@@ -0,0 +1,70 @@
+;=========================== begin_copyright_notice ============================
+;
+; Copyright (C) 2023 Intel Corporation
+;
+; SPDX-License-Identifier: MIT
+;
+;============================ end_copyright_notice =============================
+
+; COM: ;;;;;;;;;; RUNNERS ;;;;;;;;;;
+
+; RUN: llc %s -march=genx64 -mcpu=XeHPG -vc-skip-ocl-runtime-info -finalizer-opts='-dumpcommonisa -isaasmToConsole' -o /dev/null \
+; RUN: | FileCheck %s
+
+
+; COM: ;;;;;;;;;; CHECKERS ;;;;;;;;;;
+
+; CHECK: .decl  [[SRC:V[^ ]+]] v_type=G type=hf num_elts=8
+; CHECK: cos (M1, 8) [[COS:V[^ ]+]](0,0)<1> [[SRC]](0,0)<1;1,0>
+; CHECK: exp (M1, 8) [[EXP:V[^ ]+]](0,0)<1> [[SRC]](0,0)<1;1,0>
+; CHECK: log (M1, 8) [[LOG:V[^ ]+]](0,0)<1> [[SRC]](0,0)<1;1,0>
+; CHECK: sin (M1, 8) [[SIN:V[^ ]+]](0,0)<1> [[SRC]](0,0)<1;1,0>
+
+; COM: ;;;;;;;;;; KERNEL ;;;;;;;;;;
+
+target datalayout = "e-p:64:64-i64:64-n8:16:32"
+target triple = "genx64-unknown-unknown"
+
+declare <8 x half> @llvm.genx.oword.ld.v8f16(i32, i32, i32)
+declare void @llvm.genx.oword.st.v8f16(i32, i32, <8 x half>)
+
+declare <8 x half> @llvm.cos.v8f16(<8 x half>)
+declare <8 x half> @llvm.exp2.v8f16(<8 x half>)
+declare <8 x half> @llvm.log2.v8f16(<8 x half>)
+declare <8 x half> @llvm.sin.v8f16(<8 x half>)
+
+define dllexport spir_kernel void @math(i32 %0, i32 %1) local_unnamed_addr #0 {
+  %src = tail call <8 x half> @llvm.genx.oword.ld.v8f16(i32 0, i32 %0, i32 0)
+
+  %cos = call afn <8 x half> @llvm.cos.v8f16(<8 x half> %src)
+  %exp = call afn <8 x half> @llvm.exp2.v8f16(<8 x half> %src)
+  %log = call afn <8 x half> @llvm.log2.v8f16(<8 x half> %src)
+  %sin = call afn <8 x half> @llvm.sin.v8f16(<8 x half> %src)
+
+  tail call void @llvm.genx.oword.st.v8f16(i32 %1, i32 0, <8 x half> %cos)
+  tail call void @llvm.genx.oword.st.v8f16(i32 %1, i32 2, <8 x half> %exp)
+  tail call void @llvm.genx.oword.st.v8f16(i32 %1, i32 4, <8 x half> %log)
+  tail call void @llvm.genx.oword.st.v8f16(i32 %1, i32 6, <8 x half> %sin)
+  ret void
+}
+
+attributes #0 = { noinline nounwind "CMGenxMain" }
+
+!spirv.Source = !{!1}
+!opencl.spir.version = !{!2}
+!opencl.ocl.version = !{!1}
+!opencl.used.extensions = !{!0}
+!opencl.used.optional.core.features = !{!0}
+!spirv.Generator = !{!3}
+!genx.kernels = !{!4}
+!genx.kernel.internal = !{!8}
+
+!0 = !{}
+!1 = !{i32 0, i32 0}
+!2 = !{i32 1, i32 2}
+!3 = !{i16 6, i16 14}
+!4 = !{void (i32, i32)* @math, !"math", !5, i32 0, !6, !1, !7, i32 0}
+!5 = !{i32 2, i32 2}
+!6 = !{i32 64, i32 68}
+!7 = !{!"buffer_t", !"buffer_t"}
+!8 = !{void (i32, i32)* @math, null, null, null, null}
diff --git a/IGC/VectorCompiler/test/CisaBuilder/math_f32.ll b/IGC/VectorCompiler/test/CisaBuilder/math_f32.ll
diff --git a/IGC/VectorCompiler/test/GenXTranslateIntrinsics/math.ll b/IGC/VectorCompiler/test/GenXTranslateIntrinsics/math.ll
diff --git a/IGC/VectorCompiler/test/Lowering/math-intrinsics.ll b/IGC/VectorCompiler/test/Lowering/math-intrinsics.ll
