Support for CacheControlLoadINTEL extension for OpenCL

pkwasnie-intel · web-flow · commit 593ab67a4e38 · 2024-06-12T14:28:03.000+02:00
prefetch Adds support for CacheControlLoadINTEL SPIR-V extension to OpenCL prefetch builtin. (cherry picked from commit 1ea0c84)
diff --git a/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.cpp b/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.cpp
@@ -166,6 +166,24 @@ void HandleSpirvDecorationMetadata::visit2DBlockWriteCallInst(CallInst& I, Strin
     }
 }
 
+void HandleSpirvDecorationMetadata::visitPrefetchCallInst(CallInst& I)
+{
+    Value* ptr = I.getArgOperand(0);
+    auto spirvDecorations = parseSPIRVDecorationsFromMD(ptr);
+    for (auto& [DecorationId, MDNodes] : spirvDecorations)
+    {
+        switch (DecorationId)
+        {
+            // IDecCacheControlLoadINTEL
+            case DecorationIdCacheControlLoad:
+            {
+                handleCacheControlINTELForPrefetch(I, MDNodes);
+                break;
+            }
+        }
+    }
+}
+
 void HandleSpirvDecorationMetadata::visitCallInst(CallInst& I)
 {
     Function* F = I.getCalledFunction();
@@ -175,6 +193,11 @@ void HandleSpirvDecorationMetadata::visitCallInst(CallInst& I)
         "_Z[0-9]+(intel_sub_group_2d_block_(prefetch|read|read_transform|read_transpose)_[0-9]+b_[0-9]+r[0-9]+x[0-9]+c)");
     Regex pattern2DBlockWrite(
         "_Z[0-9]+(intel_sub_group_2d_block_write_[0-9]+b_[0-9]+r[0-9]+x[0-9]+c)");
+#if defined(IGC_SCALAR_USE_KHRONOS_SPIRV_TRANSLATOR)
+    Regex patternPrefetch("_Z[0-9]+__spirv_ocl_prefetch");
+#else // IGC Legacy SPIRV Translator
+    Regex patternPrefetch("__builtin_spirv_OpenCL_prefetch");
+#endif
 
     SmallVector<StringRef, 4> Matches;
     StringRef funcName = F->getName();
@@ -187,6 +210,10 @@ void HandleSpirvDecorationMetadata::visitCallInst(CallInst& I)
     {
         visit2DBlockWriteCallInst(I, Matches[1]);
     }
+    else if (patternPrefetch.match(funcName, &Matches))
+    {
+        visitPrefetchCallInst(I);
+    }
 }
 
 template<typename T>
@@ -242,3 +269,61 @@ void HandleSpirvDecorationMetadata::handleCacheControlINTELFor2DBlockIO(CallInst
     if (F->getNumUses() == 0)
         m_BuiltinsToRemove.insert(F);
 }
+
+void HandleSpirvDecorationMetadata::handleCacheControlINTELForPrefetch(llvm::CallInst& I, llvm::SmallPtrSetImpl<llvm::MDNode*>& MDNodes)
+{
+    CacheControlFromMDNodes cacheControl = resolveCacheControlFromMDNodes<LoadCacheControl>(m_pCtx, MDNodes);
+    if (cacheControl.isEmpty) return;
+    if (cacheControl.isInvalid)
+    {
+        m_pCtx->EmitWarning("Unsupported cache controls configuration requested. Applying default configuration.");
+        return;
+    }
+
+    Function* F = I.getCalledFunction();
+    IGC_ASSERT(F);
+
+    // Convert prefetch call to: __lsc_prefetch_cache_controls(global void* p, int element_size, int num_elements, enum LSC_LDCC cache_opt)
+    SmallVector<Value*, 4> args;
+    args.push_back(I.getArgOperand(0));
+
+    // OpenCL spec states for prefetch: "Prefetch num_gentypes * sizeof(gentype) bytes into the global cache.".
+    // This design is not friendly to opaque pointers, as it assumes element size can be read from pointer.
+    // For now read size from typed pointer, and in future this will be replaced with opaque prefetch with
+    // explicit element size as arg.
+    PointerType* PTy = dyn_cast<PointerType>(I.getArgOperand(0)->getType());
+    IGC_ASSERT(PTy);
+    args.push_back(ConstantInt::get(Type::getInt32Ty(I.getContext()), IGCLLVM::getNonOpaquePtrEltTy(PTy)->getPrimitiveSizeInBits() / 8));
+
+    // OpenCL prefetch overloads num_elements to either i32 or i64. Convert to i32.
+    IGCLLVM::IRBuilder<> builder(&I);
+    args.push_back(builder.CreateZExtOrTrunc(I.getArgOperand(1), Type::getInt32Ty(I.getContext())));
+
+    auto config = supportedLoadConfigs.find(static_cast<LSC_L1_L3_CC>(cacheControl.value));
+    if (m_pCtx->platform.getPlatformInfo().eProductFamily == IGFX_PVC && config != supportedLoadConfigs.end() && config->second.L1 == LoadCacheControl::Cached)
+    {
+        m_pCtx->EmitWarning("Prefetch to L1 is unsupported on this platform.");
+        args.push_back(ConstantInt::get(Type::getInt32Ty(I.getContext()), mapToLSCCacheControl(LoadCacheControl::Uncached, config->second.L3)));
+    }
+    else
+    {
+        args.push_back(ConstantInt::get(Type::getInt32Ty(I.getContext()), cacheControl.value));
+    }
+
+    SmallVector<Type*, 4> argTypes;
+    for (const auto& arg : args)
+        argTypes.push_back(arg->getType());
+
+    FunctionType* FT = FunctionType::get(I.getType(), argTypes, false);
+    std::string newFuncName = "__lsc_prefetch_cache_controls";
+    auto newFunction = m_Module->getOrInsertFunction(newFuncName, FT);
+
+    auto newCall = CallInst::Create(newFunction, args, "", &I);
+    I.replaceAllUsesWith(newCall);
+    I.eraseFromParent();
+    m_changed = true;
+
+    // Cleanup unused function if all calls have been replaced with the internal version
+    if (F->getNumUses() == 0)
+        m_BuiltinsToRemove.insert(F);
+}
diff --git a/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.h b/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.h
@@ -49,6 +49,7 @@ namespace IGC
         void visitCallInst(llvm::CallInst& I);
         void visit2DBlockReadCallInst(llvm::CallInst& I, llvm::StringRef unmangledName);
         void visit2DBlockWriteCallInst(llvm::CallInst& I, llvm::StringRef unmangledName);
+        void visitPrefetchCallInst(llvm::CallInst& I);
 
     private:
         llvm::Module* m_Module = nullptr;
@@ -65,5 +66,6 @@ namespace IGC
         void handleCacheControlINTEL(llvm::Instruction& I, llvm::SmallPtrSetImpl<llvm::MDNode*>& MDNodes);
         template<typename T>
         void handleCacheControlINTELFor2DBlockIO(llvm::CallInst& I, llvm::SmallPtrSetImpl<llvm::MDNode*>& MDNodes, llvm::StringRef unmangledName);
+        void handleCacheControlINTELForPrefetch(llvm::CallInst& I, llvm::SmallPtrSetImpl<llvm::MDNode*>& MDNodes);
     };
 }
diff --git a/IGC/BiFModule/Implementation/prefetch.cl b/IGC/BiFModule/Implementation/prefetch.cl
@@ -8,10 +8,11 @@ SPDX-License-Identifier: MIT
 
 #include "IGCBiF_Intrinsics_Lsc.cl"
 
-#define LSC_PREFETCH(p, num_elements) lsc_prefetch(p, sizeof(*p), num_elements)
+#define LSC_PREFETCH(p, num_elements) __lsc_prefetch(p, sizeof(*p), num_elements)
 
-// Mapping from OpenCL prefetch to LSC prefetch.
-INLINE void lsc_prefetch(global void* p, int element_size, int num_elements)
+// Mapping from OpenCL prefetch to LSC prefetch. OpenCL prefetch doesn't have
+// cache control options; default cache control options are used.
+INLINE void __lsc_prefetch(global void* p, int element_size, int num_elements)
 {
     enum LSC_LDCC cacheOpt = BIF_FLAG_CTRL_GET(ForceL1Prefetch) ? LSC_LDCC_L1C_L3C : LSC_LDCC_L1UC_L3C;
 
@@ -149,6 +150,80 @@ INLINE void lsc_prefetch(global void* p, int element_size, int num_elements)
     }
 }
 
+// Mapping from OpenCL prefetch to LSC prefetch with exposed cache controls.
+INLINE void __lsc_prefetch_cache_controls(global void* p, int element_size, int num_elements, enum LSC_LDCC cache_opt)
+{
+    int size = element_size * num_elements;
+
+    // Assumptions:
+    // 1. Vector data type can be used only for i32/i64 types and only if
+    //    pointer is aligned. OpenCL defines alignment to the size of the
+    //    data type in bytes. Instead of checking alignment at runtime:
+    //      a. Assume i32/i64 types are aligned to 4 bytes.
+    //      b. Assume i8/i16 types are not aligned.
+    // 2. Assume overfetch is safe. For platforms generating page faults
+    //    on out of bounds prefetch, cache controls must be corrected
+    //    before calling builtin.
+
+    if (element_size % 4)
+    {
+        // unaligned, don't use vectors
+        if (size > 8)
+        {
+            __builtin_IB_lsc_prefetch_global_ulong(p, 0, cache_opt);
+            __builtin_IB_lsc_prefetch_global_ulong((global ulong*)p + 1, 0, cache_opt);
+        }
+        else if (size > 4)
+        {
+            __builtin_IB_lsc_prefetch_global_ulong(p, 0, cache_opt);
+        }
+        else if (size > 2)
+        {
+            __builtin_IB_lsc_prefetch_global_uint(p, 0, cache_opt);
+        }
+        else if (size > 1)
+        {
+            __builtin_IB_lsc_prefetch_global_ushort(p, 0, cache_opt);
+        }
+        else if (size == 1)
+        {
+            __builtin_IB_lsc_prefetch_global_uchar(p, 0, cache_opt);
+        }
+    }
+    else
+    {
+        // aligned, can use vectors
+        if (size > 16)
+        {
+            __builtin_IB_lsc_prefetch_global_uint8(p, 0, cache_opt);
+        }
+        else if (size > 12)
+        {
+            __builtin_IB_lsc_prefetch_global_uint4(p, 0, cache_opt);
+        }
+        else if (size > 8)
+        {
+            __builtin_IB_lsc_prefetch_global_uint3(p, 0, cache_opt);
+        }
+        else if (size > 4)
+        {
+            __builtin_IB_lsc_prefetch_global_ulong(p, 0, cache_opt);
+        }
+        else if (size > 2)
+        {
+            __builtin_IB_lsc_prefetch_global_uint(p, 0, cache_opt);
+        }
+        else if (size > 1)
+        {
+            __builtin_IB_lsc_prefetch_global_ushort(p, 0, cache_opt);
+        }
+        else if (size == 1)
+        {
+            __builtin_IB_lsc_prefetch_global_uchar(p, 0, cache_opt);
+        }
+    }
+}
+
 //Prefetch function
 
 void SPIRV_OVERLOADABLE OPTNONE SPIRV_OCL_BUILTIN(prefetch, _p1i8_i32, )( global char* p, int num_elements)
diff --git a/IGC/ocloc_tests/features/cache_controls/prefetch.ll b/IGC/ocloc_tests/features/cache_controls/prefetch.ll
@@ -0,0 +1,127 @@
+;=========================== begin_copyright_notice ============================
+;
+; Copyright (C) 2024 Intel Corporation
+;
+; SPDX-License-Identifier: MIT
+;
+;============================ end_copyright_notice =============================
+
+; REQUIRES: llvm-spirv, regkeys, pvc-supported
+
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_cache_controls -o %t.spv
+; RUN: ocloc compile -spirv_input -file %t.spv -device pvc -options " -igc_opts 'PrintToConsole=1 PrintAfter=Layout'" 2>&1 | FileCheck %s
+
+; LSC prefetch args:
+;   1. anyptr: memory address
+;   2. int: immediate offset (in bytes)
+;   3. int: data size (LSC_DATA_SIZE)
+;   4. int: vector size (LSC_DATA_ELEMS)
+;   5. int: cache controls options (LSC_CACHE_OPTS)
+;
+; LSC_CACHE_OPTS:
+;   1 = L1 uncached, L3 uncached
+;   2 = L1 uncached, L3 cached
+;   3 = L1 cached, L3 uncached
+;   4 = L1 cached, L3 cached
+;
+; For PVC, cache to L1 is disabled; L1 cache control options are ignored.
+
+target triple = "spir64-unknown-unknown"
+
+declare spir_func i64 @_Z12get_local_idj(i32)
+declare spir_func void @_Z20__spirv_ocl_prefetchPU3AS1cl(i8 addrspace(1)*, i64) #0
+declare spir_func void @_Z20__spirv_ocl_prefetchPU3AS1fl(float addrspace(1)*, i64)
+
+define spir_kernel void @test_i8_uncached_cached(i8 addrspace(1)* %input) !intel_reqd_sub_group_size !100 {
+entry:
+; CHECK-LABEL: @test_i8_uncached_cached(
+; CHECK:         call void @llvm.genx.GenISA.LSCPrefetch.p1i8(i8 addrspace(1)* %{{[0-9]+}}, i32 0, i32 5, i32 1, i32 2)
+  %i = call spir_func i64 @_Z12get_local_idj(i32 0)
+  %decorated_ptr = getelementptr inbounds i8, i8 addrspace(1)* %input, i64 %i, !spirv.Decorations !3
+  call spir_func void @_Z20__spirv_ocl_prefetchPU3AS1cl(i8 addrspace(1)* %decorated_ptr, i64 1)
+  ret void
+}
+
+define spir_kernel void @test_i8v16_uncached_cached(i8 addrspace(1)* %input) !intel_reqd_sub_group_size !100 {
+entry:
+; COM: i8 type can be unaligned, vector data type can't be used, i8v16 is broken into two i64 messages.
+; CHECK-LABEL: @test_i8v16_uncached_cached(
+; CHECK:         call void @llvm.genx.GenISA.LSCPrefetch.p1i64(i64 addrspace(1)* %{{[0-9]+}}, i32 0, i32 4, i32 1, i32 2)
+; CHECK:         call void @llvm.genx.GenISA.LSCPrefetch.p1i64(i64 addrspace(1)* %{{[0-9]+}}, i32 0, i32 4, i32 1, i32 2)
+  %i = call spir_func i64 @_Z12get_local_idj(i32 0)
+  %decorated_ptr = getelementptr inbounds i8, i8 addrspace(1)* %input, i64 %i, !spirv.Decorations !3
+  call spir_func void @_Z20__spirv_ocl_prefetchPU3AS1cl(i8 addrspace(1)* %decorated_ptr, i64 16)
+  ret void
+}
+
+define spir_kernel void @test_float_uncached_uncached(float addrspace(1)* %input) !intel_reqd_sub_group_size !100 {
+entry:
+; CHECK-LABEL: @test_float_uncached_uncached(
+; CHECK:         call void @llvm.genx.GenISA.LSCPrefetch.p1i32(i32 addrspace(1)* %{{[0-9]+}}, i32 0, i32 3, i32 1, i32 1)
+  %i = call spir_func i64 @_Z12get_local_idj(i32 0)
+  %decorated_ptr = getelementptr inbounds float, float addrspace(1)* %input, i64 %i, !spirv.Decorations !0
+  call spir_func void @_Z20__spirv_ocl_prefetchPU3AS1fl(float addrspace(1)* %decorated_ptr, i64 1)
+  ret void
+}
+
+define spir_kernel void @test_float_uncached_cached(float addrspace(1)* %input) !intel_reqd_sub_group_size !100 {
+entry:
+; CHECK-LABEL: @test_float_uncached_cached(
+; CHECK:         call void @llvm.genx.GenISA.LSCPrefetch.p1i32(i32 addrspace(1)* %{{[0-9]+}}, i32 0, i32 3, i32 1, i32 2)
+  %i = call spir_func i64 @_Z12get_local_idj(i32 0)
+  %decorated_ptr = getelementptr inbounds float, float addrspace(1)* %input, i64 %i, !spirv.Decorations !3
+  call spir_func void @_Z20__spirv_ocl_prefetchPU3AS1fl(float addrspace(1)* %decorated_ptr, i64 1)
+  ret void
+}
+
+define spir_kernel void @test_float_cached_uncached(float addrspace(1)* %input) !intel_reqd_sub_group_size !100 {
+entry:
+; COM: Cache to L1 unsupported, ignore L1 cache control options.
+; CHECK-LABEL: @test_float_cached_uncached(
+; CHECK:         call void @llvm.genx.GenISA.LSCPrefetch.p1i32(i32 addrspace(1)* %{{[0-9]+}}, i32 0, i32 3, i32 1, i32 1)
+  %i = call spir_func i64 @_Z12get_local_idj(i32 0)
+  %decorated_ptr = getelementptr inbounds float, float addrspace(1)* %input, i64 %i, !spirv.Decorations !6
+  call spir_func void @_Z20__spirv_ocl_prefetchPU3AS1fl(float addrspace(1)* %decorated_ptr, i64 1)
+  ret void
+}
+
+define spir_kernel void @test_float_cached_cached(float addrspace(1)* %input) !intel_reqd_sub_group_size !100 {
+entry:
+; COM: Cache to L1 unsupported, ignore L1 cache control options.
+; CHECK-LABEL: @test_float_cached_cached(
+; CHECK:         call void @llvm.genx.GenISA.LSCPrefetch.p1i32(i32 addrspace(1)* %{{[0-9]+}}, i32 0, i32 3, i32 1, i32 2)
+  %i = call spir_func i64 @_Z12get_local_idj(i32 0)
+  %decorated_ptr = getelementptr inbounds float, float addrspace(1)* %input, i64 %i, !spirv.Decorations !9
+  call spir_func void @_Z20__spirv_ocl_prefetchPU3AS1fl(float addrspace(1)* %decorated_ptr, i64 1)
+  ret void
+}
+
+define spir_kernel void @test_floatv8_uncached_cached(float addrspace(1)* %input) !intel_reqd_sub_group_size !100 {
+entry:
+; COM: Float type is aligned, vector data type can be used.
+; CHECK-LABEL: @test_floatv8_uncached_cached(
+; CHECK:         call void @llvm.genx.GenISA.LSCPrefetch.p1v8i32(<8 x i32> addrspace(1)* %{{[0-9]+}}, i32 0, i32 3, i32 5, i32 2)
+  %i = call spir_func i64 @_Z12get_local_idj(i32 0)
+  %decorated_ptr = getelementptr inbounds float, float addrspace(1)* %input, i64 %i, !spirv.Decorations !3
+  call spir_func void @_Z20__spirv_ocl_prefetchPU3AS1fl(float addrspace(1)* %decorated_ptr, i64 8)
+  ret void
+}
+
+!0 = !{!1, !2}
+!1 = !{i32 6442, i32 0, i32 0}  ; {CacheControlLoadINTEL, CacheLevel=0, Uncached}
+!2 = !{i32 6442, i32 1, i32 0}  ; {CacheControlLoadINTEL, CacheLevel=1, Uncached}
+
+!3 = !{!4, !5}
+!4 = !{i32 6442, i32 0, i32 0}  ; {CacheControlLoadINTEL, CacheLevel=0, Uncached}
+!5 = !{i32 6442, i32 1, i32 1}  ; {CacheControlLoadINTEL, CacheLevel=1, Cached}
+
+!6 = !{!7, !8}
+!7 = !{i32 6442, i32 0, i32 1}  ; {CacheControlLoadINTEL, CacheLevel=0, Cached}
+!8 = !{i32 6442, i32 1, i32 0}  ; {CacheControlLoadINTEL, CacheLevel=1, Uncached}
+
+!9 = !{!10, !11}
+!10 = !{i32 6442, i32 0, i32 1}  ; {CacheControlLoadINTEL, CacheLevel=0, Cached}
+!11 = !{i32 6442, i32 1, i32 1}  ; {CacheControlLoadINTEL, CacheLevel=1, Cached}
+
+!100 = !{i32 16}
