Fix explicit generic casts handling

Explicit generic casts allows user to check whether a particular generic
pointer is pointing either to global, local or private address space.

They are represented in OpenCL C as `to_global`, `to_local` or `to_private`
functions. Specification: https://man.opencl.org/addressSpaceQualifierFuncs.html

For example: `to_private` returns a pointer that points to a region in
the private address space if to_private can cast ptr to the private
address space. Otherwise it returns NULL.

These builtins are handled incorrectly if IGC decides to allocate private
memory in a global buffer (private memory is allocated in a global buffer
to minimize negative performance implications caused by dynamic generic
address space resolution), since in such case, there is no possibility
to distinguish between private and global pointer.

This change introduces the following logic:
1. Detect usage of explicit generic casts in a kernel.
2. If the casts are present:
    a) Force tagging private pointers - to distinguish from global pointers
    b) Clear tag for addrspacecasts from global to generic - since private
       accesses may be statically resolved to global accesses when casting
       back to named addresspace, generic pointer casted to global address
       space may actually be a private pointer for which a tag has been set
       in step a).

Author: aratajew

IGC/Compiler/CISACodeGen/EmitVISAPass.cpp

@@ -8649,10 +8649,13 @@ void EmitPass::emitAddrSpaceCast(llvm::AddrSpaceCastInst* addrSpaceCast)
 
     CVariable* srcV = GetSymbol(addrSpaceCast->getOperand(0));
 
-    if (!m_canGenericPointToPrivate && !m_canGenericPointToLocal)
+    bool skipTagging =
+        !m_canGenericPointToLocal &&
+        !m_canGenericPointToPrivate &&
+        !m_pCtx->mustDistinguishBetweenPrivateAndGlobalPtr();
+
+    if (skipTagging)
     {
-        // If forcing global memory allocation and there are no generic pointers to local AS,
-        // there is no need to tag generic pointers.
         m_encoder->Cast(m_destination, srcV);
         m_encoder->Push();
         return;
@@ -8682,7 +8685,7 @@ void EmitPass::emitAddrSpaceCast(llvm::AddrSpaceCastInst* addrSpaceCast)
             return;
         }
 
-        if (sourceAddrSpace == ADDRESS_SPACE_PRIVATE && !m_pCtx->allocatePrivateAsGlobalBuffer())
+        if (sourceAddrSpace == ADDRESS_SPACE_PRIVATE && (!m_pCtx->allocatePrivateAsGlobalBuffer() || m_pCtx->mustDistinguishBetweenPrivateAndGlobalPtr()))
         {
             emitAddrSpaceToGenericCast(addrSpaceCast, srcV, 1);
         }
@@ -8697,7 +8700,9 @@ void EmitPass::emitAddrSpaceCast(llvm::AddrSpaceCastInst* addrSpaceCast)
         }
     }
     else if (sourceAddrSpace == ADDRESS_SPACE_GENERIC &&
-        (destAddrSpace == ADDRESS_SPACE_PRIVATE || destAddrSpace == ADDRESS_SPACE_LOCAL))
+            (destAddrSpace == ADDRESS_SPACE_PRIVATE ||
+             destAddrSpace == ADDRESS_SPACE_LOCAL ||
+             (destAddrSpace == ADDRESS_SPACE_GLOBAL && m_pCtx->mustDistinguishBetweenPrivateAndGlobalPtr())))
     {
         // Address space cast is in the form of generic -> {private, local, global}
         // Tag is removed according to the address space of the destination

IGC/Compiler/CISACodeGen/OpenCLKernelCodeGen.cpp

@@ -111,6 +111,16 @@ namespace IGC
         return m_InternalOptions.NoLocalToGeneric;
     }
 
+    bool OpenCLProgramContext::mustDistinguishBetweenPrivateAndGlobalPtr() const
+    {
+        return m_mustDistinguishBetweenPrivateAndGlobalPtr;
+    }
+
+    void OpenCLProgramContext::setDistinguishBetweenPrivateAndGlobalPtr(bool distinguish)
+    {
+        m_mustDistinguishBetweenPrivateAndGlobalPtr = distinguish;
+    }
+
     bool OpenCLProgramContext::enableTakeGlobalAddress() const
     {
         return m_Options.EnableTakeGlobalAddress || getModuleMetaData()->capabilities.globalVariableDecorationsINTEL;

IGC/Compiler/CISACodeGen/OpenCLKernelCodeGen.hpp

@@ -238,6 +238,16 @@ namespace IGC
     private:
         bool m_enableZEBinary;
 
+        // To minimize negative performance implications caused by a dynamic generic address
+        // space resolution, private memory can be allocated in the same address space as
+        // global memory. It gives a possibility to treat private memory operations as global
+        // memory operations, so there is no necessity to distinguish between them.
+        // However, when a module uses `to_global` or `to_private` OpenCL builtins, differentiating
+        // between private and global pointer is necessary to preserve conformity.
+        // Below flag is set to true when IGC detects that any of these builtins is called in
+        // a module and could not be resolved statically at compile time.
+        bool m_mustDistinguishBetweenPrivateAndGlobalPtr = false;
+
     public:
         // Additional text visaasm to link.
         std::vector<const char*> m_VISAAsmToLink;
@@ -308,6 +318,8 @@ namespace IGC
         bool forceGlobalMemoryAllocation() const override;
         bool allocatePrivateAsGlobalBuffer() const override;
         bool noLocalToGenericOptionEnabled() const override;
+        bool mustDistinguishBetweenPrivateAndGlobalPtr() const override;
+        void setDistinguishBetweenPrivateAndGlobalPtr(bool);
         bool enableTakeGlobalAddress() const override;
         int16_t getVectorCoalescingControl() const override;
         uint32_t getPrivateMemoryMinimalSizePerThread() const override;

IGC/Compiler/CodeGenContext.cpp

@@ -661,6 +661,11 @@ namespace IGC
         return false;
     }
 
+    bool CodeGenContext::mustDistinguishBetweenPrivateAndGlobalPtr() const
+    {
+        return false;
+    }
+
     bool CodeGenContext::enableTakeGlobalAddress() const
     {
         return false;

IGC/Compiler/CodeGenPublic.h

@@ -1027,6 +1027,7 @@ namespace IGC
         virtual bool forceGlobalMemoryAllocation() const;
         virtual bool allocatePrivateAsGlobalBuffer() const;
         virtual bool noLocalToGenericOptionEnabled() const;
+        virtual bool mustDistinguishBetweenPrivateAndGlobalPtr() const;
         virtual bool enableTakeGlobalAddress() const;
         virtual int16_t getVectorCoalescingControl() const;
         virtual uint32_t getPrivateMemoryMinimalSizePerThread() const;

IGC/Compiler/Optimizer/OpenCLPasses/GenericAddressResolution/GenericAddressDynamicResolution.cpp

@@ -11,6 +11,7 @@ SPDX-License-Identifier: MIT
 #include "AdaptorCommon/ImplicitArgs.hpp"
 #include "Compiler/CodeGenContextWrapper.hpp"
 #include "Compiler/CodeGenPublicEnums.h"
+#include "Compiler/CISACodeGen/OpenCLKernelCodeGen.hpp"
 #include "Compiler/IGCPassSupport.h"
 #include "Compiler/MetaDataUtilsWrapper.h"
 #include "common/LLVMWarningsPush.hpp"
@@ -460,6 +461,12 @@ bool GenericAddressDynamicResolution::visitIntrinsicCall(CallInst& I)
         Value* newPtrNull = nullptr;
         Value* cmpTag = nullptr;
 
+        if (targetAS == ADDRESS_SPACE_GLOBAL || targetAS == ADDRESS_SPACE_PRIVATE)
+        {
+            auto ClContext = static_cast<OpenCLProgramContext*>(m_ctx);
+            ClContext->setDistinguishBetweenPrivateAndGlobalPtr(true);
+        }
+
         // Tag was already obtained from GAS pointer, now we check its address space (AS)
         // and the target AS for this intrinsic call
         if (targetAS == ADDRESS_SPACE_PRIVATE)