From 20d51790abe44dc04fa66841513d70a64b27e8f2 Mon Sep 17 00:00:00 2001 From: Jacob Ledbetter Date: Wed, 5 Aug 2026 08:55:16 -0600 Subject: [PATCH 1/5] feat(os): Add UTF8 support using ICU --- .../Source/Common/System/AsciiString.cpp | 20 +- .../Source/Common/System/UnicodeString.cpp | 34 +- .../Source/GameNetwork/GameInfo.cpp | 15 +- .../GameSpy/Thread/ThreadUtils.cpp | 53 +- .../Source/WWVegas/WWLib/CMakeLists.txt | 5 + Core/Libraries/Source/WWVegas/WWLib/utf8.cpp | 707 ++++++++++++------ Core/Libraries/Source/WWVegas/WWLib/utf8.h | 34 +- vcpkg-lock.json | 18 + vcpkg.json | 4 + 9 files changed, 584 insertions(+), 306 deletions(-) diff --git a/Core/GameEngine/Source/Common/System/AsciiString.cpp b/Core/GameEngine/Source/Common/System/AsciiString.cpp index 61c1dc84448..278fff90794 100644 --- a/Core/GameEngine/Source/Common/System/AsciiString.cpp +++ b/Core/GameEngine/Source/Common/System/AsciiString.cpp @@ -308,19 +308,27 @@ char* AsciiString::getBufferForRead(Int len) void AsciiString::translate(const UnicodeString& stringSrc) { validate(); - // TheSuperHackers @fix bobtista 02/04/2026 Implement UTF-8 conversion replacing 7-bit ASCII only implementation + // TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Convert wide text to UTF-8 with ICU4C. const WideChar* src = stringSrc.str(); - const size_t srcLen = wcslen(src); - const size_t dstLen = Wide_To_Utf8_Len(src, srcLen); - if (dstLen == 0) + const size_t srcLen = stringSrc.getLength(); + const size_t len = Wide_To_Utf8_Len(src, srcLen); + if (len == 0) { clear(); } + else if (len >= static_cast(MAX_LEN)) + { + DEBUG_ASSERTCRASH(false, + ("AsciiString::translate exceeds max string length %d with required UTF-8 length %u", + MAX_LEN, static_cast(len))); + clear(); + } else { - ensureUniqueBufferOfSize((Int)dstLen + 1, false, nullptr, nullptr); - Wide_To_Utf8(peek(), dstLen + 1, src, srcLen); + ensureUniqueBufferOfSize(static_cast(len) + 1, false, nullptr, nullptr); + Wide_To_Utf8(peek(), len + 1, src, srcLen); } + validate(); } diff --git a/Core/GameEngine/Source/Common/System/UnicodeString.cpp b/Core/GameEngine/Source/Common/System/UnicodeString.cpp index e2a58acc2a5..4c6d6541b5a 100644 --- a/Core/GameEngine/Source/Common/System/UnicodeString.cpp +++ b/Core/GameEngine/Source/Common/System/UnicodeString.cpp @@ -222,34 +222,32 @@ WideChar* UnicodeString::getBufferForRead(Int len) void UnicodeString::translate(const AsciiString& stringSrc) { validate(); - // TheSuperHackers @fix bobtista 02/04/2026 Convert UTF-8 to wide, replacing the 7-bit ASCII only - // implementation. Data that is not valid UTF-8 (e.g. legacy CP1252) falls back to a 1:1 byte cast - // to preserve the original characters instead of producing replacement characters. + // TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Convert UTF-8 to wide text with ICU4C. const char* src = stringSrc.str(); - const size_t srcLen = strlen(src); - const size_t dstLen = Utf8_To_Wide_Len(src, srcLen); - if (dstLen != UTF8_INVALID) + const size_t srcLen = stringSrc.getLength(); + const size_t len = Utf8_To_Wide_Len(src, srcLen); + if (srcLen == 0) { - if (dstLen == 0) - { - clear(); - } - else - { - ensureUniqueBufferOfSize((Int)dstLen + 1, false, nullptr, nullptr); - Utf8_To_Wide(peek(), dstLen + 1, src, srcLen); - } + clear(); } - else + else if (len == UTF8_INVALID) { - ensureUniqueBufferOfSize((Int)srcLen + 1, false, nullptr, nullptr); + // Preserve legacy non-UTF-8 data with the original one-byte-to-one-wide-unit behavior. + ensureUniqueBufferOfSize(static_cast(srcLen) + 1, false, nullptr, nullptr); WideChar* buf = peek(); for (size_t i = 0; i < srcLen; ++i) { - buf[i] = (WideChar)(unsigned char)src[i]; + buf[i] = static_cast(static_cast(src[i])); } + buf[srcLen] = 0; } + else + { + ensureUniqueBufferOfSize(static_cast(len) + 1, false, nullptr, nullptr); + Utf8_To_Wide(peek(), len + 1, src, srcLen); + } + validate(); } diff --git a/Core/GameEngine/Source/GameNetwork/GameInfo.cpp b/Core/GameEngine/Source/GameNetwork/GameInfo.cpp index 5b05e9eb369..53ebe571129 100644 --- a/Core/GameEngine/Source/GameNetwork/GameInfo.cpp +++ b/Core/GameEngine/Source/GameNetwork/GameInfo.cpp @@ -953,8 +953,19 @@ AsciiString GameInfoToAsciiString( const GameInfo *game ) int lenRem = m_lanMaxOptionsLength - lenCur; //length remaining before overflowing int lenMax = lenRem / (MAX_SLOTS-i); //share lenRem with all remaining slots AsciiString name = WideCharStringToMultiByte(slot->getName().str()).c_str(); - while( name.getLength() > lenMax ) - name.removeLastChar(); //what a horrible way to truncate. I hate AsciiString. + if (name.getLength() > lenMax) + { + // TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Truncate UTF-8 only at a code-point boundary. + Int truncatedLength = lenMax > 0 ? lenMax : 0; + while (truncatedLength > 0 + && (static_cast(name.getCharAt(truncatedLength)) & 0xC0) == 0x80) + { + --truncatedLength; + } + + AsciiString truncatedName(name.str(), truncatedLength); + name = truncatedName; + } str.format( "H%s%s", name.str(), tmp.str() ); } diff --git a/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp b/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp index 84ce0c9b19a..07920c2075f 100644 --- a/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp +++ b/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp @@ -32,61 +32,54 @@ //------------------------------------------------------------------------- -// TheSuperHackers @refactor bobtista 02/04/2026 Use WWLib UTF-8 functions instead of raw Win32 API calls -std::wstring MultiByteToWideCharSingleLine( const char *orig ) +// TheSuperHackers @refactor CryoTheRenegade 04/08/2026 Use the shared ICU4C UTF conversion functions. +std::wstring MultiByteToWideCharSingleLine( const char* orig ) { const size_t srcLen = strlen(orig); - const size_t dstLen = Utf8_To_Wide_Len(orig, srcLen); - if (dstLen == 0) + const size_t len = Utf8_To_Wide_Len(orig, srcLen); + if (len == 0) + { return std::wstring(); + } + std::wstring ret; - if (dstLen == UTF8_INVALID) + if (len == UTF8_INVALID) { - // Not UTF-8. Fall back to a 1:1 byte cast so legacy data keeps its characters, matching - // UnicodeString::translate. ret.resize(srcLen); for (size_t i = 0; i < srcLen; ++i) { - ret[i] = (WideChar)(unsigned char)orig[i]; + ret[i] = static_cast(static_cast(orig[i])); } } else { - ret.resize(dstLen); - Utf8_To_Wide(&ret[0], dstLen, orig, srcLen); + ret.resize(len); + Utf8_To_Wide(&ret[0], len, orig, srcLen); } - WideChar *c = nullptr; - do - { - c = wcschr(&ret[0], L'\n'); - if (c) - { - *c = L' '; - } - } - while ( c != nullptr ); - do + + for (size_t i = 0; i < ret.size(); ++i) { - c = wcschr(&ret[0], L'\r'); - if (c) + if (ret[i] == L'\n' || ret[i] == L'\x0D') { - *c = L' '; + ret[i] = L' '; } } - while ( c != nullptr ); return ret; } -std::string WideCharStringToMultiByte( const WideChar *orig ) +std::string WideCharStringToMultiByte( const WideChar* orig ) { const size_t srcLen = wcslen(orig); - const size_t dstLen = Wide_To_Utf8_Len(orig, srcLen); - if (dstLen == 0) + const size_t len = Wide_To_Utf8_Len(orig, srcLen); + if (len == 0) + { return std::string(); + } + std::string ret; - ret.resize(dstLen); - Wide_To_Utf8(&ret[0], dstLen, orig, srcLen); + ret.resize(len); + Wide_To_Utf8(&ret[0], len, orig, srcLen); return ret; } diff --git a/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt b/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt index 77721250c6c..882c60ce586 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt +++ b/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt @@ -188,3 +188,8 @@ target_link_libraries(core_wwlib PRIVATE core_wwcommon corei_always ) + +if(NOT WIN32) + find_package(ICU REQUIRED COMPONENTS uc data) + target_link_libraries(core_wwlib PRIVATE ICU::uc ICU::data) +endif() diff --git a/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp b/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp index 9faa1d64ef1..005fb3ff681 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp +++ b/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp @@ -19,265 +19,524 @@ #include "always.h" #include "utf8.h" -// wchar_t is a 16-bit UTF-16 code unit on Windows and a 32-bit UTF-32 codepoint on most other -// platforms. WCHAR_MAX lets us distinguish the two at compile time so the surrogate-pair paths -// are excluded entirely (not just constant-folded) where wchar_t is wide enough to hold a codepoint. -#if defined(WCHAR_MAX) && (WCHAR_MAX <= 0xFFFF) -#define UTF8_WCHAR_IS_UTF16 1 -#else -#define UTF8_WCHAR_IS_UTF16 0 -#endif +#include +#include + +#ifdef _WIN32 + +#include + +namespace +{ + +typedef unsigned short IcuChar; +typedef int IcuChar32; +typedef int IcuErrorCode; +typedef IcuChar* (__cdecl* IcuStrFromUtf8)( + IcuChar*, int, int*, const char*, int, IcuErrorCode*); +typedef char* (__cdecl* IcuStrToUtf8WithSub)( + char*, int, int*, const IcuChar*, int, IcuChar32, int*, IcuErrorCode*); + +enum +{ + ICU_ZERO_ERROR = 0, + ICU_BUFFER_OVERFLOW_ERROR = 15, + ICU_REPLACEMENT_CHARACTER = 0xFFFD +}; + +class WindowsIcuFunctions +{ +public: + WindowsIcuFunctions() : m_fromUtf8(nullptr), m_toUtf8WithSub(nullptr), m_module(nullptr) + { + char systemIcuPath[MAX_PATH]; + const UINT systemDirectoryLength = GetSystemDirectoryA(systemIcuPath, MAX_PATH); + if (systemDirectoryLength > 0 && systemDirectoryLength <= MAX_PATH - sizeof("\\icu.dll")) + { + memcpy(systemIcuPath + systemDirectoryLength, "\\icu.dll", sizeof("\\icu.dll")); + m_module = LoadLibraryA(systemIcuPath); + } + + if (m_module != nullptr) + { + m_fromUtf8 = reinterpret_cast(GetProcAddress(m_module, "u_strFromUTF8")); + m_toUtf8WithSub = reinterpret_cast(GetProcAddress(m_module, "u_strToUTF8WithSub")); + } + + if (m_fromUtf8 == nullptr || m_toUtf8WithSub == nullptr) + { + if (m_module != nullptr) + { + FreeLibrary(m_module); + } + + m_module = nullptr; + m_fromUtf8 = nullptr; + m_toUtf8WithSub = nullptr; + } + } + + bool isAvailable() const + { + return m_module != nullptr; + } -static const unsigned int UTF8_CODEPOINT_MAX = 0x10FFFF; -static const unsigned int UTF8_SURROGATE_MIN = 0xD800; -static const unsigned int UTF8_SURROGATE_MAX = 0xDFFF; -static const unsigned int UTF8_REPLACEMENT_CHAR = 0xFFFD; + IcuStrFromUtf8 m_fromUtf8; + IcuStrToUtf8WithSub m_toUtf8WithSub; -// Number of UTF-8 bytes required to encode a codepoint. -static size_t Utf8_Encoded_Length(unsigned int cp) +private: + HMODULE m_module; +}; + +// Keep ICU loaded for the process lifetime so conversions remain safe during global destruction. +WindowsIcuFunctions g_icu; + +bool FitsInt(size_t length) +{ + return length <= static_cast(INT_MAX); +} + +bool IcuPreflightSucceeded(IcuErrorCode error) +{ + return error <= ICU_ZERO_ERROR || error == ICU_BUFFER_OVERFLOW_ERROR; +} + +bool IcuConversionSucceeded(IcuErrorCode error) { - if (cp < 0x80) - { - return 1; - } - if (cp < 0x800) - { - return 2; - } - if (cp < 0x10000) - { - return 3; - } - return 4; + return error <= ICU_ZERO_ERROR; } -// Encode a codepoint to dest, which is assumed to have room. Returns the number of bytes written. -static size_t Utf8_Encode(char* dest, unsigned int cp) +size_t IcuWideToUtf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) { - if (cp < 0x80) - { - dest[0] = (char)cp; - return 1; - } - if (cp < 0x800) - { - dest[0] = (char)(0xC0 | (cp >> 6)); - dest[1] = (char)(0x80 | (cp & 0x3F)); - return 2; - } - if (cp < 0x10000) - { - dest[0] = (char)(0xE0 | (cp >> 12)); - dest[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); - dest[2] = (char)(0x80 | (cp & 0x3F)); - return 3; - } - dest[0] = (char)(0xF0 | (cp >> 18)); - dest[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); - dest[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); - dest[3] = (char)(0x80 | (cp & 0x3F)); - return 4; + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + WWASSERT(false); + return 0; + } + + WWASSERT(sizeof(wchar_t) == sizeof(IcuChar)); + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_toUtf8WithSub(dest, static_cast(destLen), &outputLength, + reinterpret_cast(src), static_cast(srcLen), ICU_REPLACEMENT_CHARACTER, nullptr, &error); + if (!IcuConversionSucceeded(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); } -// Decode one UTF-8 sequence at src, with srcLen bytes remaining. On success returns the number of -// bytes consumed (1-4) and sets cp. Returns 0 on any malformed, overlong, out-of-range or surrogate -// encoding. -static size_t Utf8_Decode(const char* src, size_t srcLen, unsigned int& cp) +size_t IcuWideToUtf8Len(const wchar_t* src, size_t srcLen) { - const unsigned char lead = (unsigned char)src[0]; - if (lead < 0x80) - { - cp = lead; - return 1; - } - - size_t count; - unsigned int lowerBound; - if ((lead & 0xE0) == 0xC0) - { - count = 2; - cp = lead & 0x1F; - lowerBound = 0x80; - } - else if ((lead & 0xF0) == 0xE0) - { - count = 3; - cp = lead & 0x0F; - lowerBound = 0x800; - } - else if ((lead & 0xF8) == 0xF0) - { - count = 4; - cp = lead & 0x07; - lowerBound = 0x10000; - } - else - { - return 0; // a continuation byte or a 5/6-byte form cannot start a sequence - } - - if (srcLen < count) - { - return 0; // truncated sequence - } - for (size_t i = 1; i < count; ++i) - { - const unsigned char trail = (unsigned char)src[i]; - if ((trail & 0xC0) != 0x80) - { - return 0; // not a continuation byte - } - cp = (cp << 6) | (trail & 0x3F); - } - - if (cp < lowerBound || cp > UTF8_CODEPOINT_MAX || (cp >= UTF8_SURROGATE_MIN && cp <= UTF8_SURROGATE_MAX)) - { - return 0; // overlong, out of range, or a surrogate codepoint - } - return count; + if (!FitsInt(srcLen)) + { + WWASSERT(false); + return 0; + } + + WWASSERT(sizeof(wchar_t) == sizeof(IcuChar)); + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_toUtf8WithSub(nullptr, 0, &outputLength, reinterpret_cast(src), + static_cast(srcLen), ICU_REPLACEMENT_CHARACTER, nullptr, &error); + if (!IcuPreflightSucceeded(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); } -// Read one codepoint at src, with srcLen wide characters remaining. Returns the number of wide -// characters consumed (1-2) and sets cp. Combines UTF-16 surrogate pairs where wchar_t is 16-bit; -// treats each element as a whole codepoint where wchar_t is 32-bit. Wide data that has no UTF-8 -// representation is reported as U+FFFD, so the encoder never emits a sequence that the decoder -// would reject. -static size_t Wide_Read(const wchar_t* src, size_t srcLen, unsigned int& cp) +size_t IcuUtf8ToWide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) { - size_t consumed = 1; -#if UTF8_WCHAR_IS_UTF16 - cp = (unsigned int)src[0] & 0xFFFF; - if (cp >= UTF8_SURROGATE_MIN && cp <= 0xDBFF && srcLen > 1) - { - const unsigned int low = (unsigned int)src[1] & 0xFFFF; - if (low >= 0xDC00 && low <= UTF8_SURROGATE_MAX) - { - cp = 0x10000 + ((cp - UTF8_SURROGATE_MIN) << 10) + (low - 0xDC00); - consumed = 2; - } - } + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + WWASSERT(sizeof(wchar_t) == sizeof(IcuChar)); + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_fromUtf8(reinterpret_cast(dest), static_cast(destLen), &outputLength, + src, static_cast(srcLen), &error); + if (!IcuConversionSucceeded(error)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + return static_cast(outputLength); +} + +size_t IcuUtf8ToWideLen(const char* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + return UTF8_INVALID; + } + + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_fromUtf8(nullptr, 0, &outputLength, src, static_cast(srcLen), &error); + if (!IcuPreflightSucceeded(error)) + { + return UTF8_INVALID; + } + + return static_cast(outputLength); +} + +size_t WindowsWideToUtf8Len(const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + WWASSERT(false); + return 0; + } + + const int outputLength = WideCharToMultiByte(CP_UTF8, 0, src, static_cast(srcLen), nullptr, 0, nullptr, nullptr); + if (outputLength == 0 && srcLen != 0) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); +} + +size_t WindowsWideToUtf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + WWASSERT(false); + return 0; + } + + const int outputLength = WideCharToMultiByte(CP_UTF8, 0, src, static_cast(srcLen), + dest, static_cast(destLen), nullptr, nullptr); + if (outputLength == 0 && srcLen != 0) + { + WWASSERT(false); + return 0; + } + + if (static_cast(outputLength) < destLen) + { + dest[outputLength] = '\0'; + } + + return static_cast(outputLength); +} + +size_t WindowsUtf8ToWideLen(const char* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + return UTF8_INVALID; + } + + const int outputLength = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, src, + static_cast(srcLen), nullptr, 0); + if (outputLength == 0 && srcLen != 0) + { + return UTF8_INVALID; + } + + return static_cast(outputLength); +} + +size_t WindowsUtf8ToWide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +{ + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + const int outputLength = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, src, + static_cast(srcLen), dest, static_cast(destLen)); + if (outputLength == 0 && srcLen != 0) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + if (static_cast(outputLength) < destLen) + { + dest[outputLength] = L'\0'; + } + + return static_cast(outputLength); +} + +} // namespace + +size_t Wide_To_Utf8_Len(const wchar_t* src, size_t srcLen) +{ + if (g_icu.isAvailable()) + { + return IcuWideToUtf8Len(src, srcLen); + } + + return WindowsWideToUtf8Len(src, srcLen); +} + +size_t Utf8_To_Wide_Len(const char* src, size_t srcLen) +{ + if (g_icu.isAvailable()) + { + return IcuUtf8ToWideLen(src, srcLen); + } + + return WindowsUtf8ToWideLen(src, srcLen); +} + +size_t Wide_To_Utf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) +{ + if (g_icu.isAvailable()) + { + return IcuWideToUtf8(dest, destLen, src, srcLen); + } + + return WindowsWideToUtf8(dest, destLen, src, srcLen); +} + +size_t Utf8_To_Wide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +{ + if (g_icu.isAvailable()) + { + return IcuUtf8ToWide(dest, destLen, src, srcLen); + } + + return WindowsUtf8ToWide(dest, destLen, src, srcLen); +} + #else - (void)srcLen; - cp = (unsigned int)src[0]; -#endif - if (cp > UTF8_CODEPOINT_MAX || (cp >= UTF8_SURROGATE_MIN && cp <= UTF8_SURROGATE_MAX)) - { - cp = UTF8_REPLACEMENT_CHAR; - } - return consumed; + +#include + +#include + +namespace +{ + +bool FitsIcuLength(size_t length) +{ + return length <= static_cast(INT32_MAX); +} + +bool IcuPreflightSucceeded(UErrorCode error) +{ + return U_SUCCESS(error) || error == U_BUFFER_OVERFLOW_ERROR; } -// Number of wide characters required to store a codepoint. -static size_t Wide_Encoded_Length(unsigned int cp) +bool WideToUtf16(std::vector& utf16, const wchar_t* src, size_t srcLen) { -#if UTF8_WCHAR_IS_UTF16 - return (cp >= 0x10000) ? 2 : 1; + if (!FitsIcuLength(srcLen)) + { + return false; + } + +#if defined(WCHAR_MAX) && (WCHAR_MAX <= 0xFFFF) + utf16.assign(reinterpret_cast(src), reinterpret_cast(src) + srcLen); + utf16.push_back(0); + return true; #else - (void)cp; - return 1; + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strFromUTF32WithSub(nullptr, 0, &outputLength, reinterpret_cast(src), + static_cast(srcLen), 0xFFFD, nullptr, &error); + if (!IcuPreflightSucceeded(error)) + { + return false; + } + + utf16.resize(static_cast(outputLength) + 1); + error = U_ZERO_ERROR; + u_strFromUTF32WithSub(&utf16[0], static_cast(utf16.size()), &outputLength, + reinterpret_cast(src), static_cast(srcLen), 0xFFFD, nullptr, &error); + return U_SUCCESS(error); #endif } -// Write one codepoint to a wide buffer, which is assumed to have room. Returns wide characters written. -static size_t Wide_Write(wchar_t* dest, unsigned int cp) +bool Utf8ToUtf16(std::vector& utf16, const char* src, size_t srcLen) { -#if UTF8_WCHAR_IS_UTF16 - if (cp >= 0x10000) - { - cp -= 0x10000; - dest[0] = (wchar_t)(0xD800 + (cp >> 10)); - dest[1] = (wchar_t)(0xDC00 + (cp & 0x3FF)); - return 2; - } -#endif - dest[0] = (wchar_t)cp; - return 1; + if (!FitsIcuLength(srcLen)) + { + return false; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strFromUTF8(nullptr, 0, &outputLength, src, static_cast(srcLen), &error); + if (!IcuPreflightSucceeded(error)) + { + return false; + } + + utf16.resize(static_cast(outputLength) + 1); + error = U_ZERO_ERROR; + u_strFromUTF8(&utf16[0], static_cast(utf16.size()), &outputLength, + src, static_cast(srcLen), &error); + return U_SUCCESS(error); } +} // namespace + size_t Wide_To_Utf8_Len(const wchar_t* src, size_t srcLen) { - size_t needed = 0; - size_t i = 0; - while (i < srcLen) - { - unsigned int cp; - i += Wide_Read(src + i, srcLen - i, cp); - needed += Utf8_Encoded_Length(cp); - } - return needed; + std::vector utf16; + if (!WideToUtf16(utf16, src, srcLen)) + { + WWASSERT(false); + return 0; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF8WithSub(nullptr, 0, &outputLength, utf16.empty() ? nullptr : &utf16[0], + static_cast(utf16.empty() ? 0 : utf16.size() - 1), 0xFFFD, nullptr, &error); + if (!IcuPreflightSucceeded(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); } size_t Utf8_To_Wide_Len(const char* src, size_t srcLen) { - size_t needed = 0; - size_t i = 0; - while (i < srcLen) - { - unsigned int cp; - const size_t consumed = Utf8_Decode(src + i, srcLen - i, cp); - if (consumed == 0) - { - return UTF8_INVALID; - } - i += consumed; - needed += Wide_Encoded_Length(cp); - } - return needed; + std::vector utf16; + if (!Utf8ToUtf16(utf16, src, srcLen)) + { + return UTF8_INVALID; + } + +#if defined(WCHAR_MAX) && (WCHAR_MAX <= 0xFFFF) + return utf16.empty() ? 0 : utf16.size() - 1; +#else + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF32(nullptr, 0, &outputLength, utf16.empty() ? nullptr : &utf16[0], + static_cast(utf16.empty() ? 0 : utf16.size() - 1), &error); + if (!IcuPreflightSucceeded(error)) + { + return UTF8_INVALID; + } + + return static_cast(outputLength); +#endif } size_t Wide_To_Utf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) { - size_t needed = 0; - size_t out = 0; - size_t i = 0; - while (i < srcLen) - { - unsigned int cp; - i += Wide_Read(src + i, srcLen - i, cp); - const size_t need = Utf8_Encoded_Length(cp); - // Stop writing at the first codepoint that does not fit, but keep counting for the caller. - if (needed == out && out + need <= destLen) - { - out += Utf8_Encode(dest + out, cp); - } - needed += need; - } - if (out < destLen) - { - dest[out] = '\0'; - } - return needed; + if (!FitsIcuLength(destLen)) + { + WWASSERT(false); + return 0; + } + + std::vector utf16; + if (!WideToUtf16(utf16, src, srcLen)) + { + WWASSERT(false); + return 0; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF8WithSub(dest, static_cast(destLen), &outputLength, + utf16.empty() ? nullptr : &utf16[0], static_cast(utf16.empty() ? 0 : utf16.size() - 1), + 0xFFFD, nullptr, &error); + if (U_FAILURE(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); } size_t Utf8_To_Wide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) { - size_t needed = 0; - size_t out = 0; - size_t i = 0; - while (i < srcLen) - { - unsigned int cp; - const size_t consumed = Utf8_Decode(src + i, srcLen - i, cp); - if (consumed == 0) - { - if (destLen > 0) - { - dest[0] = L'\0'; - } - return UTF8_INVALID; - } - i += consumed; - const size_t need = Wide_Encoded_Length(cp); - // Stop writing at the first codepoint that does not fit, but keep counting for the caller. - if (needed == out && out + need <= destLen) - { - out += Wide_Write(dest + out, cp); - } - needed += need; - } - if (out < destLen) - { - dest[out] = L'\0'; - } - return needed; + if (!FitsIcuLength(destLen)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + std::vector utf16; + if (!Utf8ToUtf16(utf16, src, srcLen)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + +#if defined(WCHAR_MAX) && (WCHAR_MAX <= 0xFFFF) + const size_t outputLength = utf16.empty() ? 0 : utf16.size() - 1; + if (outputLength > destLen) + { + WWASSERT(false); + return UTF8_INVALID; + } + + for (size_t i = 0; i < outputLength; ++i) + { + dest[i] = static_cast(utf16[i]); + } + + if (outputLength < destLen) + { + dest[outputLength] = L'\0'; + } + + return outputLength; +#else + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF32(reinterpret_cast(dest), static_cast(destLen), &outputLength, + utf16.empty() ? nullptr : &utf16[0], static_cast(utf16.empty() ? 0 : utf16.size() - 1), &error); + if (U_FAILURE(error)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + return static_cast(outputLength); +#endif } + +#endif diff --git a/Core/Libraries/Source/WWVegas/WWLib/utf8.h b/Core/Libraries/Source/WWVegas/WWLib/utf8.h index 7424943b014..4ea226bf2e6 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/utf8.h +++ b/Core/Libraries/Source/WWVegas/WWLib/utf8.h @@ -21,37 +21,19 @@ #include #include -// UTF-8 <-> wide-character transcoding, hand-rolled per RFC 3629, using no platform text APIs. -// The wide side is wchar_t, whose width is platform-dependent: on Windows it is a 16-bit UTF-16 -// code unit (astral codepoints use surrogate pairs); on most other platforms it is a 32-bit -// UTF-32 codepoint. Both are handled transparently based on the width of wchar_t. +// UTF-8 <-> wide-character conversion backed by ICU4C. Windows dynamically uses the ICU4C +// implementation shipped with the operating system and falls back to the native text APIs when +// ICU is unavailable. Other platforms link ICU4C's common library. -// Returned by the decoding functions when the source is not well-formed UTF-8. A return of 0 means -// an empty result, which is a success and must not be confused with a decoding failure. +// Returned when UTF-8 input is malformed. Zero is reserved for a successful empty conversion. const size_t UTF8_INVALID = (size_t)-1; -// Returns the number of UTF-8 bytes needed for the UTF-8 representation of srcLen wide characters -// from src, not counting a null terminator. Returns 0 if srcLen is 0. Wide values that have no -// UTF-8 representation are counted as U+FFFD. +// Return the required destination length without counting a null terminator. size_t Wide_To_Utf8_Len(const wchar_t* src, size_t srcLen); - -// Returns the number of wide characters needed for the wide representation of srcLen bytes from the -// UTF-8 string src, not counting a null terminator. Returns 0 if srcLen is 0, or UTF8_INVALID if -// src is not well-formed UTF-8. size_t Utf8_To_Wide_Len(const char* src, size_t srcLen); -// Converts srcLen wide characters from src to UTF-8. destLen is the destination buffer capacity in -// bytes. Writes a null terminator if room remains, otherwise not. Wide values that have no UTF-8 -// representation are written as U+FFFD, so the output always decodes back through Utf8_To_Wide. -// Returns the number of bytes the whole conversion needs, not counting a null terminator. A return -// greater than destLen means the output was truncated on a codepoint boundary; retry with that many -// bytes plus one for the terminator. Pass destLen 0 to measure without writing. +// Convert exactly srcLen source units. The destination is null-terminated when it has spare +// capacity. Wide input that cannot be represented as Unicode is replaced with U+FFFD. Malformed +// UTF-8 returns UTF8_INVALID and clears dest when destLen is nonzero. size_t Wide_To_Utf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen); - -// Converts srcLen bytes from the UTF-8 string src to wide characters. destLen is the destination -// buffer capacity in wide characters. Writes a null terminator if room remains, otherwise not. -// Returns the number of wide characters the whole conversion needs, not counting a null terminator. -// A return greater than destLen means the output was truncated on a codepoint boundary; retry with -// that many wide characters plus one for the terminator. Pass destLen 0 to measure without writing. -// Returns UTF8_INVALID if src is not well-formed UTF-8, setting dest[0] to L'\0' if destLen > 0. size_t Utf8_To_Wide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen); diff --git a/vcpkg-lock.json b/vcpkg-lock.json index 422998f0378..5926be63217 100644 --- a/vcpkg-lock.json +++ b/vcpkg-lock.json @@ -7,12 +7,24 @@ "port-version": 1, "git-tree": "6ff75f1f596ada519241989f44077cda442480b2" }, + { + "name": "icu", + "version-string": "74.2", + "port-version": 5, + "git-tree": "91b03be43850140e64dba01dcadc8b3f73b1083b" + }, { "name": "pkgconf", "version-string": "2.3.0", "port-version": 0, "git-tree": "ae3886d8a627ec99dd18890389b6d5d331e29799" }, + { + "name": "stb", + "version-string": "2024-07-29", + "port-version": 1, + "git-tree": "c362152d1d02973e7b1ce777960f8a6876656560" + }, { "name": "vcpkg-cmake", "version-string": "2024-04-23", @@ -25,6 +37,12 @@ "port-version": 0, "git-tree": "f23148add155147f3d95ae622d3b0031beb25acf" }, + { + "name": "vcpkg-make", + "version-string": "2025-02-08", + "port-version": 0, + "git-tree": "84480c9e9de4cc4cf60f44d5de24c79d61dfe966" + }, { "name": "vcpkg-pkgconfig-get-modules", "version-string": "2024-04-03", diff --git a/vcpkg.json b/vcpkg.json index 9ce3c6667c3..08039cd140f 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -3,6 +3,10 @@ "builtin-baseline": "b02e341c927f16d991edbd915d8ea43eac52096c", "dependencies": [ "zlib", + { + "name": "icu", + "platform": "!windows" + }, "ffmpeg", "stb" ] From 18055af0d4e799d209bcb59fc26561bbe176e9c5 Mon Sep 17 00:00:00 2001 From: Jacob Ledbetter Date: Thu, 20 Aug 2026 17:33:45 -0600 Subject: [PATCH 2/5] feat(icu): Link the full ICU4C suite across toolchains Use vcpkg or the Windows SDK C API on modern builds, keep VC6 on runtime LoadLibrary, and only probe system icu.dll for the delay-loaded SDK path. --- CMakeLists.txt | 1 + Core/GameEngine/CMakeLists.txt | 1 + .../Source/WWVegas/WWLib/CMakeLists.txt | 8 +- .../Source/WWVegas/WWLib/IcuSupport.h | 58 ++ Core/Libraries/Source/WWVegas/WWLib/utf8.cpp | 559 +++++++++++------- Core/Libraries/Source/WWVegas/WWLib/utf8.h | 7 +- cmake/icu.cmake | 74 +++ vcpkg.json | 5 +- 8 files changed, 481 insertions(+), 232 deletions(-) create mode 100644 Core/Libraries/Source/WWVegas/WWLib/IcuSupport.h create mode 100644 cmake/icu.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 39062e3fbe6..1ebf375d4c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,6 +63,7 @@ else() endif() include(cmake/config.cmake) +include(cmake/icu.cmake) include(cmake/gamespy.cmake) include(cmake/lzhl.cmake) include(cmake/stb.cmake) diff --git a/Core/GameEngine/CMakeLists.txt b/Core/GameEngine/CMakeLists.txt index 0b48d20c4eb..213ff1889d8 100644 --- a/Core/GameEngine/CMakeLists.txt +++ b/Core/GameEngine/CMakeLists.txt @@ -1198,6 +1198,7 @@ target_link_libraries(corei_gameengine_public INTERFACE core_compression core_browserdispatch #core_wwvegas + core_icu d3d8lib gamespy::gamespy stlport diff --git a/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt b/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt index 882c60ce586..1ba05b0e393 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt +++ b/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt @@ -48,6 +48,7 @@ set(WWLIB_SRC #global.h hash.cpp hash.h + IcuSupport.h hashcalc.h HASHLIST.h #hashtab.h @@ -189,7 +190,6 @@ target_link_libraries(core_wwlib PRIVATE corei_always ) -if(NOT WIN32) - find_package(ICU REQUIRED COMPONENTS uc data) - target_link_libraries(core_wwlib PRIVATE ICU::uc ICU::data) -endif() +target_link_libraries(core_wwlib PUBLIC + core_icu +) diff --git a/Core/Libraries/Source/WWVegas/WWLib/IcuSupport.h b/Core/Libraries/Source/WWVegas/WWLib/IcuSupport.h new file mode 100644 index 00000000000..6ce7ff720f2 --- /dev/null +++ b/Core/Libraries/Source/WWVegas/WWLib/IcuSupport.h @@ -0,0 +1,58 @@ +/* +** Command & Conquer Generals Zero Hour(tm) +** Copyright 2026 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +#pragma once + +// Engine entry point for ICU4C. +// +// RTS_HAS_ICU - ICU C API is linked; include this header and call ICU functions. +// RTS_HAS_ICU_CXX - ICU C++ API (icu::UnicodeString, icu::Locale, ...). +// RTS_HAS_ICU_I18N - Collation, break iteration, converters, and related i18n APIs. +// RTS_HAS_ICU_WINSDK - Windows SDK merged C API via (no C++ API). +// RTS_ICU_DYNAMIC - No import library; utf8.cpp LoadLibrary's OS icu.dll (VC6). + +#if defined(RTS_HAS_ICU_WINSDK) + +#include + +#elif defined(RTS_HAS_ICU) + +#include +#include +#include +#include +#include + +#if defined(RTS_HAS_ICU_I18N) +#include +#include +#include +#include +#include +#endif + +#if defined(RTS_HAS_ICU_CXX) +#include +#include +#include +#if defined(RTS_HAS_ICU_I18N) +#include +#endif +#endif + +#endif diff --git a/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp b/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp index 005fb3ff681..cd710c2ac1e 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp +++ b/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp @@ -22,180 +22,49 @@ #include #include +#if defined(RTS_HAS_ICU_WINSDK) +#include +#include +#elif defined(RTS_HAS_ICU) +#include #ifdef _WIN32 - #include +#endif +#include +#elif defined(_WIN32) +#include +#endif namespace { -typedef unsigned short IcuChar; -typedef int IcuChar32; -typedef int IcuErrorCode; -typedef IcuChar* (__cdecl* IcuStrFromUtf8)( - IcuChar*, int, int*, const char*, int, IcuErrorCode*); -typedef char* (__cdecl* IcuStrToUtf8WithSub)( - char*, int, int*, const IcuChar*, int, IcuChar32, int*, IcuErrorCode*); - -enum -{ - ICU_ZERO_ERROR = 0, - ICU_BUFFER_OVERFLOW_ERROR = 15, - ICU_REPLACEMENT_CHARACTER = 0xFFFD -}; - -class WindowsIcuFunctions -{ -public: - WindowsIcuFunctions() : m_fromUtf8(nullptr), m_toUtf8WithSub(nullptr), m_module(nullptr) - { - char systemIcuPath[MAX_PATH]; - const UINT systemDirectoryLength = GetSystemDirectoryA(systemIcuPath, MAX_PATH); - if (systemDirectoryLength > 0 && systemDirectoryLength <= MAX_PATH - sizeof("\\icu.dll")) - { - memcpy(systemIcuPath + systemDirectoryLength, "\\icu.dll", sizeof("\\icu.dll")); - m_module = LoadLibraryA(systemIcuPath); - } - - if (m_module != nullptr) - { - m_fromUtf8 = reinterpret_cast(GetProcAddress(m_module, "u_strFromUTF8")); - m_toUtf8WithSub = reinterpret_cast(GetProcAddress(m_module, "u_strToUTF8WithSub")); - } - - if (m_fromUtf8 == nullptr || m_toUtf8WithSub == nullptr) - { - if (m_module != nullptr) - { - FreeLibrary(m_module); - } - - m_module = nullptr; - m_fromUtf8 = nullptr; - m_toUtf8WithSub = nullptr; - } - } - - bool isAvailable() const - { - return m_module != nullptr; - } - - IcuStrFromUtf8 m_fromUtf8; - IcuStrToUtf8WithSub m_toUtf8WithSub; - -private: - HMODULE m_module; -}; - -// Keep ICU loaded for the process lifetime so conversions remain safe during global destruction. -WindowsIcuFunctions g_icu; - bool FitsInt(size_t length) { return length <= static_cast(INT_MAX); } -bool IcuPreflightSucceeded(IcuErrorCode error) -{ - return error <= ICU_ZERO_ERROR || error == ICU_BUFFER_OVERFLOW_ERROR; -} - -bool IcuConversionSucceeded(IcuErrorCode error) -{ - return error <= ICU_ZERO_ERROR; -} - -size_t IcuWideToUtf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) -{ - if (!FitsInt(destLen) || !FitsInt(srcLen)) - { - WWASSERT(false); - return 0; - } - - WWASSERT(sizeof(wchar_t) == sizeof(IcuChar)); - IcuErrorCode error = ICU_ZERO_ERROR; - int outputLength = 0; - g_icu.m_toUtf8WithSub(dest, static_cast(destLen), &outputLength, - reinterpret_cast(src), static_cast(srcLen), ICU_REPLACEMENT_CHARACTER, nullptr, &error); - if (!IcuConversionSucceeded(error)) - { - WWASSERT(false); - return 0; - } - - return static_cast(outputLength); -} - -size_t IcuWideToUtf8Len(const wchar_t* src, size_t srcLen) -{ - if (!FitsInt(srcLen)) - { - WWASSERT(false); - return 0; - } - - WWASSERT(sizeof(wchar_t) == sizeof(IcuChar)); - IcuErrorCode error = ICU_ZERO_ERROR; - int outputLength = 0; - g_icu.m_toUtf8WithSub(nullptr, 0, &outputLength, reinterpret_cast(src), - static_cast(srcLen), ICU_REPLACEMENT_CHARACTER, nullptr, &error); - if (!IcuPreflightSucceeded(error)) - { - WWASSERT(false); - return 0; - } - - return static_cast(outputLength); -} +#ifdef _WIN32 -size_t IcuUtf8ToWide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +bool LoadSystemIcu() { - if (!FitsInt(destLen) || !FitsInt(srcLen)) - { - if (destLen > 0) - { - dest[0] = L'\0'; - } - - return UTF8_INVALID; - } - - WWASSERT(sizeof(wchar_t) == sizeof(IcuChar)); - IcuErrorCode error = ICU_ZERO_ERROR; - int outputLength = 0; - g_icu.m_fromUtf8(reinterpret_cast(dest), static_cast(destLen), &outputLength, - src, static_cast(srcLen), &error); - if (!IcuConversionSucceeded(error)) + static bool attempted = false; + static bool available = false; + if (attempted) { - if (destLen > 0) - { - dest[0] = L'\0'; - } - - return UTF8_INVALID; + return available; } - return static_cast(outputLength); -} - -size_t IcuUtf8ToWideLen(const char* src, size_t srcLen) -{ - if (!FitsInt(srcLen)) - { - return UTF8_INVALID; - } + attempted = true; - IcuErrorCode error = ICU_ZERO_ERROR; - int outputLength = 0; - g_icu.m_fromUtf8(nullptr, 0, &outputLength, src, static_cast(srcLen), &error); - if (!IcuPreflightSucceeded(error)) + char systemIcuPath[MAX_PATH]; + const UINT systemDirectoryLength = GetSystemDirectoryA(systemIcuPath, MAX_PATH); + if (systemDirectoryLength > 0 && systemDirectoryLength <= MAX_PATH - sizeof("\\icu.dll")) { - return UTF8_INVALID; + memcpy(systemIcuPath + systemDirectoryLength, "\\icu.dll", sizeof("\\icu.dll")); + available = LoadLibraryA(systemIcuPath) != nullptr; } - return static_cast(outputLength); + return available; } size_t WindowsWideToUtf8Len(const wchar_t* src, size_t srcLen) @@ -289,79 +158,120 @@ size_t WindowsUtf8ToWide(wchar_t* dest, size_t destLen, const char* src, size_t return static_cast(outputLength); } -} // namespace +#endif -size_t Wide_To_Utf8_Len(const wchar_t* src, size_t srcLen) +#if defined(RTS_HAS_ICU) + +bool IcuPreflightSucceeded(UErrorCode error) { - if (g_icu.isAvailable()) - { - return IcuWideToUtf8Len(src, srcLen); - } + return U_SUCCESS(error) || error == U_BUFFER_OVERFLOW_ERROR; +} - return WindowsWideToUtf8Len(src, srcLen); +bool IcuConversionSucceeded(UErrorCode error) +{ + return U_SUCCESS(error); } -size_t Utf8_To_Wide_Len(const char* src, size_t srcLen) +#if defined(WCHAR_MAX) && (WCHAR_MAX <= 0xFFFF) + +size_t IcuWideToUtf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) { - if (g_icu.isAvailable()) + if (!FitsInt(destLen) || !FitsInt(srcLen)) { - return IcuUtf8ToWideLen(src, srcLen); + WWASSERT(false); + return 0; } - return WindowsUtf8ToWideLen(src, srcLen); -} - -size_t Wide_To_Utf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) -{ - if (g_icu.isAvailable()) + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF8WithSub(dest, static_cast(destLen), &outputLength, + reinterpret_cast(src), static_cast(srcLen), 0xFFFD, nullptr, &error); + if (!IcuConversionSucceeded(error)) { - return IcuWideToUtf8(dest, destLen, src, srcLen); + WWASSERT(false); + return 0; } - return WindowsWideToUtf8(dest, destLen, src, srcLen); + return static_cast(outputLength); } -size_t Utf8_To_Wide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +size_t IcuWideToUtf8Len(const wchar_t* src, size_t srcLen) { - if (g_icu.isAvailable()) + if (!FitsInt(srcLen)) { - return IcuUtf8ToWide(dest, destLen, src, srcLen); + WWASSERT(false); + return 0; } - return WindowsUtf8ToWide(dest, destLen, src, srcLen); + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF8WithSub(nullptr, 0, &outputLength, reinterpret_cast(src), + static_cast(srcLen), 0xFFFD, nullptr, &error); + if (!IcuPreflightSucceeded(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); } -#else +size_t IcuUtf8ToWide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +{ + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } -#include + return UTF8_INVALID; + } -#include + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strFromUTF8(reinterpret_cast(dest), static_cast(destLen), &outputLength, + src, static_cast(srcLen), &error); + if (!IcuConversionSucceeded(error)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } -namespace -{ + return UTF8_INVALID; + } -bool FitsIcuLength(size_t length) -{ - return length <= static_cast(INT32_MAX); + return static_cast(outputLength); } -bool IcuPreflightSucceeded(UErrorCode error) +size_t IcuUtf8ToWideLen(const char* src, size_t srcLen) { - return U_SUCCESS(error) || error == U_BUFFER_OVERFLOW_ERROR; + if (!FitsInt(srcLen)) + { + return UTF8_INVALID; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strFromUTF8(nullptr, 0, &outputLength, src, static_cast(srcLen), &error); + if (!IcuPreflightSucceeded(error)) + { + return UTF8_INVALID; + } + + return static_cast(outputLength); } +#else + bool WideToUtf16(std::vector& utf16, const wchar_t* src, size_t srcLen) { - if (!FitsIcuLength(srcLen)) + if (!FitsInt(srcLen)) { return false; } -#if defined(WCHAR_MAX) && (WCHAR_MAX <= 0xFFFF) - utf16.assign(reinterpret_cast(src), reinterpret_cast(src) + srcLen); - utf16.push_back(0); - return true; -#else UErrorCode error = U_ZERO_ERROR; int32_t outputLength = 0; u_strFromUTF32WithSub(nullptr, 0, &outputLength, reinterpret_cast(src), @@ -376,12 +286,11 @@ bool WideToUtf16(std::vector& utf16, const wchar_t* src, size_t srcLen) u_strFromUTF32WithSub(&utf16[0], static_cast(utf16.size()), &outputLength, reinterpret_cast(src), static_cast(srcLen), 0xFFFD, nullptr, &error); return U_SUCCESS(error); -#endif } bool Utf8ToUtf16(std::vector& utf16, const char* src, size_t srcLen) { - if (!FitsIcuLength(srcLen)) + if (!FitsInt(srcLen)) { return false; } @@ -401,9 +310,7 @@ bool Utf8ToUtf16(std::vector& utf16, const char* src, size_t srcLen) return U_SUCCESS(error); } -} // namespace - -size_t Wide_To_Utf8_Len(const wchar_t* src, size_t srcLen) +size_t IcuWideToUtf8Len(const wchar_t* src, size_t srcLen) { std::vector utf16; if (!WideToUtf16(utf16, src, srcLen)) @@ -425,7 +332,7 @@ size_t Wide_To_Utf8_Len(const wchar_t* src, size_t srcLen) return static_cast(outputLength); } -size_t Utf8_To_Wide_Len(const char* src, size_t srcLen) +size_t IcuUtf8ToWideLen(const char* src, size_t srcLen) { std::vector utf16; if (!Utf8ToUtf16(utf16, src, srcLen)) @@ -433,9 +340,6 @@ size_t Utf8_To_Wide_Len(const char* src, size_t srcLen) return UTF8_INVALID; } -#if defined(WCHAR_MAX) && (WCHAR_MAX <= 0xFFFF) - return utf16.empty() ? 0 : utf16.size() - 1; -#else UErrorCode error = U_ZERO_ERROR; int32_t outputLength = 0; u_strToUTF32(nullptr, 0, &outputLength, utf16.empty() ? nullptr : &utf16[0], @@ -446,12 +350,11 @@ size_t Utf8_To_Wide_Len(const char* src, size_t srcLen) } return static_cast(outputLength); -#endif } -size_t Wide_To_Utf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) +size_t IcuWideToUtf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) { - if (!FitsIcuLength(destLen)) + if (!FitsInt(destLen)) { WWASSERT(false); return 0; @@ -478,9 +381,9 @@ size_t Wide_To_Utf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLe return static_cast(outputLength); } -size_t Utf8_To_Wide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +size_t IcuUtf8ToWide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) { - if (!FitsIcuLength(destLen)) + if (!FitsInt(destLen)) { if (destLen > 0) { @@ -501,31 +404,150 @@ size_t Utf8_To_Wide(wchar_t* dest, size_t destLen, const char* src, size_t srcLe return UTF8_INVALID; } -#if defined(WCHAR_MAX) && (WCHAR_MAX <= 0xFFFF) - const size_t outputLength = utf16.empty() ? 0 : utf16.size() - 1; - if (outputLength > destLen) + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF32(reinterpret_cast(dest), static_cast(destLen), &outputLength, + utf16.empty() ? nullptr : &utf16[0], static_cast(utf16.empty() ? 0 : utf16.size() - 1), &error); + if (U_FAILURE(error)) { - WWASSERT(false); + if (destLen > 0) + { + dest[0] = L'\0'; + } + return UTF8_INVALID; } - for (size_t i = 0; i < outputLength; ++i) + return static_cast(outputLength); +} + +#endif + +#elif defined(RTS_ICU_DYNAMIC) + +typedef unsigned short IcuChar; +typedef int IcuChar32; +typedef int IcuErrorCode; +typedef IcuChar* (__cdecl* IcuStrFromUtf8)( + IcuChar*, int, int*, const char*, int, IcuErrorCode*); +typedef char* (__cdecl* IcuStrToUtf8WithSub)( + char*, int, int*, const IcuChar*, int, IcuChar32, int*, IcuErrorCode*); + +enum +{ + ICU_ZERO_ERROR = 0, + ICU_BUFFER_OVERFLOW_ERROR = 15, + ICU_REPLACEMENT_CHARACTER = 0xFFFD +}; + +class WindowsIcuFunctions +{ +public: + WindowsIcuFunctions() : m_fromUtf8(nullptr), m_toUtf8WithSub(nullptr), m_module(nullptr) + { + if (!LoadSystemIcu()) + { + return; + } + + m_module = GetModuleHandleA("icu.dll"); + if (m_module != nullptr) + { + m_fromUtf8 = reinterpret_cast(GetProcAddress(m_module, "u_strFromUTF8")); + m_toUtf8WithSub = reinterpret_cast(GetProcAddress(m_module, "u_strToUTF8WithSub")); + } + + if (m_fromUtf8 == nullptr || m_toUtf8WithSub == nullptr) + { + m_module = nullptr; + m_fromUtf8 = nullptr; + m_toUtf8WithSub = nullptr; + } + } + + bool isAvailable() const { - dest[i] = static_cast(utf16[i]); + return m_fromUtf8 != nullptr && m_toUtf8WithSub != nullptr; } - if (outputLength < destLen) + IcuStrFromUtf8 m_fromUtf8; + IcuStrToUtf8WithSub m_toUtf8WithSub; + +private: + HMODULE m_module; +}; + +WindowsIcuFunctions g_icu; + +bool IcuPreflightSucceeded(IcuErrorCode error) +{ + return error <= ICU_ZERO_ERROR || error == ICU_BUFFER_OVERFLOW_ERROR; +} + +bool IcuConversionSucceeded(IcuErrorCode error) +{ + return error <= ICU_ZERO_ERROR; +} + +size_t IcuWideToUtf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(destLen) || !FitsInt(srcLen)) { - dest[outputLength] = L'\0'; + WWASSERT(false); + return 0; } - return outputLength; -#else - UErrorCode error = U_ZERO_ERROR; - int32_t outputLength = 0; - u_strToUTF32(reinterpret_cast(dest), static_cast(destLen), &outputLength, - utf16.empty() ? nullptr : &utf16[0], static_cast(utf16.empty() ? 0 : utf16.size() - 1), &error); - if (U_FAILURE(error)) + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_toUtf8WithSub(dest, static_cast(destLen), &outputLength, + reinterpret_cast(src), static_cast(srcLen), ICU_REPLACEMENT_CHARACTER, nullptr, &error); + if (!IcuConversionSucceeded(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); +} + +size_t IcuWideToUtf8Len(const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + WWASSERT(false); + return 0; + } + + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_toUtf8WithSub(nullptr, 0, &outputLength, reinterpret_cast(src), + static_cast(srcLen), ICU_REPLACEMENT_CHARACTER, nullptr, &error); + if (!IcuPreflightSucceeded(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); +} + +size_t IcuUtf8ToWide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +{ + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_fromUtf8(reinterpret_cast(dest), static_cast(destLen), &outputLength, + src, static_cast(srcLen), &error); + if (!IcuConversionSucceeded(error)) { if (destLen > 0) { @@ -536,7 +558,102 @@ size_t Utf8_To_Wide(wchar_t* dest, size_t destLen, const char* src, size_t srcLe } return static_cast(outputLength); +} + +size_t IcuUtf8ToWideLen(const char* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + return UTF8_INVALID; + } + + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_fromUtf8(nullptr, 0, &outputLength, src, static_cast(srcLen), &error); + if (!IcuPreflightSucceeded(error)) + { + return UTF8_INVALID; + } + + return static_cast(outputLength); +} + +#endif + +bool IcuIsAvailable() +{ +#if defined(RTS_HAS_ICU_WINSDK) + return LoadSystemIcu(); +#elif defined(RTS_HAS_ICU) + return true; +#elif defined(RTS_ICU_DYNAMIC) + return g_icu.isAvailable(); +#else + return false; #endif } +} // namespace + +size_t Wide_To_Utf8_Len(const wchar_t* src, size_t srcLen) +{ + if (IcuIsAvailable()) + { + return IcuWideToUtf8Len(src, srcLen); + } + +#ifdef _WIN32 + return WindowsWideToUtf8Len(src, srcLen); +#else + WWASSERT(false); + return 0; #endif +} + +size_t Utf8_To_Wide_Len(const char* src, size_t srcLen) +{ + if (IcuIsAvailable()) + { + return IcuUtf8ToWideLen(src, srcLen); + } + +#ifdef _WIN32 + return WindowsUtf8ToWideLen(src, srcLen); +#else + return UTF8_INVALID; +#endif +} + +size_t Wide_To_Utf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) +{ + if (IcuIsAvailable()) + { + return IcuWideToUtf8(dest, destLen, src, srcLen); + } + +#ifdef _WIN32 + return WindowsWideToUtf8(dest, destLen, src, srcLen); +#else + WWASSERT(false); + return 0; +#endif +} + +size_t Utf8_To_Wide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +{ + if (IcuIsAvailable()) + { + return IcuUtf8ToWide(dest, destLen, src, srcLen); + } + +#ifdef _WIN32 + return WindowsUtf8ToWide(dest, destLen, src, srcLen); +#else + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; +#endif +} diff --git a/Core/Libraries/Source/WWVegas/WWLib/utf8.h b/Core/Libraries/Source/WWVegas/WWLib/utf8.h index 4ea226bf2e6..e9a983ebdf8 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/utf8.h +++ b/Core/Libraries/Source/WWVegas/WWLib/utf8.h @@ -21,9 +21,10 @@ #include #include -// UTF-8 <-> wide-character conversion backed by ICU4C. Windows dynamically uses the ICU4C -// implementation shipped with the operating system and falls back to the native text APIs when -// ICU is unavailable. Other platforms link ICU4C's common library. +// UTF-8 <-> wide-character conversion backed by ICU4C. +// Modern toolchains link the ICU C API (Windows SDK or vcpkg). VC6 LoadLibrary's OS icu.dll +// and falls back to Win32 CP_UTF8 when it is missing. Include WWLib/IcuSupport.h to use the +// rest of the linked ICU suite from engine code. // Returned when UTF-8 input is malformed. Zero is reserved for a successful empty conversion. const size_t UTF8_INVALID = (size_t)-1; diff --git a/cmake/icu.cmake b/cmake/icu.cmake new file mode 100644 index 00000000000..6c8e99ec752 --- /dev/null +++ b/cmake/icu.cmake @@ -0,0 +1,74 @@ +# ICU4C for the engine. +# +# Preference order: +# 1. find_package(ICU) from vcpkg or the system (C + C++ APIs: uc, i18n, data) +# 2. Windows SDK icu.lib for modern MSVC (C API: common + i18n via ) +# 3. Runtime LoadLibrary of OS icu.dll (VC6 and other Windows toolchains without an import lib) +# +# VC6 cannot consume modern ICU headers or import libraries. It keeps the dynamic loader. + +add_library(core_icu INTERFACE) + +set(RTS_ICU_LINKED FALSE) +set(RTS_ICU_CXX FALSE) +set(RTS_ICU_I18N FALSE) +set(RTS_ICU_WINSDK FALSE) +set(RTS_ICU_DYNAMIC FALSE) + +if(NOT IS_VS6_BUILD) + find_package(ICU QUIET COMPONENTS uc i18n data) + if(NOT ICU_FOUND) + find_package(ICU QUIET COMPONENTS uc i18n) + endif() + if(NOT ICU_FOUND) + find_package(ICU QUIET COMPONENTS uc data) + endif() + if(NOT ICU_FOUND) + find_package(ICU QUIET COMPONENTS uc) + endif() +endif() + +if(ICU_FOUND AND NOT IS_VS6_BUILD) + set(RTS_ICU_LINKED TRUE) + set(RTS_ICU_CXX TRUE) + target_link_libraries(core_icu INTERFACE ICU::uc) + if(TARGET ICU::i18n) + set(RTS_ICU_I18N TRUE) + target_link_libraries(core_icu INTERFACE ICU::i18n) + endif() + if(TARGET ICU::data) + target_link_libraries(core_icu INTERFACE ICU::data) + endif() + target_compile_definitions(core_icu INTERFACE RTS_HAS_ICU RTS_HAS_ICU_CXX) + if(RTS_ICU_I18N) + target_compile_definitions(core_icu INTERFACE RTS_HAS_ICU_I18N) + endif() + message(STATUS "ICU4C: linked via find_package (C++ API enabled)") +elseif(WIN32 AND NOT IS_VS6_BUILD AND NOT MINGW) + find_library(RTS_ICU_WINSDK_LIB NAMES icu) + if(RTS_ICU_WINSDK_LIB) + set(RTS_ICU_LINKED TRUE) + set(RTS_ICU_I18N TRUE) + set(RTS_ICU_WINSDK TRUE) + target_link_libraries(core_icu INTERFACE icu delayimp) + target_link_options(core_icu INTERFACE "/DELAYLOAD:icu.dll") + target_compile_definitions(core_icu INTERFACE RTS_HAS_ICU RTS_HAS_ICU_WINSDK RTS_HAS_ICU_I18N) + message(STATUS "ICU4C: linked via Windows SDK (${RTS_ICU_WINSDK_LIB})") + endif() +endif() + +if(NOT RTS_ICU_LINKED) + if(WIN32) + set(RTS_ICU_DYNAMIC TRUE) + target_compile_definitions(core_icu INTERFACE RTS_ICU_DYNAMIC) + message(STATUS "ICU4C: runtime load of system icu.dll") + else() + message(FATAL_ERROR "ICU4C is required on non-Windows platforms. Install libicu or enable vcpkg.") + endif() +endif() + +add_feature_info(IcuLinked RTS_ICU_LINKED "Link ICU4C into the engine") +add_feature_info(IcuCxx RTS_ICU_CXX "ICU C++ API available") +add_feature_info(IcuI18n RTS_ICU_I18N "ICU i18n API available") +add_feature_info(IcuWinSdk RTS_ICU_WINSDK "Windows SDK ICU C API") +add_feature_info(IcuDynamic RTS_ICU_DYNAMIC "Load OS icu.dll at runtime") diff --git a/vcpkg.json b/vcpkg.json index 08039cd140f..cc243a0e053 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -3,10 +3,7 @@ "builtin-baseline": "b02e341c927f16d991edbd915d8ea43eac52096c", "dependencies": [ "zlib", - { - "name": "icu", - "platform": "!windows" - }, + "icu", "ffmpeg", "stb" ] From 1550d072f79afc5a81cacde3c1ac295d6c9c350e Mon Sep 17 00:00:00 2001 From: Jacob Ledbetter Date: Fri, 21 Aug 2026 10:19:12 -0600 Subject: [PATCH 3/5] docs(icu): Keep prior UTF-8 comments alongside the ICU notes Preserve bobtista's original change comments and append the ICU conversion notes instead of replacing them. --- Core/GameEngine/Source/Common/System/AsciiString.cpp | 1 + Core/GameEngine/Source/Common/System/UnicodeString.cpp | 3 +++ .../Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp | 1 + 3 files changed, 5 insertions(+) diff --git a/Core/GameEngine/Source/Common/System/AsciiString.cpp b/Core/GameEngine/Source/Common/System/AsciiString.cpp index 278fff90794..e95c74d99b5 100644 --- a/Core/GameEngine/Source/Common/System/AsciiString.cpp +++ b/Core/GameEngine/Source/Common/System/AsciiString.cpp @@ -308,6 +308,7 @@ char* AsciiString::getBufferForRead(Int len) void AsciiString::translate(const UnicodeString& stringSrc) { validate(); + // TheSuperHackers @fix bobtista 02/04/2026 Implement UTF-8 conversion replacing 7-bit ASCII only implementation // TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Convert wide text to UTF-8 with ICU4C. const WideChar* src = stringSrc.str(); const size_t srcLen = stringSrc.getLength(); diff --git a/Core/GameEngine/Source/Common/System/UnicodeString.cpp b/Core/GameEngine/Source/Common/System/UnicodeString.cpp index 4c6d6541b5a..e28a29c2fbd 100644 --- a/Core/GameEngine/Source/Common/System/UnicodeString.cpp +++ b/Core/GameEngine/Source/Common/System/UnicodeString.cpp @@ -222,6 +222,9 @@ WideChar* UnicodeString::getBufferForRead(Int len) void UnicodeString::translate(const AsciiString& stringSrc) { validate(); + // TheSuperHackers @fix bobtista 02/04/2026 Convert UTF-8 to wide, replacing the 7-bit ASCII only + // implementation. Data that is not valid UTF-8 (e.g. legacy CP1252) falls back to a 1:1 byte cast + // to preserve the original characters instead of producing replacement characters. // TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Convert UTF-8 to wide text with ICU4C. const char* src = stringSrc.str(); const size_t srcLen = stringSrc.getLength(); diff --git a/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp b/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp index 07920c2075f..9b7c16b7a65 100644 --- a/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp +++ b/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp @@ -32,6 +32,7 @@ //------------------------------------------------------------------------- +// TheSuperHackers @refactor bobtista 02/04/2026 Use WWLib UTF-8 functions instead of raw Win32 API calls // TheSuperHackers @refactor CryoTheRenegade 04/08/2026 Use the shared ICU4C UTF conversion functions. std::wstring MultiByteToWideCharSingleLine( const char* orig ) { From 3eb2eef8a8ed1d05e8ff44e4995178aa57f503ef Mon Sep 17 00:00:00 2001 From: Jacob Ledbetter Date: Sun, 23 Aug 2026 16:14:27 -0600 Subject: [PATCH 4/5] fix(icu): Probe icu.dll once and link the found SDK import lib Publish the availability result with InterlockedCompareExchange, search the normal DLL path instead of System32 only, and pass the CMake-found icu.lib into the link line. --- Core/Libraries/Source/WWVegas/WWLib/utf8.cpp | 32 ++++++++++++-------- cmake/icu.cmake | 2 +- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp b/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp index cd710c2ac1e..4b6ef0f37e0 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp +++ b/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp @@ -45,26 +45,34 @@ bool FitsInt(size_t length) #ifdef _WIN32 +#include "Utility/interlocked_adapter.h" + +enum +{ + IcuProbeUnknown = 0, + IcuProbeMissing = 1, + IcuProbeLoaded = 2 +}; + +// TheSuperHackers @fix CryoTheRenegade 23/08/2026 Probe icu.dll once through the normal DLL search order. bool LoadSystemIcu() { - static bool attempted = false; - static bool available = false; - if (attempted) + static volatile LONG cached = IcuProbeUnknown; + + const LONG existing = cached; + if (existing != IcuProbeUnknown) { - return available; + return existing == IcuProbeLoaded; } - attempted = true; - - char systemIcuPath[MAX_PATH]; - const UINT systemDirectoryLength = GetSystemDirectoryA(systemIcuPath, MAX_PATH); - if (systemDirectoryLength > 0 && systemDirectoryLength <= MAX_PATH - sizeof("\\icu.dll")) + const LONG result = LoadLibraryA("icu.dll") != nullptr ? IcuProbeLoaded : IcuProbeMissing; + const LONG previous = InterlockedCompareExchange(&cached, result, IcuProbeUnknown); + if (previous != IcuProbeUnknown) { - memcpy(systemIcuPath + systemDirectoryLength, "\\icu.dll", sizeof("\\icu.dll")); - available = LoadLibraryA(systemIcuPath) != nullptr; + return previous == IcuProbeLoaded; } - return available; + return result == IcuProbeLoaded; } size_t WindowsWideToUtf8Len(const wchar_t* src, size_t srcLen) diff --git a/cmake/icu.cmake b/cmake/icu.cmake index 6c8e99ec752..15555a0af8a 100644 --- a/cmake/icu.cmake +++ b/cmake/icu.cmake @@ -50,7 +50,7 @@ elseif(WIN32 AND NOT IS_VS6_BUILD AND NOT MINGW) set(RTS_ICU_LINKED TRUE) set(RTS_ICU_I18N TRUE) set(RTS_ICU_WINSDK TRUE) - target_link_libraries(core_icu INTERFACE icu delayimp) + target_link_libraries(core_icu INTERFACE ${RTS_ICU_WINSDK_LIB} delayimp) target_link_options(core_icu INTERFACE "/DELAYLOAD:icu.dll") target_compile_definitions(core_icu INTERFACE RTS_HAS_ICU RTS_HAS_ICU_WINSDK RTS_HAS_ICU_I18N) message(STATUS "ICU4C: linked via Windows SDK (${RTS_ICU_WINSDK_LIB})") From 0240a443c4b19ad680a39161ef9ccb731e59f6fb Mon Sep 17 00:00:00 2001 From: Jacob Ledbetter Date: Sun, 23 Aug 2026 16:29:39 -0600 Subject: [PATCH 5/5] fix(icu): Use InterlockedIncrement for the VC6 icu.dll probe Avoid InterlockedCompareExchange, whose VC6 and later SDK signatures disagree, so utf8.cpp compiles on both toolchains. --- Core/Libraries/Source/WWVegas/WWLib/utf8.cpp | 30 +++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp b/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp index 4b6ef0f37e0..6f61b3dca3e 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp +++ b/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp @@ -45,8 +45,6 @@ bool FitsInt(size_t length) #ifdef _WIN32 -#include "Utility/interlocked_adapter.h" - enum { IcuProbeUnknown = 0, @@ -58,21 +56,33 @@ enum bool LoadSystemIcu() { static volatile LONG cached = IcuProbeUnknown; + static volatile LONG initGate = 0; - const LONG existing = cached; - if (existing != IcuProbeUnknown) + if (cached != IcuProbeUnknown) { - return existing == IcuProbeLoaded; + return cached == IcuProbeLoaded; } - const LONG result = LoadLibraryA("icu.dll") != nullptr ? IcuProbeLoaded : IcuProbeMissing; - const LONG previous = InterlockedCompareExchange(&cached, result, IcuProbeUnknown); - if (previous != IcuProbeUnknown) + // InterlockedIncrement is LONG* on every supported SDK. InterlockedCompareExchange is not: + // VC6 winbase.h takes PVOID*, while later SDKs take LONG*. +#if defined(_MSC_VER) && _MSC_VER < 1300 + const LONG gate = InterlockedIncrement(const_cast(&initGate)); +#else + const LONG gate = InterlockedIncrement(&initGate); +#endif + if (gate == 1) { - return previous == IcuProbeLoaded; + cached = LoadLibraryA("icu.dll") != nullptr ? IcuProbeLoaded : IcuProbeMissing; + } + else + { + while (cached == IcuProbeUnknown) + { + Sleep(0); + } } - return result == IcuProbeLoaded; + return cached == IcuProbeLoaded; } size_t WindowsWideToUtf8Len(const wchar_t* src, size_t srcLen)