diff --git a/CMakeLists.txt b/CMakeLists.txt index 39062e3fbe6..1ebf375d4c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,6 +63,7 @@ else() endif() include(cmake/config.cmake) +include(cmake/icu.cmake) include(cmake/gamespy.cmake) include(cmake/lzhl.cmake) include(cmake/stb.cmake) diff --git a/Core/GameEngine/CMakeLists.txt b/Core/GameEngine/CMakeLists.txt index 0b48d20c4eb..213ff1889d8 100644 --- a/Core/GameEngine/CMakeLists.txt +++ b/Core/GameEngine/CMakeLists.txt @@ -1198,6 +1198,7 @@ target_link_libraries(corei_gameengine_public INTERFACE core_compression core_browserdispatch #core_wwvegas + core_icu d3d8lib gamespy::gamespy stlport diff --git a/Core/GameEngine/Source/Common/System/AsciiString.cpp b/Core/GameEngine/Source/Common/System/AsciiString.cpp index 61c1dc84448..e95c74d99b5 100644 --- a/Core/GameEngine/Source/Common/System/AsciiString.cpp +++ b/Core/GameEngine/Source/Common/System/AsciiString.cpp @@ -309,18 +309,27 @@ void AsciiString::translate(const UnicodeString& stringSrc) { validate(); // TheSuperHackers @fix bobtista 02/04/2026 Implement UTF-8 conversion replacing 7-bit ASCII only implementation + // TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Convert wide text to UTF-8 with ICU4C. const WideChar* src = stringSrc.str(); - const size_t srcLen = wcslen(src); - const size_t dstLen = Wide_To_Utf8_Len(src, srcLen); - if (dstLen == 0) + const size_t srcLen = stringSrc.getLength(); + const size_t len = Wide_To_Utf8_Len(src, srcLen); + if (len == 0) { clear(); } + else if (len >= static_cast(MAX_LEN)) + { + DEBUG_ASSERTCRASH(false, + ("AsciiString::translate exceeds max string length %d with required UTF-8 length %u", + MAX_LEN, static_cast(len))); + clear(); + } else { - ensureUniqueBufferOfSize((Int)dstLen + 1, false, nullptr, nullptr); - Wide_To_Utf8(peek(), dstLen + 1, src, srcLen); + ensureUniqueBufferOfSize(static_cast(len) + 1, false, nullptr, nullptr); + Wide_To_Utf8(peek(), len + 1, src, srcLen); } + validate(); } diff --git a/Core/GameEngine/Source/Common/System/UnicodeString.cpp b/Core/GameEngine/Source/Common/System/UnicodeString.cpp index e2a58acc2a5..e28a29c2fbd 100644 --- a/Core/GameEngine/Source/Common/System/UnicodeString.cpp +++ b/Core/GameEngine/Source/Common/System/UnicodeString.cpp @@ -225,31 +225,32 @@ void UnicodeString::translate(const AsciiString& stringSrc) // TheSuperHackers @fix bobtista 02/04/2026 Convert UTF-8 to wide, replacing the 7-bit ASCII only // implementation. Data that is not valid UTF-8 (e.g. legacy CP1252) falls back to a 1:1 byte cast // to preserve the original characters instead of producing replacement characters. + // TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Convert UTF-8 to wide text with ICU4C. const char* src = stringSrc.str(); - const size_t srcLen = strlen(src); - const size_t dstLen = Utf8_To_Wide_Len(src, srcLen); - if (dstLen != UTF8_INVALID) + const size_t srcLen = stringSrc.getLength(); + const size_t len = Utf8_To_Wide_Len(src, srcLen); + if (srcLen == 0) { - if (dstLen == 0) - { - clear(); - } - else - { - ensureUniqueBufferOfSize((Int)dstLen + 1, false, nullptr, nullptr); - Utf8_To_Wide(peek(), dstLen + 1, src, srcLen); - } + clear(); } - else + else if (len == UTF8_INVALID) { - ensureUniqueBufferOfSize((Int)srcLen + 1, false, nullptr, nullptr); + // Preserve legacy non-UTF-8 data with the original one-byte-to-one-wide-unit behavior. + ensureUniqueBufferOfSize(static_cast(srcLen) + 1, false, nullptr, nullptr); WideChar* buf = peek(); for (size_t i = 0; i < srcLen; ++i) { - buf[i] = (WideChar)(unsigned char)src[i]; + buf[i] = static_cast(static_cast(src[i])); } + buf[srcLen] = 0; } + else + { + ensureUniqueBufferOfSize(static_cast(len) + 1, false, nullptr, nullptr); + Utf8_To_Wide(peek(), len + 1, src, srcLen); + } + validate(); } diff --git a/Core/GameEngine/Source/GameNetwork/GameInfo.cpp b/Core/GameEngine/Source/GameNetwork/GameInfo.cpp index 5b05e9eb369..53ebe571129 100644 --- a/Core/GameEngine/Source/GameNetwork/GameInfo.cpp +++ b/Core/GameEngine/Source/GameNetwork/GameInfo.cpp @@ -953,8 +953,19 @@ AsciiString GameInfoToAsciiString( const GameInfo *game ) int lenRem = m_lanMaxOptionsLength - lenCur; //length remaining before overflowing int lenMax = lenRem / (MAX_SLOTS-i); //share lenRem with all remaining slots AsciiString name = WideCharStringToMultiByte(slot->getName().str()).c_str(); - while( name.getLength() > lenMax ) - name.removeLastChar(); //what a horrible way to truncate. I hate AsciiString. + if (name.getLength() > lenMax) + { + // TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Truncate UTF-8 only at a code-point boundary. + Int truncatedLength = lenMax > 0 ? lenMax : 0; + while (truncatedLength > 0 + && (static_cast(name.getCharAt(truncatedLength)) & 0xC0) == 0x80) + { + --truncatedLength; + } + + AsciiString truncatedName(name.str(), truncatedLength); + name = truncatedName; + } str.format( "H%s%s", name.str(), tmp.str() ); } diff --git a/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp b/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp index 84ce0c9b19a..9b7c16b7a65 100644 --- a/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp +++ b/Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp @@ -33,60 +33,54 @@ //------------------------------------------------------------------------- // TheSuperHackers @refactor bobtista 02/04/2026 Use WWLib UTF-8 functions instead of raw Win32 API calls -std::wstring MultiByteToWideCharSingleLine( const char *orig ) +// TheSuperHackers @refactor CryoTheRenegade 04/08/2026 Use the shared ICU4C UTF conversion functions. +std::wstring MultiByteToWideCharSingleLine( const char* orig ) { const size_t srcLen = strlen(orig); - const size_t dstLen = Utf8_To_Wide_Len(orig, srcLen); - if (dstLen == 0) + const size_t len = Utf8_To_Wide_Len(orig, srcLen); + if (len == 0) + { return std::wstring(); + } + std::wstring ret; - if (dstLen == UTF8_INVALID) + if (len == UTF8_INVALID) { - // Not UTF-8. Fall back to a 1:1 byte cast so legacy data keeps its characters, matching - // UnicodeString::translate. ret.resize(srcLen); for (size_t i = 0; i < srcLen; ++i) { - ret[i] = (WideChar)(unsigned char)orig[i]; + ret[i] = static_cast(static_cast(orig[i])); } } else { - ret.resize(dstLen); - Utf8_To_Wide(&ret[0], dstLen, orig, srcLen); + ret.resize(len); + Utf8_To_Wide(&ret[0], len, orig, srcLen); } - WideChar *c = nullptr; - do - { - c = wcschr(&ret[0], L'\n'); - if (c) - { - *c = L' '; - } - } - while ( c != nullptr ); - do + + for (size_t i = 0; i < ret.size(); ++i) { - c = wcschr(&ret[0], L'\r'); - if (c) + if (ret[i] == L'\n' || ret[i] == L'\x0D') { - *c = L' '; + ret[i] = L' '; } } - while ( c != nullptr ); return ret; } -std::string WideCharStringToMultiByte( const WideChar *orig ) +std::string WideCharStringToMultiByte( const WideChar* orig ) { const size_t srcLen = wcslen(orig); - const size_t dstLen = Wide_To_Utf8_Len(orig, srcLen); - if (dstLen == 0) + const size_t len = Wide_To_Utf8_Len(orig, srcLen); + if (len == 0) + { return std::string(); + } + std::string ret; - ret.resize(dstLen); - Wide_To_Utf8(&ret[0], dstLen, orig, srcLen); + ret.resize(len); + Wide_To_Utf8(&ret[0], len, orig, srcLen); return ret; } diff --git a/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt b/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt index 77721250c6c..1ba05b0e393 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt +++ b/Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt @@ -48,6 +48,7 @@ set(WWLIB_SRC #global.h hash.cpp hash.h + IcuSupport.h hashcalc.h HASHLIST.h #hashtab.h @@ -188,3 +189,7 @@ target_link_libraries(core_wwlib PRIVATE core_wwcommon corei_always ) + +target_link_libraries(core_wwlib PUBLIC + core_icu +) diff --git a/Core/Libraries/Source/WWVegas/WWLib/IcuSupport.h b/Core/Libraries/Source/WWVegas/WWLib/IcuSupport.h new file mode 100644 index 00000000000..6ce7ff720f2 --- /dev/null +++ b/Core/Libraries/Source/WWVegas/WWLib/IcuSupport.h @@ -0,0 +1,58 @@ +/* +** Command & Conquer Generals Zero Hour(tm) +** Copyright 2026 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +#pragma once + +// Engine entry point for ICU4C. +// +// RTS_HAS_ICU - ICU C API is linked; include this header and call ICU functions. +// RTS_HAS_ICU_CXX - ICU C++ API (icu::UnicodeString, icu::Locale, ...). +// RTS_HAS_ICU_I18N - Collation, break iteration, converters, and related i18n APIs. +// RTS_HAS_ICU_WINSDK - Windows SDK merged C API via (no C++ API). +// RTS_ICU_DYNAMIC - No import library; utf8.cpp LoadLibrary's OS icu.dll (VC6). + +#if defined(RTS_HAS_ICU_WINSDK) + +#include + +#elif defined(RTS_HAS_ICU) + +#include +#include +#include +#include +#include + +#if defined(RTS_HAS_ICU_I18N) +#include +#include +#include +#include +#include +#endif + +#if defined(RTS_HAS_ICU_CXX) +#include +#include +#include +#if defined(RTS_HAS_ICU_I18N) +#include +#endif +#endif + +#endif diff --git a/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp b/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp index 9faa1d64ef1..6f61b3dca3e 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp +++ b/Core/Libraries/Source/WWVegas/WWLib/utf8.cpp @@ -19,265 +19,659 @@ #include "always.h" #include "utf8.h" -// wchar_t is a 16-bit UTF-16 code unit on Windows and a 32-bit UTF-32 codepoint on most other -// platforms. WCHAR_MAX lets us distinguish the two at compile time so the surrogate-pair paths -// are excluded entirely (not just constant-folded) where wchar_t is wide enough to hold a codepoint. -#if defined(WCHAR_MAX) && (WCHAR_MAX <= 0xFFFF) -#define UTF8_WCHAR_IS_UTF16 1 +#include +#include + +#if defined(RTS_HAS_ICU_WINSDK) +#include +#include +#elif defined(RTS_HAS_ICU) +#include +#ifdef _WIN32 +#include +#endif +#include +#elif defined(_WIN32) +#include +#endif + +namespace +{ + +bool FitsInt(size_t length) +{ + return length <= static_cast(INT_MAX); +} + +#ifdef _WIN32 + +enum +{ + IcuProbeUnknown = 0, + IcuProbeMissing = 1, + IcuProbeLoaded = 2 +}; + +// TheSuperHackers @fix CryoTheRenegade 23/08/2026 Probe icu.dll once through the normal DLL search order. +bool LoadSystemIcu() +{ + static volatile LONG cached = IcuProbeUnknown; + static volatile LONG initGate = 0; + + if (cached != IcuProbeUnknown) + { + return cached == IcuProbeLoaded; + } + + // InterlockedIncrement is LONG* on every supported SDK. InterlockedCompareExchange is not: + // VC6 winbase.h takes PVOID*, while later SDKs take LONG*. +#if defined(_MSC_VER) && _MSC_VER < 1300 + const LONG gate = InterlockedIncrement(const_cast(&initGate)); #else -#define UTF8_WCHAR_IS_UTF16 0 + const LONG gate = InterlockedIncrement(&initGate); #endif + if (gate == 1) + { + cached = LoadLibraryA("icu.dll") != nullptr ? IcuProbeLoaded : IcuProbeMissing; + } + else + { + while (cached == IcuProbeUnknown) + { + Sleep(0); + } + } + + return cached == IcuProbeLoaded; +} + +size_t WindowsWideToUtf8Len(const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + WWASSERT(false); + return 0; + } -static const unsigned int UTF8_CODEPOINT_MAX = 0x10FFFF; -static const unsigned int UTF8_SURROGATE_MIN = 0xD800; -static const unsigned int UTF8_SURROGATE_MAX = 0xDFFF; -static const unsigned int UTF8_REPLACEMENT_CHAR = 0xFFFD; - -// Number of UTF-8 bytes required to encode a codepoint. -static size_t Utf8_Encoded_Length(unsigned int cp) -{ - if (cp < 0x80) - { - return 1; - } - if (cp < 0x800) - { - return 2; - } - if (cp < 0x10000) - { - return 3; - } - return 4; + const int outputLength = WideCharToMultiByte(CP_UTF8, 0, src, static_cast(srcLen), nullptr, 0, nullptr, nullptr); + if (outputLength == 0 && srcLen != 0) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); } -// Encode a codepoint to dest, which is assumed to have room. Returns the number of bytes written. -static size_t Utf8_Encode(char* dest, unsigned int cp) -{ - if (cp < 0x80) - { - dest[0] = (char)cp; - return 1; - } - if (cp < 0x800) - { - dest[0] = (char)(0xC0 | (cp >> 6)); - dest[1] = (char)(0x80 | (cp & 0x3F)); - return 2; - } - if (cp < 0x10000) - { - dest[0] = (char)(0xE0 | (cp >> 12)); - dest[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); - dest[2] = (char)(0x80 | (cp & 0x3F)); - return 3; - } - dest[0] = (char)(0xF0 | (cp >> 18)); - dest[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); - dest[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); - dest[3] = (char)(0x80 | (cp & 0x3F)); - return 4; +size_t WindowsWideToUtf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + WWASSERT(false); + return 0; + } + + const int outputLength = WideCharToMultiByte(CP_UTF8, 0, src, static_cast(srcLen), + dest, static_cast(destLen), nullptr, nullptr); + if (outputLength == 0 && srcLen != 0) + { + WWASSERT(false); + return 0; + } + + if (static_cast(outputLength) < destLen) + { + dest[outputLength] = '\0'; + } + + return static_cast(outputLength); } -// Decode one UTF-8 sequence at src, with srcLen bytes remaining. On success returns the number of -// bytes consumed (1-4) and sets cp. Returns 0 on any malformed, overlong, out-of-range or surrogate -// encoding. -static size_t Utf8_Decode(const char* src, size_t srcLen, unsigned int& cp) -{ - const unsigned char lead = (unsigned char)src[0]; - if (lead < 0x80) - { - cp = lead; - return 1; - } - - size_t count; - unsigned int lowerBound; - if ((lead & 0xE0) == 0xC0) - { - count = 2; - cp = lead & 0x1F; - lowerBound = 0x80; - } - else if ((lead & 0xF0) == 0xE0) - { - count = 3; - cp = lead & 0x0F; - lowerBound = 0x800; - } - else if ((lead & 0xF8) == 0xF0) - { - count = 4; - cp = lead & 0x07; - lowerBound = 0x10000; - } - else - { - return 0; // a continuation byte or a 5/6-byte form cannot start a sequence - } - - if (srcLen < count) - { - return 0; // truncated sequence - } - for (size_t i = 1; i < count; ++i) - { - const unsigned char trail = (unsigned char)src[i]; - if ((trail & 0xC0) != 0x80) - { - return 0; // not a continuation byte - } - cp = (cp << 6) | (trail & 0x3F); - } - - if (cp < lowerBound || cp > UTF8_CODEPOINT_MAX || (cp >= UTF8_SURROGATE_MIN && cp <= UTF8_SURROGATE_MAX)) - { - return 0; // overlong, out of range, or a surrogate codepoint - } - return count; +size_t WindowsUtf8ToWideLen(const char* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + return UTF8_INVALID; + } + + const int outputLength = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, src, + static_cast(srcLen), nullptr, 0); + if (outputLength == 0 && srcLen != 0) + { + return UTF8_INVALID; + } + + return static_cast(outputLength); +} + +size_t WindowsUtf8ToWide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +{ + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + const int outputLength = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, src, + static_cast(srcLen), dest, static_cast(destLen)); + if (outputLength == 0 && srcLen != 0) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + if (static_cast(outputLength) < destLen) + { + dest[outputLength] = L'\0'; + } + + return static_cast(outputLength); } -// Read one codepoint at src, with srcLen wide characters remaining. Returns the number of wide -// characters consumed (1-2) and sets cp. Combines UTF-16 surrogate pairs where wchar_t is 16-bit; -// treats each element as a whole codepoint where wchar_t is 32-bit. Wide data that has no UTF-8 -// representation is reported as U+FFFD, so the encoder never emits a sequence that the decoder -// would reject. -static size_t Wide_Read(const wchar_t* src, size_t srcLen, unsigned int& cp) -{ - size_t consumed = 1; -#if UTF8_WCHAR_IS_UTF16 - cp = (unsigned int)src[0] & 0xFFFF; - if (cp >= UTF8_SURROGATE_MIN && cp <= 0xDBFF && srcLen > 1) - { - const unsigned int low = (unsigned int)src[1] & 0xFFFF; - if (low >= 0xDC00 && low <= UTF8_SURROGATE_MAX) - { - cp = 0x10000 + ((cp - UTF8_SURROGATE_MIN) << 10) + (low - 0xDC00); - consumed = 2; - } - } -#else - (void)srcLen; - cp = (unsigned int)src[0]; #endif - if (cp > UTF8_CODEPOINT_MAX || (cp >= UTF8_SURROGATE_MIN && cp <= UTF8_SURROGATE_MAX)) - { - cp = UTF8_REPLACEMENT_CHAR; - } - return consumed; + +#if defined(RTS_HAS_ICU) + +bool IcuPreflightSucceeded(UErrorCode error) +{ + return U_SUCCESS(error) || error == U_BUFFER_OVERFLOW_ERROR; } -// Number of wide characters required to store a codepoint. -static size_t Wide_Encoded_Length(unsigned int cp) +bool IcuConversionSucceeded(UErrorCode error) { -#if UTF8_WCHAR_IS_UTF16 - return (cp >= 0x10000) ? 2 : 1; + return U_SUCCESS(error); +} + +#if defined(WCHAR_MAX) && (WCHAR_MAX <= 0xFFFF) + +size_t IcuWideToUtf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + WWASSERT(false); + return 0; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF8WithSub(dest, static_cast(destLen), &outputLength, + reinterpret_cast(src), static_cast(srcLen), 0xFFFD, nullptr, &error); + if (!IcuConversionSucceeded(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); +} + +size_t IcuWideToUtf8Len(const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + WWASSERT(false); + return 0; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF8WithSub(nullptr, 0, &outputLength, reinterpret_cast(src), + static_cast(srcLen), 0xFFFD, nullptr, &error); + if (!IcuPreflightSucceeded(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); +} + +size_t IcuUtf8ToWide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +{ + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strFromUTF8(reinterpret_cast(dest), static_cast(destLen), &outputLength, + src, static_cast(srcLen), &error); + if (!IcuConversionSucceeded(error)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + return static_cast(outputLength); +} + +size_t IcuUtf8ToWideLen(const char* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + return UTF8_INVALID; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strFromUTF8(nullptr, 0, &outputLength, src, static_cast(srcLen), &error); + if (!IcuPreflightSucceeded(error)) + { + return UTF8_INVALID; + } + + return static_cast(outputLength); +} + #else - (void)cp; - return 1; + +bool WideToUtf16(std::vector& utf16, const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + return false; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strFromUTF32WithSub(nullptr, 0, &outputLength, reinterpret_cast(src), + static_cast(srcLen), 0xFFFD, nullptr, &error); + if (!IcuPreflightSucceeded(error)) + { + return false; + } + + utf16.resize(static_cast(outputLength) + 1); + error = U_ZERO_ERROR; + u_strFromUTF32WithSub(&utf16[0], static_cast(utf16.size()), &outputLength, + reinterpret_cast(src), static_cast(srcLen), 0xFFFD, nullptr, &error); + return U_SUCCESS(error); +} + +bool Utf8ToUtf16(std::vector& utf16, const char* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + return false; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strFromUTF8(nullptr, 0, &outputLength, src, static_cast(srcLen), &error); + if (!IcuPreflightSucceeded(error)) + { + return false; + } + + utf16.resize(static_cast(outputLength) + 1); + error = U_ZERO_ERROR; + u_strFromUTF8(&utf16[0], static_cast(utf16.size()), &outputLength, + src, static_cast(srcLen), &error); + return U_SUCCESS(error); +} + +size_t IcuWideToUtf8Len(const wchar_t* src, size_t srcLen) +{ + std::vector utf16; + if (!WideToUtf16(utf16, src, srcLen)) + { + WWASSERT(false); + return 0; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF8WithSub(nullptr, 0, &outputLength, utf16.empty() ? nullptr : &utf16[0], + static_cast(utf16.empty() ? 0 : utf16.size() - 1), 0xFFFD, nullptr, &error); + if (!IcuPreflightSucceeded(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); +} + +size_t IcuUtf8ToWideLen(const char* src, size_t srcLen) +{ + std::vector utf16; + if (!Utf8ToUtf16(utf16, src, srcLen)) + { + return UTF8_INVALID; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF32(nullptr, 0, &outputLength, utf16.empty() ? nullptr : &utf16[0], + static_cast(utf16.empty() ? 0 : utf16.size() - 1), &error); + if (!IcuPreflightSucceeded(error)) + { + return UTF8_INVALID; + } + + return static_cast(outputLength); +} + +size_t IcuWideToUtf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(destLen)) + { + WWASSERT(false); + return 0; + } + + std::vector utf16; + if (!WideToUtf16(utf16, src, srcLen)) + { + WWASSERT(false); + return 0; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF8WithSub(dest, static_cast(destLen), &outputLength, + utf16.empty() ? nullptr : &utf16[0], static_cast(utf16.empty() ? 0 : utf16.size() - 1), + 0xFFFD, nullptr, &error); + if (U_FAILURE(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); +} + +size_t IcuUtf8ToWide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +{ + if (!FitsInt(destLen)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + std::vector utf16; + if (!Utf8ToUtf16(utf16, src, srcLen)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + UErrorCode error = U_ZERO_ERROR; + int32_t outputLength = 0; + u_strToUTF32(reinterpret_cast(dest), static_cast(destLen), &outputLength, + utf16.empty() ? nullptr : &utf16[0], static_cast(utf16.empty() ? 0 : utf16.size() - 1), &error); + if (U_FAILURE(error)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + return static_cast(outputLength); +} + #endif + +#elif defined(RTS_ICU_DYNAMIC) + +typedef unsigned short IcuChar; +typedef int IcuChar32; +typedef int IcuErrorCode; +typedef IcuChar* (__cdecl* IcuStrFromUtf8)( + IcuChar*, int, int*, const char*, int, IcuErrorCode*); +typedef char* (__cdecl* IcuStrToUtf8WithSub)( + char*, int, int*, const IcuChar*, int, IcuChar32, int*, IcuErrorCode*); + +enum +{ + ICU_ZERO_ERROR = 0, + ICU_BUFFER_OVERFLOW_ERROR = 15, + ICU_REPLACEMENT_CHARACTER = 0xFFFD +}; + +class WindowsIcuFunctions +{ +public: + WindowsIcuFunctions() : m_fromUtf8(nullptr), m_toUtf8WithSub(nullptr), m_module(nullptr) + { + if (!LoadSystemIcu()) + { + return; + } + + m_module = GetModuleHandleA("icu.dll"); + if (m_module != nullptr) + { + m_fromUtf8 = reinterpret_cast(GetProcAddress(m_module, "u_strFromUTF8")); + m_toUtf8WithSub = reinterpret_cast(GetProcAddress(m_module, "u_strToUTF8WithSub")); + } + + if (m_fromUtf8 == nullptr || m_toUtf8WithSub == nullptr) + { + m_module = nullptr; + m_fromUtf8 = nullptr; + m_toUtf8WithSub = nullptr; + } + } + + bool isAvailable() const + { + return m_fromUtf8 != nullptr && m_toUtf8WithSub != nullptr; + } + + IcuStrFromUtf8 m_fromUtf8; + IcuStrToUtf8WithSub m_toUtf8WithSub; + +private: + HMODULE m_module; +}; + +WindowsIcuFunctions g_icu; + +bool IcuPreflightSucceeded(IcuErrorCode error) +{ + return error <= ICU_ZERO_ERROR || error == ICU_BUFFER_OVERFLOW_ERROR; } -// Write one codepoint to a wide buffer, which is assumed to have room. Returns wide characters written. -static size_t Wide_Write(wchar_t* dest, unsigned int cp) -{ -#if UTF8_WCHAR_IS_UTF16 - if (cp >= 0x10000) - { - cp -= 0x10000; - dest[0] = (wchar_t)(0xD800 + (cp >> 10)); - dest[1] = (wchar_t)(0xDC00 + (cp & 0x3FF)); - return 2; - } +bool IcuConversionSucceeded(IcuErrorCode error) +{ + return error <= ICU_ZERO_ERROR; +} + +size_t IcuWideToUtf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + WWASSERT(false); + return 0; + } + + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_toUtf8WithSub(dest, static_cast(destLen), &outputLength, + reinterpret_cast(src), static_cast(srcLen), ICU_REPLACEMENT_CHARACTER, nullptr, &error); + if (!IcuConversionSucceeded(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); +} + +size_t IcuWideToUtf8Len(const wchar_t* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + WWASSERT(false); + return 0; + } + + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_toUtf8WithSub(nullptr, 0, &outputLength, reinterpret_cast(src), + static_cast(srcLen), ICU_REPLACEMENT_CHARACTER, nullptr, &error); + if (!IcuPreflightSucceeded(error)) + { + WWASSERT(false); + return 0; + } + + return static_cast(outputLength); +} + +size_t IcuUtf8ToWide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) +{ + if (!FitsInt(destLen) || !FitsInt(srcLen)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_fromUtf8(reinterpret_cast(dest), static_cast(destLen), &outputLength, + src, static_cast(srcLen), &error); + if (!IcuConversionSucceeded(error)) + { + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; + } + + return static_cast(outputLength); +} + +size_t IcuUtf8ToWideLen(const char* src, size_t srcLen) +{ + if (!FitsInt(srcLen)) + { + return UTF8_INVALID; + } + + IcuErrorCode error = ICU_ZERO_ERROR; + int outputLength = 0; + g_icu.m_fromUtf8(nullptr, 0, &outputLength, src, static_cast(srcLen), &error); + if (!IcuPreflightSucceeded(error)) + { + return UTF8_INVALID; + } + + return static_cast(outputLength); +} + +#endif + +bool IcuIsAvailable() +{ +#if defined(RTS_HAS_ICU_WINSDK) + return LoadSystemIcu(); +#elif defined(RTS_HAS_ICU) + return true; +#elif defined(RTS_ICU_DYNAMIC) + return g_icu.isAvailable(); +#else + return false; #endif - dest[0] = (wchar_t)cp; - return 1; } +} // namespace + size_t Wide_To_Utf8_Len(const wchar_t* src, size_t srcLen) { - size_t needed = 0; - size_t i = 0; - while (i < srcLen) - { - unsigned int cp; - i += Wide_Read(src + i, srcLen - i, cp); - needed += Utf8_Encoded_Length(cp); - } - return needed; + if (IcuIsAvailable()) + { + return IcuWideToUtf8Len(src, srcLen); + } + +#ifdef _WIN32 + return WindowsWideToUtf8Len(src, srcLen); +#else + WWASSERT(false); + return 0; +#endif } size_t Utf8_To_Wide_Len(const char* src, size_t srcLen) { - size_t needed = 0; - size_t i = 0; - while (i < srcLen) - { - unsigned int cp; - const size_t consumed = Utf8_Decode(src + i, srcLen - i, cp); - if (consumed == 0) - { - return UTF8_INVALID; - } - i += consumed; - needed += Wide_Encoded_Length(cp); - } - return needed; + if (IcuIsAvailable()) + { + return IcuUtf8ToWideLen(src, srcLen); + } + +#ifdef _WIN32 + return WindowsUtf8ToWideLen(src, srcLen); +#else + return UTF8_INVALID; +#endif } size_t Wide_To_Utf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen) { - size_t needed = 0; - size_t out = 0; - size_t i = 0; - while (i < srcLen) - { - unsigned int cp; - i += Wide_Read(src + i, srcLen - i, cp); - const size_t need = Utf8_Encoded_Length(cp); - // Stop writing at the first codepoint that does not fit, but keep counting for the caller. - if (needed == out && out + need <= destLen) - { - out += Utf8_Encode(dest + out, cp); - } - needed += need; - } - if (out < destLen) - { - dest[out] = '\0'; - } - return needed; + if (IcuIsAvailable()) + { + return IcuWideToUtf8(dest, destLen, src, srcLen); + } + +#ifdef _WIN32 + return WindowsWideToUtf8(dest, destLen, src, srcLen); +#else + WWASSERT(false); + return 0; +#endif } size_t Utf8_To_Wide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen) { - size_t needed = 0; - size_t out = 0; - size_t i = 0; - while (i < srcLen) - { - unsigned int cp; - const size_t consumed = Utf8_Decode(src + i, srcLen - i, cp); - if (consumed == 0) - { - if (destLen > 0) - { - dest[0] = L'\0'; - } - return UTF8_INVALID; - } - i += consumed; - const size_t need = Wide_Encoded_Length(cp); - // Stop writing at the first codepoint that does not fit, but keep counting for the caller. - if (needed == out && out + need <= destLen) - { - out += Wide_Write(dest + out, cp); - } - needed += need; - } - if (out < destLen) - { - dest[out] = L'\0'; - } - return needed; + if (IcuIsAvailable()) + { + return IcuUtf8ToWide(dest, destLen, src, srcLen); + } + +#ifdef _WIN32 + return WindowsUtf8ToWide(dest, destLen, src, srcLen); +#else + if (destLen > 0) + { + dest[0] = L'\0'; + } + + return UTF8_INVALID; +#endif } diff --git a/Core/Libraries/Source/WWVegas/WWLib/utf8.h b/Core/Libraries/Source/WWVegas/WWLib/utf8.h index 7424943b014..e9a983ebdf8 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/utf8.h +++ b/Core/Libraries/Source/WWVegas/WWLib/utf8.h @@ -21,37 +21,20 @@ #include #include -// UTF-8 <-> wide-character transcoding, hand-rolled per RFC 3629, using no platform text APIs. -// The wide side is wchar_t, whose width is platform-dependent: on Windows it is a 16-bit UTF-16 -// code unit (astral codepoints use surrogate pairs); on most other platforms it is a 32-bit -// UTF-32 codepoint. Both are handled transparently based on the width of wchar_t. +// UTF-8 <-> wide-character conversion backed by ICU4C. +// Modern toolchains link the ICU C API (Windows SDK or vcpkg). VC6 LoadLibrary's OS icu.dll +// and falls back to Win32 CP_UTF8 when it is missing. Include WWLib/IcuSupport.h to use the +// rest of the linked ICU suite from engine code. -// Returned by the decoding functions when the source is not well-formed UTF-8. A return of 0 means -// an empty result, which is a success and must not be confused with a decoding failure. +// Returned when UTF-8 input is malformed. Zero is reserved for a successful empty conversion. const size_t UTF8_INVALID = (size_t)-1; -// Returns the number of UTF-8 bytes needed for the UTF-8 representation of srcLen wide characters -// from src, not counting a null terminator. Returns 0 if srcLen is 0. Wide values that have no -// UTF-8 representation are counted as U+FFFD. +// Return the required destination length without counting a null terminator. size_t Wide_To_Utf8_Len(const wchar_t* src, size_t srcLen); - -// Returns the number of wide characters needed for the wide representation of srcLen bytes from the -// UTF-8 string src, not counting a null terminator. Returns 0 if srcLen is 0, or UTF8_INVALID if -// src is not well-formed UTF-8. size_t Utf8_To_Wide_Len(const char* src, size_t srcLen); -// Converts srcLen wide characters from src to UTF-8. destLen is the destination buffer capacity in -// bytes. Writes a null terminator if room remains, otherwise not. Wide values that have no UTF-8 -// representation are written as U+FFFD, so the output always decodes back through Utf8_To_Wide. -// Returns the number of bytes the whole conversion needs, not counting a null terminator. A return -// greater than destLen means the output was truncated on a codepoint boundary; retry with that many -// bytes plus one for the terminator. Pass destLen 0 to measure without writing. +// Convert exactly srcLen source units. The destination is null-terminated when it has spare +// capacity. Wide input that cannot be represented as Unicode is replaced with U+FFFD. Malformed +// UTF-8 returns UTF8_INVALID and clears dest when destLen is nonzero. size_t Wide_To_Utf8(char* dest, size_t destLen, const wchar_t* src, size_t srcLen); - -// Converts srcLen bytes from the UTF-8 string src to wide characters. destLen is the destination -// buffer capacity in wide characters. Writes a null terminator if room remains, otherwise not. -// Returns the number of wide characters the whole conversion needs, not counting a null terminator. -// A return greater than destLen means the output was truncated on a codepoint boundary; retry with -// that many wide characters plus one for the terminator. Pass destLen 0 to measure without writing. -// Returns UTF8_INVALID if src is not well-formed UTF-8, setting dest[0] to L'\0' if destLen > 0. size_t Utf8_To_Wide(wchar_t* dest, size_t destLen, const char* src, size_t srcLen); diff --git a/cmake/icu.cmake b/cmake/icu.cmake new file mode 100644 index 00000000000..15555a0af8a --- /dev/null +++ b/cmake/icu.cmake @@ -0,0 +1,74 @@ +# ICU4C for the engine. +# +# Preference order: +# 1. find_package(ICU) from vcpkg or the system (C + C++ APIs: uc, i18n, data) +# 2. Windows SDK icu.lib for modern MSVC (C API: common + i18n via ) +# 3. Runtime LoadLibrary of OS icu.dll (VC6 and other Windows toolchains without an import lib) +# +# VC6 cannot consume modern ICU headers or import libraries. It keeps the dynamic loader. + +add_library(core_icu INTERFACE) + +set(RTS_ICU_LINKED FALSE) +set(RTS_ICU_CXX FALSE) +set(RTS_ICU_I18N FALSE) +set(RTS_ICU_WINSDK FALSE) +set(RTS_ICU_DYNAMIC FALSE) + +if(NOT IS_VS6_BUILD) + find_package(ICU QUIET COMPONENTS uc i18n data) + if(NOT ICU_FOUND) + find_package(ICU QUIET COMPONENTS uc i18n) + endif() + if(NOT ICU_FOUND) + find_package(ICU QUIET COMPONENTS uc data) + endif() + if(NOT ICU_FOUND) + find_package(ICU QUIET COMPONENTS uc) + endif() +endif() + +if(ICU_FOUND AND NOT IS_VS6_BUILD) + set(RTS_ICU_LINKED TRUE) + set(RTS_ICU_CXX TRUE) + target_link_libraries(core_icu INTERFACE ICU::uc) + if(TARGET ICU::i18n) + set(RTS_ICU_I18N TRUE) + target_link_libraries(core_icu INTERFACE ICU::i18n) + endif() + if(TARGET ICU::data) + target_link_libraries(core_icu INTERFACE ICU::data) + endif() + target_compile_definitions(core_icu INTERFACE RTS_HAS_ICU RTS_HAS_ICU_CXX) + if(RTS_ICU_I18N) + target_compile_definitions(core_icu INTERFACE RTS_HAS_ICU_I18N) + endif() + message(STATUS "ICU4C: linked via find_package (C++ API enabled)") +elseif(WIN32 AND NOT IS_VS6_BUILD AND NOT MINGW) + find_library(RTS_ICU_WINSDK_LIB NAMES icu) + if(RTS_ICU_WINSDK_LIB) + set(RTS_ICU_LINKED TRUE) + set(RTS_ICU_I18N TRUE) + set(RTS_ICU_WINSDK TRUE) + target_link_libraries(core_icu INTERFACE ${RTS_ICU_WINSDK_LIB} delayimp) + target_link_options(core_icu INTERFACE "/DELAYLOAD:icu.dll") + target_compile_definitions(core_icu INTERFACE RTS_HAS_ICU RTS_HAS_ICU_WINSDK RTS_HAS_ICU_I18N) + message(STATUS "ICU4C: linked via Windows SDK (${RTS_ICU_WINSDK_LIB})") + endif() +endif() + +if(NOT RTS_ICU_LINKED) + if(WIN32) + set(RTS_ICU_DYNAMIC TRUE) + target_compile_definitions(core_icu INTERFACE RTS_ICU_DYNAMIC) + message(STATUS "ICU4C: runtime load of system icu.dll") + else() + message(FATAL_ERROR "ICU4C is required on non-Windows platforms. Install libicu or enable vcpkg.") + endif() +endif() + +add_feature_info(IcuLinked RTS_ICU_LINKED "Link ICU4C into the engine") +add_feature_info(IcuCxx RTS_ICU_CXX "ICU C++ API available") +add_feature_info(IcuI18n RTS_ICU_I18N "ICU i18n API available") +add_feature_info(IcuWinSdk RTS_ICU_WINSDK "Windows SDK ICU C API") +add_feature_info(IcuDynamic RTS_ICU_DYNAMIC "Load OS icu.dll at runtime") diff --git a/vcpkg-lock.json b/vcpkg-lock.json index 422998f0378..5926be63217 100644 --- a/vcpkg-lock.json +++ b/vcpkg-lock.json @@ -7,12 +7,24 @@ "port-version": 1, "git-tree": "6ff75f1f596ada519241989f44077cda442480b2" }, + { + "name": "icu", + "version-string": "74.2", + "port-version": 5, + "git-tree": "91b03be43850140e64dba01dcadc8b3f73b1083b" + }, { "name": "pkgconf", "version-string": "2.3.0", "port-version": 0, "git-tree": "ae3886d8a627ec99dd18890389b6d5d331e29799" }, + { + "name": "stb", + "version-string": "2024-07-29", + "port-version": 1, + "git-tree": "c362152d1d02973e7b1ce777960f8a6876656560" + }, { "name": "vcpkg-cmake", "version-string": "2024-04-23", @@ -25,6 +37,12 @@ "port-version": 0, "git-tree": "f23148add155147f3d95ae622d3b0031beb25acf" }, + { + "name": "vcpkg-make", + "version-string": "2025-02-08", + "port-version": 0, + "git-tree": "84480c9e9de4cc4cf60f44d5de24c79d61dfe966" + }, { "name": "vcpkg-pkgconfig-get-modules", "version-string": "2024-04-03", diff --git a/vcpkg.json b/vcpkg.json index 9ce3c6667c3..cc243a0e053 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -3,6 +3,7 @@ "builtin-baseline": "b02e341c927f16d991edbd915d8ea43eac52096c", "dependencies": [ "zlib", + "icu", "ffmpeg", "stb" ]