feat(icu4c): Add ICU4C support to UTF8 string operations

CMakeLists.txt

@@ -63,6 +63,7 @@ else()
 endif()
 
 include(cmake/config.cmake)
+include(cmake/icu.cmake)
 include(cmake/gamespy.cmake)
 include(cmake/lzhl.cmake)
 include(cmake/stb.cmake)

Core/GameEngine/CMakeLists.txt

@@ -1198,6 +1198,7 @@ target_link_libraries(corei_gameengine_public INTERFACE
     core_compression
     core_browserdispatch
     #core_wwvegas
+    core_icu
     d3d8lib
     gamespy::gamespy
     stlport

Core/GameEngine/Source/Common/System/AsciiString.cpp

@@ -309,18 +309,27 @@ void AsciiString::translate(const UnicodeString& stringSrc)
 {
 	validate();
 	// TheSuperHackers @fix bobtista 02/04/2026 Implement UTF-8 conversion replacing 7-bit ASCII only implementation
+	// TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Convert wide text to UTF-8 with ICU4C.
 	const WideChar* src = stringSrc.str();
-	const size_t srcLen = wcslen(src);
-	const size_t dstLen = Wide_To_Utf8_Len(src, srcLen);
-	if (dstLen == 0)
+	const size_t srcLen = stringSrc.getLength();
+	const size_t len = Wide_To_Utf8_Len(src, srcLen);
+	if (len == 0)
 	{
 		clear();
 	}
+	else if (len >= static_cast<size_t>(MAX_LEN))
+	{
+		DEBUG_ASSERTCRASH(false,
+			("AsciiString::translate exceeds max string length %d with required UTF-8 length %u",
+				MAX_LEN, static_cast<unsigned int>(len)));
+		clear();
+	}
 	else
 	{
-		ensureUniqueBufferOfSize((Int)dstLen + 1, false, nullptr, nullptr);
-		Wide_To_Utf8(peek(), dstLen + 1, src, srcLen);
+		ensureUniqueBufferOfSize(static_cast<Int>(len) + 1, false, nullptr, nullptr);
+		Wide_To_Utf8(peek(), len + 1, src, srcLen);
 	}
+
 	validate();
 }
 

Core/GameEngine/Source/Common/System/UnicodeString.cpp

@@ -225,31 +225,32 @@ void UnicodeString::translate(const AsciiString& stringSrc)
 	// TheSuperHackers @fix bobtista 02/04/2026 Convert UTF-8 to wide, replacing the 7-bit ASCII only
 	// implementation. Data that is not valid UTF-8 (e.g. legacy CP1252) falls back to a 1:1 byte cast
 	// to preserve the original characters instead of producing replacement characters.
+	// TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Convert UTF-8 to wide text with ICU4C.
 	const char* src = stringSrc.str();
-	const size_t srcLen = strlen(src);
-	const size_t dstLen = Utf8_To_Wide_Len(src, srcLen);
-	if (dstLen != UTF8_INVALID)
+	const size_t srcLen = stringSrc.getLength();
+	const size_t len = Utf8_To_Wide_Len(src, srcLen);
+	if (srcLen == 0)
 	{
-		if (dstLen == 0)
-		{
-			clear();
-		}
-		else
-		{
-			ensureUniqueBufferOfSize((Int)dstLen + 1, false, nullptr, nullptr);
-			Utf8_To_Wide(peek(), dstLen + 1, src, srcLen);
-		}
+		clear();
 	}
-	else
+	else if (len == UTF8_INVALID)
 	{
-		ensureUniqueBufferOfSize((Int)srcLen + 1, false, nullptr, nullptr);
+		// Preserve legacy non-UTF-8 data with the original one-byte-to-one-wide-unit behavior.
+		ensureUniqueBufferOfSize(static_cast<Int>(srcLen) + 1, false, nullptr, nullptr);
 		WideChar* buf = peek();
 		for (size_t i = 0; i < srcLen; ++i)
 		{
-			buf[i] = (WideChar)(unsigned char)src[i];
+			buf[i] = static_cast<WideChar>(static_cast<unsigned char>(src[i]));
 		}
+
 		buf[srcLen] = 0;
 	}
+	else
+	{
+		ensureUniqueBufferOfSize(static_cast<Int>(len) + 1, false, nullptr, nullptr);
+		Utf8_To_Wide(peek(), len + 1, src, srcLen);
+	}
+
 	validate();
 }
 

Core/GameEngine/Source/GameNetwork/GameInfo.cpp

@@ -953,8 +953,19 @@ AsciiString GameInfoToAsciiString( const GameInfo *game )
 			int lenRem = m_lanMaxOptionsLength - lenCur;  //length remaining before overflowing
 			int lenMax = lenRem / (MAX_SLOTS-i);  //share lenRem with all remaining slots
 			AsciiString name = WideCharStringToMultiByte(slot->getName().str()).c_str();
-			while( name.getLength() > lenMax )
-				name.removeLastChar();  //what a horrible way to truncate.  I hate AsciiString.
+			if (name.getLength() > lenMax)
+			{
+				// TheSuperHackers @bugfix CryoTheRenegade 04/08/2026 Truncate UTF-8 only at a code-point boundary.
+				Int truncatedLength = lenMax > 0 ? lenMax : 0;
+				while (truncatedLength > 0
+					&& (static_cast<unsigned char>(name.getCharAt(truncatedLength)) & 0xC0) == 0x80)
+				{
+					--truncatedLength;
+				}
+
+				AsciiString truncatedName(name.str(), truncatedLength);
+				name = truncatedName;
+			}
 
 			str.format( "H%s%s", name.str(), tmp.str() );
 		}

Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp

@@ -33,60 +33,54 @@
 //-------------------------------------------------------------------------
 
 // TheSuperHackers @refactor bobtista 02/04/2026 Use WWLib UTF-8 functions instead of raw Win32 API calls
-std::wstring MultiByteToWideCharSingleLine( const char *orig )
+// TheSuperHackers @refactor CryoTheRenegade 04/08/2026 Use the shared ICU4C UTF conversion functions.
+std::wstring MultiByteToWideCharSingleLine( const char* orig )
 {
 	const size_t srcLen = strlen(orig);
-	const size_t dstLen = Utf8_To_Wide_Len(orig, srcLen);
-	if (dstLen == 0)
+	const size_t len = Utf8_To_Wide_Len(orig, srcLen);
+	if (len == 0)
+	{
 		return std::wstring();
+	}
+
 	std::wstring ret;
-	if (dstLen == UTF8_INVALID)
+	if (len == UTF8_INVALID)
 	{
-		// Not UTF-8. Fall back to a 1:1 byte cast so legacy data keeps its characters, matching
-		// UnicodeString::translate.
 		ret.resize(srcLen);
 		for (size_t i = 0; i < srcLen; ++i)
 		{
-			ret[i] = (WideChar)(unsigned char)orig[i];
+			ret[i] = static_cast<WideChar>(static_cast<unsigned char>(orig[i]));
 		}
 	}
 	else
 	{
-		ret.resize(dstLen);
-		Utf8_To_Wide(&ret[0], dstLen, orig, srcLen);
+		ret.resize(len);
+		Utf8_To_Wide(&ret[0], len, orig, srcLen);
 	}
-	WideChar *c = nullptr;
-	do
-	{
-		c = wcschr(&ret[0], L'\n');
-		if (c)
-		{
-			*c = L' ';
-		}
-	}
-	while ( c != nullptr );
-	do
+
+	for (size_t i = 0; i < ret.size(); ++i)
 	{
-		c = wcschr(&ret[0], L'\r');
-		if (c)
+		if (ret[i] == L'\n' || ret[i] == L'\x0D')
 		{
-			*c = L' ';
+			ret[i] = L' ';
 		}
 	}
-	while ( c != nullptr );
 
 	return ret;
 }
 
-std::string WideCharStringToMultiByte( const WideChar *orig )
+std::string WideCharStringToMultiByte( const WideChar* orig )
 {
 	const size_t srcLen = wcslen(orig);
-	const size_t dstLen = Wide_To_Utf8_Len(orig, srcLen);
-	if (dstLen == 0)
+	const size_t len = Wide_To_Utf8_Len(orig, srcLen);
+	if (len == 0)
+	{
 		return std::string();
+	}
+
 	std::string ret;
-	ret.resize(dstLen);
-	Wide_To_Utf8(&ret[0], dstLen, orig, srcLen);
+	ret.resize(len);
+	Wide_To_Utf8(&ret[0], len, orig, srcLen);
 	return ret;
 }
 

Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt

@@ -48,6 +48,7 @@ set(WWLIB_SRC
     #global.h
     hash.cpp
     hash.h
+    IcuSupport.h
     hashcalc.h
     HASHLIST.h
     #hashtab.h
@@ -188,3 +189,7 @@ target_link_libraries(core_wwlib PRIVATE
     core_wwcommon
     corei_always
 )
+
+target_link_libraries(core_wwlib PUBLIC
+    core_icu
+)

Core/Libraries/Source/WWVegas/WWLib/IcuSupport.h

@@ -0,0 +1,58 @@
+/*
+**	Command & Conquer Generals Zero Hour(tm)
+**	Copyright 2026 TheSuperHackers
+**
+**	This program is free software: you can redistribute it and/or modify
+**	it under the terms of the GNU General Public License as published by
+**	the Free Software Foundation, either version 3 of the License, or
+**	(at your option) any later version.
+**
+**	This program is distributed in the hope that it will be useful,
+**	but WITHOUT ANY WARRANTY; without even the implied warranty of
+**	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+**	GNU General Public License for more details.
+**
+**	You should have received a copy of the GNU General Public License
+**	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+// Engine entry point for ICU4C.
+//
+// RTS_HAS_ICU        - ICU C API is linked; include this header and call ICU functions.
+// RTS_HAS_ICU_CXX    - ICU C++ API (icu::UnicodeString, icu::Locale, ...).
+// RTS_HAS_ICU_I18N   - Collation, break iteration, converters, and related i18n APIs.
+// RTS_HAS_ICU_WINSDK - Windows SDK merged C API via <icu.h> (no C++ API).
+// RTS_ICU_DYNAMIC    - No import library; utf8.cpp LoadLibrary's OS icu.dll (VC6).
+
+#if defined(RTS_HAS_ICU_WINSDK)
+
+#include <icu.h>
+
+#elif defined(RTS_HAS_ICU)
+
+#include <unicode/uchar.h>
+#include <unicode/ucnv.h>
+#include <unicode/ustring.h>
+#include <unicode/utf8.h>
+#include <unicode/utypes.h>
+
+#if defined(RTS_HAS_ICU_I18N)
+#include <unicode/ubrk.h>
+#include <unicode/ucol.h>
+#include <unicode/uidna.h>
+#include <unicode/uloc.h>
+#include <unicode/unorm2.h>
+#endif
+
+#if defined(RTS_HAS_ICU_CXX)
+#include <unicode/locid.h>
+#include <unicode/normalizer2.h>
+#include <unicode/unistr.h>
+#if defined(RTS_HAS_ICU_I18N)
+#include <unicode/coll.h>
+#endif
+#endif
+
+#endif