Add tests for UB in CI, correct some more float->int UB

Author: HaroldCindy

Analysis/src/BuiltinDefinitions.cpp

@@ -1282,6 +1282,10 @@ TypeId makeStringMetatable(NotNull<BuiltinTypes> builtinTypes, SolverMode mode)
     return arena->addType(TableType{{{{"__index", {tableType}}}}, std::nullopt, TypeLevel{}, TableState::Sealed});
 }
 
+// ServerLua: Suppress UBSan for out-of-range select() index (bounds-checked afterward)
+#if defined(__clang__) || defined(__GNUC__)
+__attribute__((no_sanitize("float-cast-overflow")))
+#endif
 std::optional<WithPredicate<TypePackId>> MagicSelect::handleOldSolver(
     TypeChecker& typechecker,
     const ScopePtr& scope,

Ast/src/Parser.cpp

@@ -3130,6 +3130,10 @@ AstExpr* Parser::parseAssertionExpr()
         return expr;
 }
 
+// ServerLua: Suppress UBSan for intentional precision-loss detection cast
+#if defined(__clang__) || defined(__GNUC__)
+__attribute__((no_sanitize("float-cast-overflow")))
+#endif
 static ConstantNumberParseResult parseInteger(double& result, const char* data, int base)
 {
     LUAU_ASSERT(base == 2 || base == 16);

Makefile

@@ -120,8 +120,8 @@ ifeq ($(config),coverage)
 endif
 
 ifeq ($(config),sanitize)
-	CXXFLAGS+=-fsanitize=address -O1 -DLUAU_ENABLE_ASAN=1
-	LDFLAGS+=-fsanitize=address
+	CXXFLAGS+=-fsanitize=address,undefined -fno-sanitize=vptr -O1 -DLUAU_ENABLE_ASAN=1
+	LDFLAGS+=-fsanitize=address,undefined
 endif
 
 ifeq ($(config),analyze)

VM/src/cjson/lua_cjson.cpp

@@ -697,8 +697,8 @@ static int lua_array_length(lua_State *l, json_config_t *cfg, strbuf_t *json)
         /* table, key, value */
         if (lua_type(l, -2) == LUA_TNUMBER &&
             (k = lua_tonumber(l, -2))) {
-            /* Integer >= 1 ? */
-            if (floor(k) == k && k >= 1) {
+            /* Integer >= 1 and in int range? (floor(inf)==inf, so check upper bound) */
+            if (floor(k) == k && k >= 1 && k <= INT_MAX) {
                 if (k > max)
                     max = k;
                 items++;

VM/src/lapi.cpp

@@ -431,9 +431,9 @@ int lua_tointegerx(lua_State* L, int idx, int* isnum)
     }
     else if (tonumber(o, &n))
     {
-        int res;
+        // ServerLua: Use safe conversion for cross-platform consistency
         double num = nvalue(o);
-        luai_num2int(res, num);
+        int res = luai_num2int_safe(num);
         if (isnum)
             *isnum = 1;
         return res;

VM/src/lnumutils.h

@@ -64,7 +64,28 @@ inline float luai_lerpf(float a, float b, float t)
     return (t == 1.0) ? b : a + (b - a) * t;
 }
 
+// ServerLua: Safe float-to-int with x86 semantics
+inline int luai_num2int_safe(double d)
+{
+    constexpr double kMaxInt = 2147483648.0; // 2^31
+    if (d >= -kMaxInt && d < kMaxInt)
+        return (int)d;
+    return (int)(-kMaxInt);
+}
+
+// ServerLua: Suppress UBSan float-cast-overflow in ASAN builds
+#ifdef LUAU_ENABLE_ASAN
+#if defined(__clang__) || defined(__GNUC__)
+__attribute__((no_sanitize("float-cast-overflow")))
+#endif
+inline int luai_num2int_impl(double d)
+{
+    return (int)d;
+}
+#define luai_num2int(i, d) ((i) = luai_num2int_impl(d))
+#else
 #define luai_num2int(i, d) ((i) = (int)(d))
+#endif
 
 // On MSVC in 32-bit, double to unsigned cast compiles into a call to __dtoui3, so we invoke x87->int64 conversion path manually
 #if defined(_MSC_VER) && defined(_M_IX86)

VM/src/lvmexecute.cpp

@@ -707,8 +707,10 @@ static void luau_execute(lua_State* L)
                 {
                     LuaTable* h = hvalue(rb);
 
+                    // ServerLua: Use luai_num2int for UBSan suppression in ASAN builds
                     double indexd = nvalue(rc);
-                    int index = int(indexd);
+                    int index;
+                    luai_num2int(index, indexd);
 
                     // index has to be an exact integer and in-bounds for the array portion
                     if (LUAU_LIKELY(unsigned(index) - 1 < unsigned(h->sizearray) && !h->metatable && double(index) == indexd))
@@ -748,8 +750,10 @@ static void luau_execute(lua_State* L)
                 {
                     LuaTable* h = hvalue(rb);
 
+                    // ServerLua: Use luai_num2int for UBSan suppression in ASAN builds
                     double indexd = nvalue(rc);
-                    int index = int(indexd);
+                    int index;
+                    luai_num2int(index, indexd);
 
                     // index has to be an exact integer and in-bounds for the array portion
                     if (LUAU_LIKELY(unsigned(index) - 1 < unsigned(h->sizearray) && !h->metatable && !h->readonly && double(index) == indexd))
@@ -1856,7 +1860,9 @@ static void luau_execute(lua_State* L)
                     {
                         // ServerLua: division by -1 is weird, and
                         // previously special-cased to prevent division errors.
-                        setintvalue(ra, -1 * intvalue(rb));
+                        // Note: -INT32_MIN overflows, so handle that case explicitly.
+                        int32_t val = intvalue(rb);
+                        setintvalue(ra, val == INT32_MIN ? INT32_MIN : -val);
                     }
                     else
                     {
@@ -2148,7 +2154,9 @@ static void luau_execute(lua_State* L)
                     {
                         // ServerLua: division by -1 is weird, and
                         // previously special-cased to prevent division errors.
-                        setintvalue(ra, -1 * intvalue(rb));
+                        // Note: -INT32_MIN overflows, so handle that case explicitly.
+                        int32_t val = intvalue(rb);
+                        setintvalue(ra, val == INT32_MIN ? INT32_MIN : -val);
                     }
                     else
                     {