Avoid overflow when multiplying integers in RankFilter

Tests/test_image_filter.py

@@ -180,6 +180,18 @@ def test_rankfilter_properties() -> None:
         ImageFilter.RankFilter(1, 1)
 
 
+def test_rankfilter_overflow() -> None:
+    # Large margins used to overflow the ImagingExpand overflow guard itself (SIGFPE),
+    # by mutating RankFilter.size after construction, bypassing __init__'s validation.
+    im = Image.new("L", (16, 16))
+    rankfilter = ImageFilter.RankFilter(3, 0)
+
+    for size in (2**31, 2**32 - 1):  # margins of 2**30 and INT_MAX
+        rankfilter.size = size
+        with pytest.raises(ValueError, match="filter size too large"):
+            im.filter(rankfilter)
+
+
 def test_builtinfilter_p() -> None:
     builtin_filter = ImageFilter.BuiltinFilter()
 

src/libImaging/Filter.c

@@ -63,7 +63,10 @@ ImagingExpand(Imaging imIn, int margin) {
     if (margin < 0) {
         return (Imaging)ImagingError_ValueError("bad kernel size");
     }
-    if (margin > 0 && margin > INT_MAX / (margin * (int)sizeof(FLOAT32))) {
+    // Compute in int64_t via division, not squaring, so the check itself
+    // can't overflow or divide by zero for any valid int margin.
+    if (margin > 0 && (int64_t)margin > (int64_t)INT_MAX / ((int64_t)margin *
+                                                            (int64_t)sizeof(FLOAT32))) {
         return (Imaging)ImagingError_ValueError("filter size too large");
     }