diff --git a/Tests/test_image_filter.py b/Tests/test_image_filter.py index 974d4b7a3ad..3a970520ccb 100644 --- a/Tests/test_image_filter.py +++ b/Tests/test_image_filter.py @@ -180,6 +180,18 @@ def test_rankfilter_properties() -> None: ImageFilter.RankFilter(1, 1) +def test_rankfilter_overflow() -> None: + # Large margins used to overflow the ImagingExpand overflow guard itself (SIGFPE), + # by mutating RankFilter.size after construction, bypassing __init__'s validation. + im = Image.new("L", (16, 16)) + rankfilter = ImageFilter.RankFilter(3, 0) + + for size in (2**31, 2**32 - 1): # margins of 2**30 and INT_MAX + rankfilter.size = size + with pytest.raises(ValueError, match="filter size too large"): + im.filter(rankfilter) + + def test_builtinfilter_p() -> None: builtin_filter = ImageFilter.BuiltinFilter() diff --git a/src/libImaging/Filter.c b/src/libImaging/Filter.c index d03aec1d85f..7982419ff16 100644 --- a/src/libImaging/Filter.c +++ b/src/libImaging/Filter.c @@ -63,7 +63,10 @@ ImagingExpand(Imaging imIn, int margin) { if (margin < 0) { return (Imaging)ImagingError_ValueError("bad kernel size"); } - if (margin > 0 && margin > INT_MAX / (margin * (int)sizeof(FLOAT32))) { + // Compute in int64_t via division, not squaring, so the check itself + // can't overflow or divide by zero for any valid int margin. + if (margin > 0 && (int64_t)margin > (int64_t)INT_MAX / ((int64_t)margin * + (int64_t)sizeof(FLOAT32))) { return (Imaging)ImagingError_ValueError("filter size too large"); }