Skip to content

Guard against 64-bit overflow in ComputePitch #733

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Roland Shum (ShumWengSang) wants to merge 1 commit into
base: main
Choose a base branch
from
+71 −34
Open

Guard against 64-bit overflow in ComputePitch #733

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 71 additions & 34 deletions DirectXTex/DirectXTexUtil.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -953,6 +953,33 @@ size_t DirectX::BytesPerBlock(DXGI_FORMAT fmt) noexcept
}


namespace
{
// Multiplies two 64-bit values, setting 'overflow' if the mathematical result is
// not representable. Returns 0 in that case; callers test the flag once after all
// computations rather than at every individual site.
inline uint64_t MulOverflow(uint64_t a, uint64_t b, bool& overflow) noexcept
{
#if defined(__GNUC__) || defined(__clang__)
uint64_t result = 0;
if (__builtin_mul_overflow(a, b, &result))
{
overflow = true;
return 0;
}
return result;
#else
const uint64_t result = a * b;
if ((a != 0) && ((result / a) != b))
{
overflow = true;
return 0;
}
return result;
#endif
}
}

//-------------------------------------------------------------------------------------
// Computes the image row pitch in bytes, and the slice ptich (size in bytes of the image)
// based on DXGI format, width, and height
Expand All @@ -963,6 +990,7 @@ HRESULT DirectX::ComputePitch(DXGI_FORMAT fmt, size_t width, size_t height,
{
uint64_t pitch = 0;
uint64_t slice = 0;
bool overflow = false;

switch (static_cast<int>(fmt))
{
Expand All @@ -981,15 +1009,15 @@ HRESULT DirectX::ComputePitch(DXGI_FORMAT fmt, size_t width, size_t height,
{
const size_t nbw = width >> 2;
const size_t nbh = height >> 2;
pitch = std::max<uint64_t>(1u, uint64_t(nbw) * 8u);
slice = std::max<uint64_t>(1u, pitch * uint64_t(nbh));
pitch = std::max<uint64_t>(1u, MulOverflow(uint64_t(nbw), 8u, overflow));
slice = std::max<uint64_t>(1u, MulOverflow(pitch, uint64_t(nbh), overflow));
}
else
{
const uint64_t nbw = std::max<uint64_t>(1u, (uint64_t(width) + 3u) / 4u);
const uint64_t nbh = std::max<uint64_t>(1u, (uint64_t(height) + 3u) / 4u);
pitch = nbw * 8u;
slice = pitch * nbh;
pitch = MulOverflow(nbw, 8u, overflow);
slice = MulOverflow(pitch, nbh, overflow);
}
}
break;
Expand All @@ -1015,15 +1043,15 @@ HRESULT DirectX::ComputePitch(DXGI_FORMAT fmt, size_t width, size_t height,
{
const size_t nbw = width >> 2;
const size_t nbh = height >> 2;
pitch = std::max<uint64_t>(1u, uint64_t(nbw) * 16u);
slice = std::max<uint64_t>(1u, pitch * uint64_t(nbh));
pitch = std::max<uint64_t>(1u, MulOverflow(uint64_t(nbw), 16u, overflow));
slice = std::max<uint64_t>(1u, MulOverflow(pitch, uint64_t(nbh), overflow));
}
else
{
const uint64_t nbw = std::max<uint64_t>(1u, (uint64_t(width) + 3u) / 4u);
const uint64_t nbh = std::max<uint64_t>(1u, (uint64_t(height) + 3u) / 4u);
pitch = nbw * 16u;
slice = pitch * nbh;
pitch = MulOverflow(nbw, 16u, overflow);
slice = MulOverflow(pitch, nbh, overflow);
}
}
break;
Expand All @@ -1032,15 +1060,15 @@ HRESULT DirectX::ComputePitch(DXGI_FORMAT fmt, size_t width, size_t height,
case DXGI_FORMAT_G8R8_G8B8_UNORM:
case DXGI_FORMAT_YUY2:
assert(IsPacked(fmt));
pitch = ((uint64_t(width) + 1u) >> 1) * 4u;
slice = pitch * uint64_t(height);
pitch = MulOverflow((uint64_t(width) + 1u) >> 1, 4u, overflow);
slice = MulOverflow(pitch, uint64_t(height), overflow);
break;

case DXGI_FORMAT_Y210:
case DXGI_FORMAT_Y216:
assert(IsPacked(fmt));
pitch = ((uint64_t(width) + 1u) >> 1) * 8u;
slice = pitch * uint64_t(height);
pitch = MulOverflow((uint64_t(width) + 1u) >> 1, 8u, overflow);
slice = MulOverflow(pitch, uint64_t(height), overflow);
break;

case DXGI_FORMAT_NV12:
Expand All @@ -1051,8 +1079,8 @@ HRESULT DirectX::ComputePitch(DXGI_FORMAT fmt, size_t width, size_t height,
return E_INVALIDARG;
}
assert(IsPlanar(fmt));
pitch = ((uint64_t(width) + 1u) >> 1) * 2u;
slice = pitch * (uint64_t(height) + ((uint64_t(height) + 1u) >> 1));
pitch = MulOverflow((uint64_t(width) + 1u) >> 1, 2u, overflow);
slice = MulOverflow(pitch, uint64_t(height) + ((uint64_t(height) + 1u) >> 1), overflow);
break;

case DXGI_FORMAT_P010:
Expand All @@ -1075,20 +1103,20 @@ HRESULT DirectX::ComputePitch(DXGI_FORMAT fmt, size_t width, size_t height,
case XBOX_DXGI_FORMAT_R16_UNORM_X8_TYPELESS:
case XBOX_DXGI_FORMAT_X16_TYPELESS_G8_UINT:
assert(IsPlanar(fmt));
pitch = ((uint64_t(width) + 1u) >> 1) * 4u;
slice = pitch * (uint64_t(height) + ((uint64_t(height) + 1u) >> 1));
pitch = MulOverflow((uint64_t(width) + 1u) >> 1, 4u, overflow);
slice = MulOverflow(pitch, uint64_t(height) + ((uint64_t(height) + 1u) >> 1), overflow);
break;

case DXGI_FORMAT_NV11:
assert(IsPlanar(fmt));
pitch = ((uint64_t(width) + 3u) >> 2) * 4u;
slice = pitch * uint64_t(height) * 2u;
pitch = MulOverflow((uint64_t(width) + 3u) >> 2, 4u, overflow);
slice = MulOverflow(MulOverflow(pitch, uint64_t(height), overflow), 2u, overflow);
break;

case WIN10_DXGI_FORMAT_P208:
assert(IsPlanar(fmt));
pitch = ((uint64_t(width) + 1u) >> 1) * 2u;
slice = pitch * uint64_t(height) * 2u;
pitch = MulOverflow((uint64_t(width) + 1u) >> 1, 2u, overflow);
slice = MulOverflow(MulOverflow(pitch, uint64_t(height), overflow), 2u, overflow);
break;

case WIN10_DXGI_FORMAT_V208:
Expand All @@ -1099,13 +1127,13 @@ HRESULT DirectX::ComputePitch(DXGI_FORMAT fmt, size_t width, size_t height,
}
assert(IsPlanar(fmt));
pitch = uint64_t(width);
slice = pitch * (uint64_t(height) + (((uint64_t(height) + 1u) >> 1) * 2u));
slice = MulOverflow(pitch, uint64_t(height) + (((uint64_t(height) + 1u) >> 1) * 2u), overflow);
break;

case WIN10_DXGI_FORMAT_V408:
assert(IsPlanar(fmt));
pitch = uint64_t(width);
slice = pitch * (uint64_t(height) + (uint64_t(height >> 1) * 4u));
slice = MulOverflow(pitch, uint64_t(height) + (uint64_t(height >> 1) * 4u), overflow);
break;

default:
Expand All @@ -1129,42 +1157,51 @@ HRESULT DirectX::ComputePitch(DXGI_FORMAT fmt, size_t width, size_t height,
{
if (flags & CP_FLAGS_PAGE4K)
{
pitch = ((uint64_t(width) * bpp + 32767u) / 32768u) * 4096u;
slice = pitch * uint64_t(height);
pitch = MulOverflow((MulOverflow(uint64_t(width), bpp, overflow) + 32767u) / 32768u, 4096u, overflow);
slice = MulOverflow(pitch, uint64_t(height), overflow);
}
else if (flags & CP_FLAGS_ZMM)
{
pitch = ((uint64_t(width) * bpp + 511u) / 512u) * 64u;
slice = pitch * uint64_t(height);
pitch = MulOverflow((MulOverflow(uint64_t(width), bpp, overflow) + 511u) / 512u, 64u, overflow);
slice = MulOverflow(pitch, uint64_t(height), overflow);
}
else if (flags & CP_FLAGS_YMM)
{
pitch = ((uint64_t(width) * bpp + 255u) / 256u) * 32u;
slice = pitch * uint64_t(height);
pitch = MulOverflow((MulOverflow(uint64_t(width), bpp, overflow) + 255u) / 256u, 32u, overflow);
slice = MulOverflow(pitch, uint64_t(height), overflow);
}
else if (flags & CP_FLAGS_PARAGRAPH)
{
pitch = ((uint64_t(width) * bpp + 127u) / 128u) * 16u;
slice = pitch * uint64_t(height);
pitch = MulOverflow((MulOverflow(uint64_t(width), bpp, overflow) + 127u) / 128u, 16u, overflow);
slice = MulOverflow(pitch, uint64_t(height), overflow);
}
else // DWORD alignment
{
// Special computation for some incorrectly created DDS files based on
// legacy DirectDraw assumptions about pitch alignment
pitch = ((uint64_t(width) * bpp + 31u) / 32u) * sizeof(uint32_t);
slice = pitch * uint64_t(height);
pitch = MulOverflow((MulOverflow(uint64_t(width), bpp, overflow) + 31u) / 32u, sizeof(uint32_t), overflow);
slice = MulOverflow(pitch, uint64_t(height), overflow);
}
}
else
{
// Default byte alignment
pitch = (uint64_t(width) * bpp + 7u) / 8u;
slice = pitch * uint64_t(height);
pitch = (MulOverflow(uint64_t(width), bpp, overflow) + 7u) / 8u;
slice = MulOverflow(pitch, uint64_t(height), overflow);
}
}
break;
}

// A 64-bit overflow in the computations above would silently produce a slice pitch
// inconsistent with the row pitch and scanline count, which callers rely on to size
// allocations. Reject rather than truncate.
if (overflow)
{
rowPitch = slicePitch = 0;
return HRESULT_E_ARITHMETIC_OVERFLOW;
}

#if defined(_M_IX86) || defined(_M_ARM) || defined(_M_HYBRID_X86_ARM64)
static_assert(sizeof(size_t) == 4, "Not a 32-bit platform!");
if (pitch > UINT32_MAX || slice > UINT32_MAX)
Expand Down
Loading