ptex: More comprehensive valid_file and validation of header

src/ptex.imageio/ptexinput.cpp

@@ -4,8 +4,16 @@
 
 #include <Ptexture.h>
 
+#include <cstddef>
+#include <cstring>
+
+#include <OpenImageIO/bit.h>
 #include <OpenImageIO/dassert.h>
+#include <OpenImageIO/filesystem.h>
 #include <OpenImageIO/imageio.h>
+#include <OpenImageIO/platform.h>
+#include <OpenImageIO/span.h>
+#include <OpenImageIO/strutil.h>
 #include <OpenImageIO/typedesc.h>
 
 OIIO_PLUGIN_NAMESPACE_BEGIN
@@ -27,6 +35,7 @@ class PtexInput final : public ImageInput {
                 || feature == "iptc"  // Because of arbitrary_metadata
                 || feature == "multiimage" || feature == "mipmap");
     }
+    bool valid_file(const std::string& filename) const override;
     bool open(const std::string& name, ImageSpec& newspec) override;
     bool close() override;
     int current_subimage(void) const override
@@ -98,9 +107,225 @@ OIIO_PLUGIN_EXPORTS_END
 
 
 
+namespace {
+
+// The on-disk Ptex file header is exactly 64 bytes, stored little-endian. See
+// the format documentation at https://ptex.us/PtexFile.html and the Header
+// struct in PtexIO.h of the Ptex distribution. We parse and sanity-check it
+// ourselves before handing the file to PtexTexture::open(), which does a poor
+// job of detecting corrupt headers and is prone to over-allocating based on
+// bogus face/level/channel counts.
+constexpr int ptex_header_size = 64;
+
+// Generous sanity caps -- far larger than any real Ptex file, but small enough
+// to reject wildly corrupt headers before they drive huge allocations. nlevels
+// is a per-face mip count, so the cap of 20 already implies a maximum face
+// resolution of 2^20 in each dimension -- well beyond anything real.
+constexpr uint32_t ptex_max_channels = 1024;
+constexpr uint32_t ptex_max_levels   = 20;
+
+// Maximum plausible expansion of a zlib-compressed block. zlib's theoretical
+// worst case is ~1032:1; we use a looser bound so that we never reject a valid
+// file, only headers whose claimed uncompressed size could not possibly have
+// come from the (smaller) number of compressed bytes actually in the file.
+constexpr uint64_t ptex_max_zip_ratio = 4096;
+
+// Sizes of the fixed on-disk records, used for consistency checks:
+//   FaceInfo  = Res[2] + adjedges[1] + flags[1] + adjfaces[4*4]
+//   LevelInfo = leveldatasize[8] + levelheadersize[4] + nfaces[4]
+constexpr uint64_t ptex_faceinfo_size  = 20;
+constexpr uint64_t ptex_levelinfo_size = 16;
+
+
+// Parsed, host-endian copy of the 64-byte on-disk Ptex header. Constructed
+// directly from the raw little-endian header bytes; each field is byte-swapped
+// on big-endian hosts. See https://ptex.us/PtexFile.html and the Header struct
+// in PtexIO.h of the Ptex distribution.
+struct PtexHeader {
+    char magic[4];
+    uint32_t version;
+    uint32_t meshtype;
+    uint32_t datatype;
+    int32_t alphachan;
+    uint16_t nchannels;
+    uint16_t nlevels;
+    uint32_t nfaces;
+    uint32_t extheadersize;
+    uint32_t faceinfosize;
+    uint32_t constdatasize;
+    uint32_t levelinfosize;
+    uint32_t minorversion;
+    uint64_t leveldatasize;
+    uint32_t metadatazipsize;
+    uint32_t metadatamemsize;
+
+    // Parse from at least `ptex_header_size` bytes of raw header data. The
+    // struct has no padding (see the static_assert below), so the on-disk
+    // layout can be copied in one shot; integer fields are then byte-swapped on
+    // big-endian hosts (the magic is a byte sequence and needs no swap).
+    explicit PtexHeader(cspan<std::byte> b)
+    {
+        OIIO_DASSERT(b.size() >= size_t(ptex_header_size));
+        std::memcpy(this, b.data(), ptex_header_size);
+        if (bigendian()) {
+            version         = byteswap(version);
+            meshtype        = byteswap(meshtype);
+            datatype        = byteswap(datatype);
+            alphachan       = byteswap(alphachan);
+            nchannels       = byteswap(nchannels);
+            nlevels         = byteswap(nlevels);
+            nfaces          = byteswap(nfaces);
+            extheadersize   = byteswap(extheadersize);
+            faceinfosize    = byteswap(faceinfosize);
+            constdatasize   = byteswap(constdatasize);
+            levelinfosize   = byteswap(levelinfosize);
+            minorversion    = byteswap(minorversion);
+            leveldatasize   = byteswap(leveldatasize);
+            metadatazipsize = byteswap(metadatazipsize);
+            metadatamemsize = byteswap(metadatamemsize);
+        }
+    }
+
+    bool valid_magic() const
+    {
+        return magic[0] == 'P' && magic[1] == 't' && magic[2] == 'e'
+               && magic[3] == 'x';
+    }
+};
+
+static_assert(sizeof(PtexHeader) == ptex_header_size,
+              "PtexHeader must exactly match the 64-byte on-disk Ptex header");
+
+
+
+// Validate the Ptex header at the start of a file held in `b`, given the total
+// file size. On any failure, set `err` to a human-readable reason and return
+// false. This catches the great majority of corrupt or malicious headers that
+// PtexTexture::open() would either miss or respond to by over-allocating.
+bool
+ptex_validate_header(cspan<std::byte> b, uint64_t filesize, std::string& err)
+{
+    if (b.size() < size_t(ptex_header_size)) {
+        err = "file is too small to contain a Ptex header";
+        return false;
+    }
+    PtexHeader h(b);
+
+    if (!h.valid_magic()) {
+        err = "not a Ptex file (wrong magic number)";
+        return false;
+    }
+    if (h.version != 1) {
+        err = Strutil::format("unsupported Ptex file version {}", h.version);
+        return false;
+    }
+    if (h.meshtype > 1) {  // mt_triangle, mt_quad
+        err = Strutil::format("invalid Ptex mesh type {}", h.meshtype);
+        return false;
+    }
+    if (h.datatype > 3) {  // dt_uint8, dt_uint16, dt_half, dt_float
+        err = Strutil::format("invalid Ptex data type {}", h.datatype);
+        return false;
+    }
+    if (h.nchannels < 1 || h.nchannels > ptex_max_channels) {
+        err = Strutil::format("unreasonable Ptex channel count {}",
+                              h.nchannels);
+        return false;
+    }
+    if (h.alphachan != -1
+        && (h.alphachan < 0 || uint32_t(h.alphachan) >= h.nchannels)) {
+        err = Strutil::format("invalid Ptex alpha channel {}", h.alphachan);
+        return false;
+    }
+    if (h.nfaces < 1) {
+        err = "Ptex file has no faces";
+        return false;
+    }
+    if (h.nlevels < 1 || h.nlevels > ptex_max_levels) {
+        err = Strutil::format("unreasonable Ptex level count {}", h.nlevels);
+        return false;
+    }
+    // The level-info block is stored uncompressed as exactly nlevels
+    // fixed-size records, so its size must match precisely.
+    if (h.levelinfosize != uint64_t(h.nlevels) * ptex_levelinfo_size) {
+        err = "inconsistent Ptex level info size (corrupt header?)";
+        return false;
+    }
+    // Every on-disk block must fit within the actual file. (Optional edit
+    // blocks may follow, so the sum can be smaller than the file, but never
+    // larger.) Check leveldatasize first so the running sum cannot overflow.
+    if (h.leveldatasize > filesize) {
+        err = "Ptex level data size exceeds the file size (corrupt header?)";
+        return false;
+    }
+    uint64_t claimed = uint64_t(ptex_header_size) + h.extheadersize
+                       + h.faceinfosize + h.constdatasize + h.levelinfosize
+                       + h.metadatazipsize + h.leveldatasize;
+    if (claimed > filesize) {
+        err = "Ptex header block sizes exceed the file size (corrupt header?)";
+        return false;
+    }
+    // The compressed blocks below are inflated into freshly-allocated buffers
+    // whose sizes are taken from the header. Reject any header whose claimed
+    // uncompressed size could not plausibly have been produced from the
+    // on-disk compressed bytes -- this is the main defense against headers
+    // engineered to provoke huge allocations.
+    static const uint64_t dtsize[4] = { 1, 2, 2, 4 };
+    const uint64_t pixelsize        = dtsize[h.datatype] * h.nchannels;
+    // faceinfo: read into an array resized to nfaces (nfaces * FaceInfo bytes).
+    if (uint64_t(h.nfaces) * ptex_faceinfo_size
+        > uint64_t(h.faceinfosize) * ptex_max_zip_ratio) {
+        err = Strutil::format("unreasonable Ptex face count {}", h.nfaces);
+        return false;
+    }
+    // constdata: one constant pixel per face (nfaces * pixelsize bytes).
+    if (uint64_t(h.nfaces) * pixelsize
+        > uint64_t(h.constdatasize) * ptex_max_zip_ratio) {
+        err = "unreasonable Ptex constant-data size (corrupt header?)";
+        return false;
+    }
+    // metadata: inflated from metadatazipsize to metadatamemsize.
+    if (uint64_t(h.metadatamemsize)
+        > uint64_t(h.metadatazipsize) * ptex_max_zip_ratio) {
+        err = "unreasonable Ptex metadata size (corrupt header?)";
+        return false;
+    }
+    return true;
+}
+
+}  // namespace
+
+
+
+bool
+PtexInput::valid_file(const std::string& filename) const
+{
+    std::byte header[ptex_header_size];
+    size_t n = Filesystem::read_bytes(filename, header, ptex_header_size);
+    std::string err;
+    return ptex_validate_header(cspan<std::byte>(header, n),
+                                Filesystem::file_size(filename), err);
+}
+
+
+
 bool
 PtexInput::open(const std::string& name, ImageSpec& newspec)
 {
+    // Validate the header ourselves before handing off to PtexTexture::open(),
+    // which is poor at detecting corruption and prone to over-allocating on
+    // bogus face/level/channel counts.
+    {
+        std::byte header[ptex_header_size];
+        size_t n = Filesystem::read_bytes(name, header, ptex_header_size);
+        std::string err;
+        if (!ptex_validate_header(cspan<std::byte>(header, n),
+                                  Filesystem::file_size(name), err)) {
+            errorfmt("{}", err);
+            return false;
+        }
+    }
+
     Ptex::String perr;
     m_ptex = PtexTexture::open(name.c_str(), perr, true /*premultiply*/);
     if (!perr.empty()) {