Check (attributes) file when using --update flag

docs/commands/add.md

@@ -58,9 +58,22 @@ $ mpqcli add wow-patch.mpq textures/ --path textures
 
 ## Skip unchanged files with --update
 
-When adding a directory, the `--update` flag skips any file whose on-disk size matches the
-size already stored in the archive. This is useful for incremental updates where only
-changed files need to be re-added.
+When adding a directory, the `--update` flag skips files that have not changed since they
+were last added to the archive. This is useful for incremental updates where only changed
+files need to be re-added.
+
+The skip decision is made in two steps:
+
+1. **File size** must match. If the sizes differ the file is always re-added.
+2. If the sizes match, the archive's `(attributes)` file is consulted for a stronger
+   check. The most reliable attribute available wins:
+   - **MD5** – if the archive stores MD5 checksums, the MD5 of the local file is computed
+     and compared. A match skips the file; a mismatch re-adds it.
+   - **CRC32** – used when MD5 is absent. Same logic.
+   - **Timestamp** – used when neither MD5 nor CRC32 is present. The file's
+     last-modification time is compared at one-second resolution.
+   - **No attributes** – if the archive has no `(attributes)` file, the file is always
+     re-added even when sizes match, because no reliable content check is possible.
 
 ```bash
 $ mpqcli add wow-patch.mpq textures/ --update --overwrite
@@ -69,10 +82,6 @@ $ mpqcli add wow-patch.mpq textures/ --update --overwrite
 [*] For textures: 1 files added, 1 files skipped, 0 files failed.
 ```
 
-Note: the skip check is size-based only. Files with the same size but different content
-are not detected as changed. If precise change detection matters, pass `--overwrite`
-without `--update` to unconditionally replace every file.
-
 ## Control where files are stored
 
 For single files, one can specify both directory and filename in one step using `-p` or `--path`:

src/completion/mpqcli.fish

@@ -129,7 +129,7 @@ complete -c mpqcli -n '__fish_seen_subcommand_from add' \
 complete -c mpqcli -n '__fish_seen_subcommand_from add' \
     -s w -l overwrite       -d 'Overwrite file if it already exists in the archive'
 complete -c mpqcli -n '__fish_seen_subcommand_from add' \
-    -s u -l update          -d 'Skip files whose archived size matches on-disk size'
+    -s u -l update          -d 'Skip unchanged files when adding a directory'
 complete -c mpqcli -n '__fish_seen_subcommand_from add' \
     -l locale               -d 'Locale to use for added file' \
     -r -a "$__mpqcli_locales"

src/completion/mpqcli.ps1

@@ -105,8 +105,8 @@ Register-ArgumentCompleter -Native -CommandName 'mpqcli', 'mpqcli.exe' -ScriptBl
             '--path'      = 'Archive path for a single file, or prefix for a directory'
             '-w'          = 'Overwrite file if it already is in MPQ archive'
             '--overwrite' = 'Overwrite file if it already is in MPQ archive'
-            '-u'          = 'Skip files whose archived size matches on-disk size'
-            '--update'    = 'Skip files whose archived size matches on-disk size'
+            '-u'          = 'Skip unchanged files when adding a directory'
+            '--update'    = 'Skip unchanged files when adding a directory'
             '--locale'    = 'Locale to use for added file'
             '-g'          = 'Game profile for compression rules'
             '--game'      = 'Game profile for compression rules'

src/helpers.cpp

@@ -1,10 +1,13 @@
 #include "helpers.h"
 
 #include <algorithm>
+#include <array>
 #include <cstring>
 #include <ctime>
 #include <filesystem>
+#include <fstream>
 #include <iostream>
+#include <sys/stat.h>
 
 #ifdef _WIN32
 #include <fcntl.h>
@@ -128,3 +131,184 @@ void PrintAsBinary(const char *buffer, uint32_t size) {
 #endif
     std::cout.write(buffer, size);
 }
+
+// ---- CRC32 --------------------------------------------------------------
+// Standard ZIP/gzip CRC32 (polynomial 0xEDB88320, reflected bit order).
+
+static const std::array<uint32_t, 256> kCrc32Table = []() {
+    std::array<uint32_t, 256> t{};
+    for (uint32_t i = 0; i < 256; ++i) {
+        uint32_t c = i;
+        for (int j = 0; j < 8; ++j)
+            c = (c >> 1) ^ (c & 1u ? 0xEDB88320u : 0u);
+        t[i] = c;
+    }
+    return t;
+}();
+
+std::optional<uint32_t> ComputeFileCrc32(const fs::path &path) {
+    std::ifstream f(path, std::ios::binary);
+    if (!f) {
+        return std::nullopt;
+    }
+    uint32_t crc = 0xFFFFFFFFu;
+    char buf[65536];
+    while (f.read(buf, sizeof(buf)) || f.gcount() > 0) {
+        for (std::streamsize i = 0; i < f.gcount(); ++i)
+            crc = (crc >> 8) ^ kCrc32Table[(crc ^ static_cast<uint8_t>(buf[i])) & 0xFFu];
+    }
+    return crc ^ 0xFFFFFFFFu;
+}
+
+// ---- MD5 ----------------------------------------------------------------
+// Compact RFC 1321 implementation – no external dependencies.
+
+namespace {
+
+// clang-format off
+constexpr uint32_t kMd5K[64] = {
+    // Round 1
+    0xd76aa478u, 0xe8c7b756u, 0x242070dbu, 0xc1bdceeeu,
+    0xf57c0fafu, 0x4787c62au, 0xa8304613u, 0xfd469501u,
+    0x698098d8u, 0x8b44f7afu, 0xffff5bb1u, 0x895cd7beu,
+    0x6b901122u, 0xfd987193u, 0xa679438eu, 0x49b40821u,
+    // Round 2
+    0xf61e2562u, 0xc040b340u, 0x265e5a51u, 0xe9b6c7aau,
+    0xd62f105du, 0x02441453u, 0xd8a1e681u, 0xe7d3fbc8u,
+    0x21e1cde6u, 0xc33707d6u, 0xf4d50d87u, 0x455a14edu,
+    0xa9e3e905u, 0xfcefa3f8u, 0x676f02d9u, 0x8d2a4c8au,
+    // Round 3
+    0xfffa3942u, 0x8771f681u, 0x6d9d6122u, 0xfde5380cu,
+    0xa4beea44u, 0x4bdecfa9u, 0xf6bb4b60u, 0xbebfbc70u,
+    0x289b7ec6u, 0xeaa127fau, 0xd4ef3085u, 0x04881d05u,
+    0xd9d4d039u, 0xe6db99e5u, 0x1fa27cf8u, 0xc4ac5665u,
+    // Round 4
+    0xf4292244u, 0x432aff97u, 0xab9423a7u, 0xfc93a039u,
+    0x655b59c3u, 0x8f0ccc92u, 0xffeff47du, 0x85845dd1u,
+    0x6fa87e4fu, 0xfe2ce6e0u, 0xa3014314u, 0x4e0811a1u,
+    0xf7537e82u, 0xbd3af235u, 0x2ad7d2bbu, 0xeb86d391u,
+};
+
+constexpr uint8_t kMd5S[64] = {
+    7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22,
+    5,  9, 14, 20, 5,  9, 14, 20, 5,  9, 14, 20, 5,  9, 14, 20,
+    4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23,
+    6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21,
+};
+// clang-format on
+
+struct Md5State {
+    uint32_t a{0x67452301u};
+    uint32_t b{0xEFCDAB89u};
+    uint32_t c{0x98BADCFEu};
+    uint32_t d{0x10325476u};
+    uint64_t count{0};
+    uint8_t buf[64]{};
+};
+
+static uint32_t RotLeft32(uint32_t x, uint8_t n) {
+    return (x << n) | (x >> (32u - n));
+}
+
+static void Md5Block(Md5State &s, const uint8_t *blk) {
+    uint32_t w[16];
+    // clang-format off
+    for (int i = 0; i < 16; ++i) {
+        w[i] =  static_cast<uint32_t>(blk[4 * i])
+             | (static_cast<uint32_t>(blk[4 * i + 1]) << 8u)
+             | (static_cast<uint32_t>(blk[4 * i + 2]) << 16u)
+             | (static_cast<uint32_t>(blk[4 * i + 3]) << 24u);
+    }
+    // clang-format on
+    uint32_t a = s.a, b = s.b, c = s.c, d = s.d;
+    for (int i = 0; i < 64; ++i) {
+        uint32_t f = 0;
+        uint32_t g = 0;
+        if (i < 16) {
+            f = (b & c) | (~b & d);
+            g = static_cast<uint32_t>(i);
+        } else if (i < 32) {
+            f = (d & b) | (~d & c);
+            g = static_cast<uint32_t>(5 * i + 1) % 16u;
+        } else if (i < 48) {
+            f = b ^ c ^ d;
+            g = static_cast<uint32_t>(3 * i + 5) % 16u;
+        } else {
+            f = c ^ (b | ~d);
+            g = static_cast<uint32_t>(7 * i) % 16u;
+        }
+        f += a + kMd5K[i] + w[g];
+        a = d;
+        d = c;
+        c = b;
+        b += RotLeft32(f, kMd5S[i]);
+    }
+    s.a += a;
+    s.b += b;
+    s.c += c;
+    s.d += d;
+}
+
+static void Md5Update(Md5State &s, const uint8_t *data, size_t len) {
+    uint32_t offset = s.count & 63u;
+    s.count += len;
+    while (len > 0) {
+        size_t chunk = std::min(len, static_cast<size_t>(64u - offset));
+        std::memcpy(s.buf + offset, data, chunk);
+        data += chunk;
+        len -= chunk;
+        offset += static_cast<uint32_t>(chunk);
+        if (offset == 64u) {
+            Md5Block(s, s.buf);
+            offset = 0;
+        }
+    }
+}
+
+static void Md5Final(Md5State &s, uint8_t *digest) {
+    const uint64_t bits = s.count * 8u;
+    uint8_t pad = 0x80u;
+    Md5Update(s, &pad, 1);
+    pad = 0;
+    while ((s.count & 63u) != 56u)
+        Md5Update(s, &pad, 1);
+    uint8_t bits_le[8];
+    for (int i = 0; i < 8; ++i)
+        bits_le[i] = static_cast<uint8_t>(bits >> (8u * static_cast<unsigned>(i)));
+    Md5Update(s, bits_le, 8);
+    const uint32_t parts[4] = {s.a, s.b, s.c, s.d};
+    for (int i = 0; i < 4; ++i) {
+        digest[4 * i] = static_cast<uint8_t>(parts[i]);
+        digest[4 * i + 1] = static_cast<uint8_t>(parts[i] >> 8u);
+        digest[4 * i + 2] = static_cast<uint8_t>(parts[i] >> 16u);
+        digest[4 * i + 3] = static_cast<uint8_t>(parts[i] >> 24u);
+    }
+}
+
+} // namespace
+
+bool ComputeFileMd5(const fs::path &path, uint8_t *md5_out) {
+    std::ifstream f(path, std::ios::binary);
+    if (!f) {
+        return false;
+    }
+    Md5State state;
+    uint8_t buf[65536];
+    while (f.read(reinterpret_cast<char *>(buf), sizeof(buf)) || f.gcount() > 0)
+        Md5Update(state, buf, static_cast<size_t>(f.gcount()));
+    Md5Final(state, md5_out);
+    return true;
+}
+
+// ---- Timestamp ----------------------------------------------------------
+// Returns the file's last-modification time as a Windows FILETIME value
+// (100-nanosecond intervals since 1601-01-01 UTC).  Returns 0 on error.
+
+uint64_t LocalFileTimestamp(const fs::path &path) {
+    struct stat st {};
+    if (stat(path.string().c_str(), &st) != 0) {
+        return 0;
+    }
+    constexpr int64_t kEpochDiff = 11644473600LL;
+    return static_cast<uint64_t>((static_cast<int64_t>(st.st_mtime) + kEpochDiff) * 10000000LL);
+}

src/helpers.h

@@ -1,7 +1,9 @@
 #ifndef HELPERS_H
 #define HELPERS_H
 
+#include <cstdint>
 #include <filesystem>
+#include <optional>
 #include <string>
 
 namespace fs = std::filesystem;
@@ -14,4 +16,10 @@ uint32_t CalculateMpqMaxFileValue(const std::string &path);
 uint32_t NextPowerOfTwo(uint32_t n);
 void PrintAsBinary(const char *buffer, uint32_t size);
 
+// Local file checksum / timestamp helpers used by the --update logic.
+// Each returns std::nullopt / false / 0 if the file cannot be read.
+std::optional<uint32_t> ComputeFileCrc32(const fs::path &path);
+bool ComputeFileMd5(const fs::path &path, uint8_t *md5_out);
+uint64_t LocalFileTimestamp(const fs::path &path);
+
 #endif

src/main.cpp

@@ -165,8 +165,9 @@ int main(int argc, char **argv) {
                     "Archive path for a single file, or prefix for a directory");
     add->add_flag("-w,--overwrite", add_overwrite,
                   "Overwrite file if it already is in MPQ archive");
-    add->add_flag("-u,--update", add_update,
-                  "Skip files whose archived size matches the on-disk size (directory add only)");
+    add->add_flag(
+        "-u,--update", add_update,
+        "Skip unchanged files when adding a directory. Compares size, then MD5/CRC32/timestamp");
     add->add_option("--locale", base_locale, "Locale to use for added file")->check(locale_valid);
     add->add_option("-g,--game", base_game_profile,
                     "Game profile for compression rules. Valid options:\n" +

src/mpq.cpp

@@ -1,6 +1,7 @@
 #include "mpq.h"
 
 #include <algorithm>
+#include <cstring>
 #include <filesystem>
 #include <fstream>
 #include <functional>
@@ -230,11 +231,56 @@ int AddFiles(HANDLE archive, const std::string &input_path, const std::string &p
                 int32_t file_locale = GetFileInfo<int32_t>(file, SFileInfoLocale);
                 if (file_locale == locale) {
                     DWORD archived_size = SFileGetFileSize(file, nullptr);
-                    SFileCloseFile(file);
                     uintmax_t disk_size = fs::file_size(entry.path());
+                    bool skip = false;
+                    std::string skip_reason;
                     if (disk_size == static_cast<uintmax_t>(archived_size)) {
-                        std::cout << "[~] Skipping unchanged file: " << archive_file_path
-                                  << std::endl;
+                        const DWORD attr_flags = SFileGetAttributes(archive);
+                        if (attr_flags & MPQ_ATTRIBUTE_MD5) {
+                            // Retrieve the MD5 stored in (attributes) via TFileEntry.
+                            // Buffer must accommodate the struct plus the trailing filename.
+                            constexpr DWORD kEntryBufSize = sizeof(TFileEntry) + 1024;
+                            uint8_t fe_buf[kEntryBufSize]{};
+                            if (SFileGetFileInfo(file, SFileInfoFileEntry, fe_buf, kEntryBufSize,
+                                                 nullptr)) {
+                                const auto *fe = reinterpret_cast<const TFileEntry *>(fe_buf);
+                                const uint8_t zero_md5[MD5_DIGEST_SIZE]{};
+                                if (std::memcmp(fe->md5, zero_md5, MD5_DIGEST_SIZE) != 0) {
+                                    uint8_t local_md5[MD5_DIGEST_SIZE]{};
+                                    if (ComputeFileMd5(entry.path(), local_md5)) {
+                                        skip =
+                                            (std::memcmp(local_md5, fe->md5, MD5_DIGEST_SIZE) == 0);
+                                        if (skip)
+                                            skip_reason = "MD5 matches";
+                                    }
+                                }
+                            }
+                        } else if (attr_flags & MPQ_ATTRIBUTE_CRC32) {
+                            const DWORD archived_crc32 = GetFileInfo<DWORD>(file, SFileInfoCRC32);
+                            if (archived_crc32 != 0) {
+                                if (auto local_crc32 = ComputeFileCrc32(entry.path())) {
+                                    skip = (*local_crc32 == archived_crc32);
+                                    if (skip)
+                                        skip_reason = "CRC32 matches";
+                                }
+                            }
+                        } else if (attr_flags & MPQ_ATTRIBUTE_FILETIME) {
+                            const uint64_t archived_time =
+                                GetFileInfo<uint64_t>(file, SFileInfoFileTime);
+                            const uint64_t local_time = LocalFileTimestamp(entry.path());
+                            // Compare at second resolution: stat() has only second precision.
+                            if (archived_time != 0 && local_time != 0) {
+                                skip = (archived_time / 10000000u == local_time / 10000000u);
+                                if (skip)
+                                    skip_reason = "Timestamp matches";
+                            }
+                        }
+                        // If no attributes are present, always add the file.
+                    }
+                    SFileCloseFile(file);
+                    if (skip) {
+                        std::cout << "[~] Skipping unchanged file: " << archive_file_path << " ("
+                                  << skip_reason << ")" << std::endl;
                         files_skipped++;
                         continue;
                     }