From f4b99eddb615a55dc0b644b20b1a75f8c555af56 Mon Sep 17 00:00:00 2001 From: Mogileswara Rao Garimidi Date: Wed, 27 May 2026 23:21:47 +0530 Subject: [PATCH 1/7] Add EVT_DROPPED reason codes via param2 --- lib/api/Logger.cpp | 4 +++- lib/include/public/Enums.hpp | 4 +++- lib/include/public/Version.hpp | 6 +++--- lib/offline/OfflineStorage_Room.cpp | 1 + lib/offline/OfflineStorage_SQLite.cpp | 1 + lib/offline/StorageObserver.cpp | 2 ++ lib/stats/Statistics.cpp | 1 + lib/system/TelemetrySystem.cpp | 12 +++++++++++- 8 files changed, 25 insertions(+), 6 deletions(-) diff --git a/lib/api/Logger.cpp b/lib/api/Logger.cpp index 54d883664..aec4b9e52 100644 --- a/lib/api/Logger.cpp +++ b/lib/api/Logger.cpp @@ -575,7 +575,9 @@ namespace MAT_NS_BEGIN if (latency == EventLatency_Off) { - DispatchEvent(DebugEventType::EVT_DROPPED); + // Event dropped because the active TransmitProfile has disabled this + // latency tier. param1 = 1 record dropped; param2 = reason code. + DispatchEvent(DebugEvent(DebugEventType::EVT_DROPPED, 1u, static_cast(DROPPED_REASON_LATENCY_DISABLED_BY_PROFILE))); LOG_INFO("Event %s/%s dropped: calculated latency 0 (Off)", tenantTokenToId(m_tenantToken).c_str(), record.baseType.c_str()); return; diff --git a/lib/include/public/Enums.hpp b/lib/include/public/Enums.hpp index 49e081a40..60ecfaadc 100644 --- a/lib/include/public/Enums.hpp +++ b/lib/include/public/Enums.hpp @@ -565,7 +565,9 @@ namespace MAT_NS_BEGIN DROPPED_REASON_SERVER_DECLINED_5XX, DROPPED_REASON_SERVER_DECLINED_OTHER, DROPPED_REASON_RETRY_EXCEEDED, - DROPPED_REASON_COUNT + DROPPED_REASON_COUNT, + DROPPED_REASON_TEARDOWN_TIMEOUT, + DROPPED_REASON_LATENCY_DISABLED_BY_PROFILE }; constexpr const static unsigned gc_NumDroppedReasons = DROPPED_REASON_COUNT; diff --git a/lib/include/public/Version.hpp b/lib/include/public/Version.hpp index 850519486..0f4f002cd 100644 --- a/lib/include/public/Version.hpp +++ b/lib/include/public/Version.hpp @@ -6,8 +6,8 @@ #define MAT_VERSION_HPP // WARNING: DO NOT MODIFY THIS FILE! // This file has been automatically generated, manual changes will be lost. -#define BUILD_VERSION_STR "3.10.100.1" -#define BUILD_VERSION 3,10,100,1 +#define BUILD_VERSION_STR "3.10.147.1" +#define BUILD_VERSION 3,10,147,1 #ifndef RESOURCE_COMPILER_INVOKED #include "ctmacros.hpp" @@ -18,7 +18,7 @@ namespace MAT_NS_BEGIN { uint64_t const Version = ((uint64_t)3 << 48) | ((uint64_t)10 << 32) | - ((uint64_t)100 << 16) | + ((uint64_t)147 << 16) | ((uint64_t)1); } MAT_NS_END diff --git a/lib/offline/OfflineStorage_Room.cpp b/lib/offline/OfflineStorage_Room.cpp index 423aecde3..72a04d0ed 100644 --- a/lib/offline/OfflineStorage_Room.cpp +++ b/lib/offline/OfflineStorage_Room.cpp @@ -1169,6 +1169,7 @@ namespace MAT_NS_BEGIN DebugEvent evt(DebugEventType::EVT_DROPPED); evt.param1 = dropped; + evt.param2 = static_cast(DROPPED_REASON_OFFLINE_STORAGE_OVERFLOW); evt.size = dropped; m_manager.DispatchEvent(evt); diff --git a/lib/offline/OfflineStorage_SQLite.cpp b/lib/offline/OfflineStorage_SQLite.cpp index 9b10ebfa2..022feb18a 100644 --- a/lib/offline/OfflineStorage_SQLite.cpp +++ b/lib/offline/OfflineStorage_SQLite.cpp @@ -972,6 +972,7 @@ namespace MAT_NS_BEGIN { m_DbSizeEstimate = GetSize(); DebugEvent evt(DebugEventType::EVT_DROPPED); evt.param1 = eventsDropped; + evt.param2 = static_cast(DROPPED_REASON_OFFLINE_STORAGE_OVERFLOW); evt.size = eventsDropped; m_logManager.DispatchEvent(evt); diff --git a/lib/offline/StorageObserver.cpp b/lib/offline/StorageObserver.cpp index e81ebc66a..a25e5456d 100644 --- a/lib/offline/StorageObserver.cpp +++ b/lib/offline/StorageObserver.cpp @@ -160,6 +160,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = EVT_DROPPED; evt.param1 = overallCount; + evt.param2 = static_cast(DROPPED_REASON_OFFLINE_STORAGE_OVERFLOW); evt.size = overallCount; DispatchEvent(evt); } @@ -180,6 +181,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = EVT_DROPPED; evt.param1 = overallCount; + evt.param2 = static_cast(DROPPED_REASON_OFFLINE_STORAGE_OVERFLOW); evt.size = overallCount; DispatchEvent(evt); } diff --git a/lib/stats/Statistics.cpp b/lib/stats/Statistics.cpp index 4abf09563..773dd4d13 100644 --- a/lib/stats/Statistics.cpp +++ b/lib/stats/Statistics.cpp @@ -160,6 +160,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = DebugEventType::EVT_DROPPED; evt.param1 = 1; + evt.param2 = static_cast(DROPPED_REASON_OFFLINE_STORAGE_SAVE_FAILED); OnDebugEvent(evt); return true; diff --git a/lib/system/TelemetrySystem.cpp b/lib/system/TelemetrySystem.cpp index 119d3f5e1..4cb9c3b33 100644 --- a/lib/system/TelemetrySystem.cpp +++ b/lib/system/TelemetrySystem.cpp @@ -76,7 +76,17 @@ namespace MAT_NS_BEGIN { auto uploadTime = GetUptimeMs() - stopTimes[0]; if (uploadTime >= (1000L * timeoutInSec)) { - // Hard-stop if it takes longer than planned + // Hard-stop if it takes longer than planned. + // Emit an EVT_DROPPED carrying the remaining record count + // so listeners can attribute teardown-timeout losses. + const size_t remaining = storage.GetRecordCount(); + if (remaining > 0) + { + DebugEvent evt(DebugEventType::EVT_DROPPED, remaining, + static_cast(DROPPED_REASON_TEARDOWN_TIMEOUT)); + evt.size = remaining; + m_logManager.DispatchEvent(evt); + } LOG_TRACE("Shutdown timer expired, exiting..."); break; } From d61bf8c585b1d57e300ca7291af47fb288cb04c3 Mon Sep 17 00:00:00 2001 From: Mogileswara Rao Garimidi Date: Fri, 29 May 2026 15:49:12 +0530 Subject: [PATCH 2/7] Improved errocodes with review comments --- lib/include/public/Enums.hpp | 4 ++-- lib/offline/OfflineStorage_SQLite.cpp | 22 ++++++++++++---------- lib/offline/StorageObserver.cpp | 2 +- lib/system/TelemetrySystem.cpp | 4 +++- lib/tpm/TransmissionPolicyManager.cpp | 14 ++++++++++++++ lib/tpm/TransmissionPolicyManager.hpp | 5 +++++ 6 files changed, 37 insertions(+), 14 deletions(-) diff --git a/lib/include/public/Enums.hpp b/lib/include/public/Enums.hpp index 60ecfaadc..0992e7602 100644 --- a/lib/include/public/Enums.hpp +++ b/lib/include/public/Enums.hpp @@ -565,9 +565,9 @@ namespace MAT_NS_BEGIN DROPPED_REASON_SERVER_DECLINED_5XX, DROPPED_REASON_SERVER_DECLINED_OTHER, DROPPED_REASON_RETRY_EXCEEDED, - DROPPED_REASON_COUNT, DROPPED_REASON_TEARDOWN_TIMEOUT, - DROPPED_REASON_LATENCY_DISABLED_BY_PROFILE + DROPPED_REASON_LATENCY_DISABLED_BY_PROFILE, + DROPPED_REASON_COUNT }; constexpr const static unsigned gc_NumDroppedReasons = DROPPED_REASON_COUNT; diff --git a/lib/offline/OfflineStorage_SQLite.cpp b/lib/offline/OfflineStorage_SQLite.cpp index 022feb18a..0a175cbd9 100644 --- a/lib/offline/OfflineStorage_SQLite.cpp +++ b/lib/offline/OfflineStorage_SQLite.cpp @@ -954,19 +954,21 @@ namespace MAT_NS_BEGIN { LOG_TRACE("DB is too big, deleting..."); Execute("DELETE FROM " TABLE_NAME_EVENTS); Execute("VACUUM"); - return true; + eventsDropped = count; } - - SqliteStatement trimStmt(*m_db, m_stmtTrimEvents_percent); - if (!trimStmt.execute(25)) + else { - // If something went wrong with trimming 25%, try more radical measure - LOG_TRACE("Evict all non-critical"); - Execute("DELETE FROM " TABLE_NAME_EVENTS " WHERE persistence=1"); + SqliteStatement trimStmt(*m_db, m_stmtTrimEvents_percent); + if (!trimStmt.execute(25)) + { + // If something went wrong with trimming 25%, try more radical measure + LOG_TRACE("Evict all non-critical"); + Execute("DELETE FROM " TABLE_NAME_EVENTS " WHERE persistence=1"); + } + eventsDropped = count - GetRecordCountUnsafe(EventLatency::EventLatency_Unspecified); + LOG_TRACE("Db resized, events dropeed: %d", eventsDropped); + trimStmt.reset(); } - eventsDropped = count - GetRecordCountUnsafe(EventLatency::EventLatency_Unspecified); - LOG_TRACE("Db resized, events dropeed: %d", eventsDropped); - trimStmt.reset(); } m_DbSizeEstimate = GetSize(); diff --git a/lib/offline/StorageObserver.cpp b/lib/offline/StorageObserver.cpp index a25e5456d..e0bcc341e 100644 --- a/lib/offline/StorageObserver.cpp +++ b/lib/offline/StorageObserver.cpp @@ -181,7 +181,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = EVT_DROPPED; evt.param1 = overallCount; - evt.param2 = static_cast(DROPPED_REASON_OFFLINE_STORAGE_OVERFLOW); + evt.param2 = static_cast(DROPPED_REASON_RETRY_EXCEEDED); evt.size = overallCount; DispatchEvent(evt); } diff --git a/lib/system/TelemetrySystem.cpp b/lib/system/TelemetrySystem.cpp index 4cb9c3b33..5589f4213 100644 --- a/lib/system/TelemetrySystem.cpp +++ b/lib/system/TelemetrySystem.cpp @@ -79,7 +79,9 @@ namespace MAT_NS_BEGIN { // Hard-stop if it takes longer than planned. // Emit an EVT_DROPPED carrying the remaining record count // so listeners can attribute teardown-timeout losses. - const size_t remaining = storage.GetRecordCount(); + // Include in-flight upload records too: GetRecordCount() can be 0 + // during teardown while records are still being uploaded. + const size_t remaining = storage.GetRecordCount() + tpm.inflightRecordCount(); if (remaining > 0) { DebugEvent evt(DebugEventType::EVT_DROPPED, remaining, diff --git a/lib/tpm/TransmissionPolicyManager.cpp b/lib/tpm/TransmissionPolicyManager.cpp index 83b82cf2a..467afa47a 100644 --- a/lib/tpm/TransmissionPolicyManager.cpp +++ b/lib/tpm/TransmissionPolicyManager.cpp @@ -479,6 +479,20 @@ namespace MAT_NS_BEGIN { return (uploadCount() > 0) || m_isUploadScheduled; } + size_t TransmissionPolicyManager::inflightRecordCount() const noexcept + { + LOCKGUARD(m_activeUploads_lock); + size_t total = 0; + for (const auto& ctx : m_activeUploads) + { + if (ctx) + { + total += ctx->recordIdsAndTenantIds.size(); + } + } + return total; + } + bool TransmissionPolicyManager::isPaused() const noexcept { return m_isPaused; diff --git a/lib/tpm/TransmissionPolicyManager.hpp b/lib/tpm/TransmissionPolicyManager.hpp index e1a91ad10..602e914ea 100644 --- a/lib/tpm/TransmissionPolicyManager.hpp +++ b/lib/tpm/TransmissionPolicyManager.hpp @@ -154,6 +154,11 @@ constexpr const char* const DefaultBackoffConfig = "E,3000,300000,2,1"; virtual bool isUploadInProgress() const noexcept; + /// + /// Returns the total number of records currently in flight across all active uploads. + /// + virtual size_t inflightRecordCount() const noexcept; + virtual bool isPaused() const noexcept; }; From e8dd7d97cf7c728a1d131ea496abc0a46d7bf998 Mon Sep 17 00:00:00 2001 From: Mogileswara Rao Garimidi Date: Sat, 30 May 2026 00:31:59 +0530 Subject: [PATCH 3/7] Improved errocodes with review comments --- lib/include/public/IOfflineStorage.hpp | 12 ++++++++++++ lib/include/public/Version.hpp | 6 +++--- lib/offline/MemoryStorage.cpp | 9 +++++++++ lib/offline/MemoryStorage.hpp | 4 +++- lib/offline/OfflineStorageHandler.cpp | 10 ++++++++++ lib/offline/OfflineStorageHandler.hpp | 2 ++ lib/offline/StorageObserver.hpp | 5 +++++ lib/system/TelemetrySystem.cpp | 7 ++++--- lib/tpm/TransmissionPolicyManager.cpp | 14 -------------- lib/tpm/TransmissionPolicyManager.hpp | 5 ----- 10 files changed, 48 insertions(+), 26 deletions(-) diff --git a/lib/include/public/IOfflineStorage.hpp b/lib/include/public/IOfflineStorage.hpp index 79f076fde..85c2b4655 100644 --- a/lib/include/public/IOfflineStorage.hpp +++ b/lib/include/public/IOfflineStorage.hpp @@ -329,6 +329,18 @@ namespace MAT_NS_BEGIN { /// Number of records virtual size_t GetRecordCount(EventLatency latency = EventLatency_Unspecified) const = 0; + /// + /// Returns the total number of records that should be considered as + /// "still pending" at shutdown for teardown-timeout reporting. + /// + /// + /// Default implementation returns GetRecordCount(), which is correct for + /// persistent storages (e.g. SQLite, Room) where in-flight records are + /// still counted in the on-disk total. Volatile storages (e.g. memory) + /// must override to also include their in-flight / reserved records. + /// + virtual size_t GetRemainingRecordCountForShutdown() const { return GetRecordCount(); } + /// /// Get Vector of records from DB /// diff --git a/lib/include/public/Version.hpp b/lib/include/public/Version.hpp index 0f4f002cd..850519486 100644 --- a/lib/include/public/Version.hpp +++ b/lib/include/public/Version.hpp @@ -6,8 +6,8 @@ #define MAT_VERSION_HPP // WARNING: DO NOT MODIFY THIS FILE! // This file has been automatically generated, manual changes will be lost. -#define BUILD_VERSION_STR "3.10.147.1" -#define BUILD_VERSION 3,10,147,1 +#define BUILD_VERSION_STR "3.10.100.1" +#define BUILD_VERSION 3,10,100,1 #ifndef RESOURCE_COMPILER_INVOKED #include "ctmacros.hpp" @@ -18,7 +18,7 @@ namespace MAT_NS_BEGIN { uint64_t const Version = ((uint64_t)3 << 48) | ((uint64_t)10 << 32) | - ((uint64_t)147 << 16) | + ((uint64_t)100 << 16) | ((uint64_t)1); } MAT_NS_END diff --git a/lib/offline/MemoryStorage.cpp b/lib/offline/MemoryStorage.cpp index 09142f8b4..7cb1686fd 100644 --- a/lib/offline/MemoryStorage.cpp +++ b/lib/offline/MemoryStorage.cpp @@ -520,5 +520,14 @@ namespace MAT_NS_BEGIN { return m_reserved_records.size(); } + /// + /// Memory storage does not include in-flight (reserved) records in + /// GetRecordCount(), so add them here for accurate shutdown reporting. + /// + size_t MemoryStorage::GetRemainingRecordCountForShutdown() const + { + return GetRecordCount() + GetReservedCount(); + } + } MAT_NS_END diff --git a/lib/offline/MemoryStorage.hpp b/lib/offline/MemoryStorage.hpp index b94cfb1fd..8a378dc5d 100644 --- a/lib/offline/MemoryStorage.hpp +++ b/lib/offline/MemoryStorage.hpp @@ -67,6 +67,8 @@ namespace MAT_NS_BEGIN { virtual size_t GetRecordCount(EventLatency latency = EventLatency_Unspecified) const override; + virtual size_t GetRemainingRecordCountForShutdown() const override; + virtual size_t GetReservedCount(); virtual std::vector GetRecords(bool shutdown = false, EventLatency minLatency = EventLatency_Unspecified, unsigned maxCount = 0) override; @@ -88,7 +90,7 @@ namespace MAT_NS_BEGIN { /// Contains reserved (aka in-flight) records. /// Current storage interface API requires deletion and release by StorageRecordId. /// - std::mutex m_reserved_lock; + mutable std::mutex m_reserved_lock; std::map m_reserved_records; size_t m_size; diff --git a/lib/offline/OfflineStorageHandler.cpp b/lib/offline/OfflineStorageHandler.cpp index 41b211d97..9049339c4 100644 --- a/lib/offline/OfflineStorageHandler.cpp +++ b/lib/offline/OfflineStorageHandler.cpp @@ -150,6 +150,16 @@ namespace MAT_NS_BEGIN { return count; } + size_t OfflineStorageHandler::GetRemainingRecordCountForShutdown() const + { + size_t count = 0; + if (m_offlineStorageMemory != nullptr) + count += m_offlineStorageMemory->GetRemainingRecordCountForShutdown(); + if (m_offlineStorageDisk != nullptr) + count += m_offlineStorageDisk->GetRemainingRecordCountForShutdown(); + return count; + } + void OfflineStorageHandler::Flush() { if (!m_logManager.StartActivity()) { diff --git a/lib/offline/OfflineStorageHandler.hpp b/lib/offline/OfflineStorageHandler.hpp index eb17ac505..e7bdce4cb 100644 --- a/lib/offline/OfflineStorageHandler.hpp +++ b/lib/offline/OfflineStorageHandler.hpp @@ -50,6 +50,8 @@ namespace MAT_NS_BEGIN { virtual size_t GetSize() override; virtual size_t GetRecordCount(EventLatency latency = EventLatency_Unspecified) const override; + virtual size_t GetRemainingRecordCountForShutdown() const override; + virtual std::vector GetRecords(bool shutdown, EventLatency minLatency = EventLatency_Unspecified, unsigned maxCount = 0) override; virtual bool ResizeDb() override; diff --git a/lib/offline/StorageObserver.hpp b/lib/offline/StorageObserver.hpp index 58f813972..4b10f1c77 100644 --- a/lib/offline/StorageObserver.hpp +++ b/lib/offline/StorageObserver.hpp @@ -62,6 +62,11 @@ namespace MAT_NS_BEGIN { return m_offlineStorage.GetRecordCount(); } + size_t GetRemainingRecordCountForShutdown() const + { + return m_offlineStorage.GetRemainingRecordCountForShutdown(); + } + RoutePassThrough start{ this, &StorageObserver::handleStart }; RoutePassThrough stop{ this, &StorageObserver::handleStop }; diff --git a/lib/system/TelemetrySystem.cpp b/lib/system/TelemetrySystem.cpp index 5589f4213..24ad34ba9 100644 --- a/lib/system/TelemetrySystem.cpp +++ b/lib/system/TelemetrySystem.cpp @@ -79,9 +79,10 @@ namespace MAT_NS_BEGIN { // Hard-stop if it takes longer than planned. // Emit an EVT_DROPPED carrying the remaining record count // so listeners can attribute teardown-timeout losses. - // Include in-flight upload records too: GetRecordCount() can be 0 - // during teardown while records are still being uploaded. - const size_t remaining = storage.GetRecordCount() + tpm.inflightRecordCount(); + // Delegate the "remaining" calculation to the storage layer + // so each backend (memory vs SQLite/Room) can decide whether + // in-flight records need to be added on top of GetRecordCount(). + const size_t remaining = storage.GetRemainingRecordCountForShutdown(); if (remaining > 0) { DebugEvent evt(DebugEventType::EVT_DROPPED, remaining, diff --git a/lib/tpm/TransmissionPolicyManager.cpp b/lib/tpm/TransmissionPolicyManager.cpp index 467afa47a..83b82cf2a 100644 --- a/lib/tpm/TransmissionPolicyManager.cpp +++ b/lib/tpm/TransmissionPolicyManager.cpp @@ -479,20 +479,6 @@ namespace MAT_NS_BEGIN { return (uploadCount() > 0) || m_isUploadScheduled; } - size_t TransmissionPolicyManager::inflightRecordCount() const noexcept - { - LOCKGUARD(m_activeUploads_lock); - size_t total = 0; - for (const auto& ctx : m_activeUploads) - { - if (ctx) - { - total += ctx->recordIdsAndTenantIds.size(); - } - } - return total; - } - bool TransmissionPolicyManager::isPaused() const noexcept { return m_isPaused; diff --git a/lib/tpm/TransmissionPolicyManager.hpp b/lib/tpm/TransmissionPolicyManager.hpp index 602e914ea..e1a91ad10 100644 --- a/lib/tpm/TransmissionPolicyManager.hpp +++ b/lib/tpm/TransmissionPolicyManager.hpp @@ -154,11 +154,6 @@ constexpr const char* const DefaultBackoffConfig = "E,3000,300000,2,1"; virtual bool isUploadInProgress() const noexcept; - /// - /// Returns the total number of records currently in flight across all active uploads. - /// - virtual size_t inflightRecordCount() const noexcept; - virtual bool isPaused() const noexcept; }; From 96b17a80713939bd096f46ca7870d54d3e938a4d Mon Sep 17 00:00:00 2001 From: Mogileswara Rao Garimidi Date: Sat, 30 May 2026 01:00:01 +0530 Subject: [PATCH 4/7] Improved errocodes with review comments --- lib/include/public/IOfflineStorage.hpp | 2 +- lib/offline/MemoryStorage.cpp | 2 +- lib/offline/MemoryStorage.hpp | 4 ++-- lib/offline/OfflineStorageHandler.cpp | 2 +- lib/offline/OfflineStorageHandler.hpp | 2 +- lib/offline/StorageObserver.hpp | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/include/public/IOfflineStorage.hpp b/lib/include/public/IOfflineStorage.hpp index 85c2b4655..32bd91e4a 100644 --- a/lib/include/public/IOfflineStorage.hpp +++ b/lib/include/public/IOfflineStorage.hpp @@ -339,7 +339,7 @@ namespace MAT_NS_BEGIN { /// still counted in the on-disk total. Volatile storages (e.g. memory) /// must override to also include their in-flight / reserved records. /// - virtual size_t GetRemainingRecordCountForShutdown() const { return GetRecordCount(); } + virtual size_t GetRemainingRecordCountForShutdown() { return GetRecordCount(); } /// /// Get Vector of records from DB diff --git a/lib/offline/MemoryStorage.cpp b/lib/offline/MemoryStorage.cpp index 7cb1686fd..5887de3fa 100644 --- a/lib/offline/MemoryStorage.cpp +++ b/lib/offline/MemoryStorage.cpp @@ -524,7 +524,7 @@ namespace MAT_NS_BEGIN { /// Memory storage does not include in-flight (reserved) records in /// GetRecordCount(), so add them here for accurate shutdown reporting. /// - size_t MemoryStorage::GetRemainingRecordCountForShutdown() const + size_t MemoryStorage::GetRemainingRecordCountForShutdown() { return GetRecordCount() + GetReservedCount(); } diff --git a/lib/offline/MemoryStorage.hpp b/lib/offline/MemoryStorage.hpp index 8a378dc5d..4ffc96c1e 100644 --- a/lib/offline/MemoryStorage.hpp +++ b/lib/offline/MemoryStorage.hpp @@ -67,7 +67,7 @@ namespace MAT_NS_BEGIN { virtual size_t GetRecordCount(EventLatency latency = EventLatency_Unspecified) const override; - virtual size_t GetRemainingRecordCountForShutdown() const override; + virtual size_t GetRemainingRecordCountForShutdown() override; virtual size_t GetReservedCount(); @@ -90,7 +90,7 @@ namespace MAT_NS_BEGIN { /// Contains reserved (aka in-flight) records. /// Current storage interface API requires deletion and release by StorageRecordId. /// - mutable std::mutex m_reserved_lock; + std::mutex m_reserved_lock; std::map m_reserved_records; size_t m_size; diff --git a/lib/offline/OfflineStorageHandler.cpp b/lib/offline/OfflineStorageHandler.cpp index 9049339c4..d7759d538 100644 --- a/lib/offline/OfflineStorageHandler.cpp +++ b/lib/offline/OfflineStorageHandler.cpp @@ -150,7 +150,7 @@ namespace MAT_NS_BEGIN { return count; } - size_t OfflineStorageHandler::GetRemainingRecordCountForShutdown() const + size_t OfflineStorageHandler::GetRemainingRecordCountForShutdown() { size_t count = 0; if (m_offlineStorageMemory != nullptr) diff --git a/lib/offline/OfflineStorageHandler.hpp b/lib/offline/OfflineStorageHandler.hpp index e7bdce4cb..f3791037c 100644 --- a/lib/offline/OfflineStorageHandler.hpp +++ b/lib/offline/OfflineStorageHandler.hpp @@ -50,7 +50,7 @@ namespace MAT_NS_BEGIN { virtual size_t GetSize() override; virtual size_t GetRecordCount(EventLatency latency = EventLatency_Unspecified) const override; - virtual size_t GetRemainingRecordCountForShutdown() const override; + virtual size_t GetRemainingRecordCountForShutdown() override; virtual std::vector GetRecords(bool shutdown, EventLatency minLatency = EventLatency_Unspecified, unsigned maxCount = 0) override; virtual bool ResizeDb() override; diff --git a/lib/offline/StorageObserver.hpp b/lib/offline/StorageObserver.hpp index 4b10f1c77..670de879a 100644 --- a/lib/offline/StorageObserver.hpp +++ b/lib/offline/StorageObserver.hpp @@ -62,7 +62,7 @@ namespace MAT_NS_BEGIN { return m_offlineStorage.GetRecordCount(); } - size_t GetRemainingRecordCountForShutdown() const + size_t GetRemainingRecordCountForShutdown() { return m_offlineStorage.GetRemainingRecordCountForShutdown(); } From fe445e5821a5777d4f78dd8d375e33022917666e Mon Sep 17 00:00:00 2001 From: mogiligarimidi23 Date: Sun, 31 May 2026 18:54:01 +0530 Subject: [PATCH 5/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- lib/offline/OfflineStorage_SQLite.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/offline/OfflineStorage_SQLite.cpp b/lib/offline/OfflineStorage_SQLite.cpp index 0a175cbd9..0ce28278e 100644 --- a/lib/offline/OfflineStorage_SQLite.cpp +++ b/lib/offline/OfflineStorage_SQLite.cpp @@ -966,7 +966,7 @@ namespace MAT_NS_BEGIN { Execute("DELETE FROM " TABLE_NAME_EVENTS " WHERE persistence=1"); } eventsDropped = count - GetRecordCountUnsafe(EventLatency::EventLatency_Unspecified); - LOG_TRACE("Db resized, events dropeed: %d", eventsDropped); + LOG_TRACE("Db resized, events dropped: %zu", eventsDropped); trimStmt.reset(); } } From df11940691ddeb534292cbd4a904b58678b70f87 Mon Sep 17 00:00:00 2001 From: Mogileswara Rao Garimidi Date: Sun, 31 May 2026 21:30:45 +0530 Subject: [PATCH 6/7] Make GetRemainingRecordCountForShutdown const and document ABI implication --- lib/include/public/IOfflineStorage.hpp | 2 +- lib/offline/MemoryStorage.cpp | 4 ++-- lib/offline/MemoryStorage.hpp | 6 +++--- lib/offline/OfflineStorageHandler.cpp | 2 +- lib/offline/OfflineStorageHandler.hpp | 2 +- lib/offline/StorageObserver.hpp | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/include/public/IOfflineStorage.hpp b/lib/include/public/IOfflineStorage.hpp index 32bd91e4a..85c2b4655 100644 --- a/lib/include/public/IOfflineStorage.hpp +++ b/lib/include/public/IOfflineStorage.hpp @@ -339,7 +339,7 @@ namespace MAT_NS_BEGIN { /// still counted in the on-disk total. Volatile storages (e.g. memory) /// must override to also include their in-flight / reserved records. /// - virtual size_t GetRemainingRecordCountForShutdown() { return GetRecordCount(); } + virtual size_t GetRemainingRecordCountForShutdown() const { return GetRecordCount(); } /// /// Get Vector of records from DB diff --git a/lib/offline/MemoryStorage.cpp b/lib/offline/MemoryStorage.cpp index 5887de3fa..801afcec0 100644 --- a/lib/offline/MemoryStorage.cpp +++ b/lib/offline/MemoryStorage.cpp @@ -514,7 +514,7 @@ namespace MAT_NS_BEGIN { /// This method is currently internal, but up for consideration to add it to common storage interface. /// /// - size_t MemoryStorage::GetReservedCount() + size_t MemoryStorage::GetReservedCount() const { LOCKGUARD(m_reserved_lock); return m_reserved_records.size(); @@ -524,7 +524,7 @@ namespace MAT_NS_BEGIN { /// Memory storage does not include in-flight (reserved) records in /// GetRecordCount(), so add them here for accurate shutdown reporting. /// - size_t MemoryStorage::GetRemainingRecordCountForShutdown() + size_t MemoryStorage::GetRemainingRecordCountForShutdown() const { return GetRecordCount() + GetReservedCount(); } diff --git a/lib/offline/MemoryStorage.hpp b/lib/offline/MemoryStorage.hpp index 4ffc96c1e..9475af3f7 100644 --- a/lib/offline/MemoryStorage.hpp +++ b/lib/offline/MemoryStorage.hpp @@ -67,9 +67,9 @@ namespace MAT_NS_BEGIN { virtual size_t GetRecordCount(EventLatency latency = EventLatency_Unspecified) const override; - virtual size_t GetRemainingRecordCountForShutdown() override; + virtual size_t GetRemainingRecordCountForShutdown() const override; - virtual size_t GetReservedCount(); + virtual size_t GetReservedCount() const; virtual std::vector GetRecords(bool shutdown = false, EventLatency minLatency = EventLatency_Unspecified, unsigned maxCount = 0) override; @@ -90,7 +90,7 @@ namespace MAT_NS_BEGIN { /// Contains reserved (aka in-flight) records. /// Current storage interface API requires deletion and release by StorageRecordId. /// - std::mutex m_reserved_lock; + mutable std::mutex m_reserved_lock; std::map m_reserved_records; size_t m_size; diff --git a/lib/offline/OfflineStorageHandler.cpp b/lib/offline/OfflineStorageHandler.cpp index d7759d538..9049339c4 100644 --- a/lib/offline/OfflineStorageHandler.cpp +++ b/lib/offline/OfflineStorageHandler.cpp @@ -150,7 +150,7 @@ namespace MAT_NS_BEGIN { return count; } - size_t OfflineStorageHandler::GetRemainingRecordCountForShutdown() + size_t OfflineStorageHandler::GetRemainingRecordCountForShutdown() const { size_t count = 0; if (m_offlineStorageMemory != nullptr) diff --git a/lib/offline/OfflineStorageHandler.hpp b/lib/offline/OfflineStorageHandler.hpp index f3791037c..e7bdce4cb 100644 --- a/lib/offline/OfflineStorageHandler.hpp +++ b/lib/offline/OfflineStorageHandler.hpp @@ -50,7 +50,7 @@ namespace MAT_NS_BEGIN { virtual size_t GetSize() override; virtual size_t GetRecordCount(EventLatency latency = EventLatency_Unspecified) const override; - virtual size_t GetRemainingRecordCountForShutdown() override; + virtual size_t GetRemainingRecordCountForShutdown() const override; virtual std::vector GetRecords(bool shutdown, EventLatency minLatency = EventLatency_Unspecified, unsigned maxCount = 0) override; virtual bool ResizeDb() override; diff --git a/lib/offline/StorageObserver.hpp b/lib/offline/StorageObserver.hpp index 670de879a..4b10f1c77 100644 --- a/lib/offline/StorageObserver.hpp +++ b/lib/offline/StorageObserver.hpp @@ -62,7 +62,7 @@ namespace MAT_NS_BEGIN { return m_offlineStorage.GetRecordCount(); } - size_t GetRemainingRecordCountForShutdown() + size_t GetRemainingRecordCountForShutdown() const { return m_offlineStorage.GetRemainingRecordCountForShutdown(); } From 4a42ee33d4765138c2656e33961c11af555af2fc Mon Sep 17 00:00:00 2001 From: Mogileswara Rao Garimidi Date: Sun, 31 May 2026 22:28:00 +0530 Subject: [PATCH 7/7] resolved review comments --- lib/offline/MemoryStorage.cpp | 16 ++++++++++++++-- lib/offline/MemoryStorage.hpp | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/offline/MemoryStorage.cpp b/lib/offline/MemoryStorage.cpp index 801afcec0..1d4ec5664 100644 --- a/lib/offline/MemoryStorage.cpp +++ b/lib/offline/MemoryStorage.cpp @@ -514,7 +514,7 @@ namespace MAT_NS_BEGIN { /// This method is currently internal, but up for consideration to add it to common storage interface. /// /// - size_t MemoryStorage::GetReservedCount() const + size_t MemoryStorage::GetReservedCount() { LOCKGUARD(m_reserved_lock); return m_reserved_records.size(); @@ -524,9 +524,21 @@ namespace MAT_NS_BEGIN { /// Memory storage does not include in-flight (reserved) records in /// GetRecordCount(), so add them here for accurate shutdown reporting. /// + /// + /// Take both locks (reserved first, then records — matching the order used + /// by Shutdown() and GetAndReserveRecords()) so the returned count is an + /// atomic snapshot. Without this, a record moving between the active queue + /// and the reserved map (e.g. via GetAndReserveRecords / ReleaseRecords) + /// could be double-counted or missed. + /// size_t MemoryStorage::GetRemainingRecordCountForShutdown() const { - return GetRecordCount() + GetReservedCount(); + LOCKGUARD(m_reserved_lock); + LOCKGUARD(m_records_lock); + size_t records = 0; + for (unsigned lat = EventLatency_Off; lat <= EventLatency_Max; lat++) + records += m_records[lat].size(); + return records + m_reserved_records.size(); } } MAT_NS_END diff --git a/lib/offline/MemoryStorage.hpp b/lib/offline/MemoryStorage.hpp index 9475af3f7..8a378dc5d 100644 --- a/lib/offline/MemoryStorage.hpp +++ b/lib/offline/MemoryStorage.hpp @@ -69,7 +69,7 @@ namespace MAT_NS_BEGIN { virtual size_t GetRemainingRecordCountForShutdown() const override; - virtual size_t GetReservedCount() const; + virtual size_t GetReservedCount(); virtual std::vector GetRecords(bool shutdown = false, EventLatency minLatency = EventLatency_Unspecified, unsigned maxCount = 0) override;