diff --git a/lib/api/Logger.cpp b/lib/api/Logger.cpp index 54d883664..aec4b9e52 100644 --- a/lib/api/Logger.cpp +++ b/lib/api/Logger.cpp @@ -575,7 +575,9 @@ namespace MAT_NS_BEGIN if (latency == EventLatency_Off) { - DispatchEvent(DebugEventType::EVT_DROPPED); + // Event dropped because the active TransmitProfile has disabled this + // latency tier. param1 = 1 record dropped; param2 = reason code. + DispatchEvent(DebugEvent(DebugEventType::EVT_DROPPED, 1u, static_cast(DROPPED_REASON_LATENCY_DISABLED_BY_PROFILE))); LOG_INFO("Event %s/%s dropped: calculated latency 0 (Off)", tenantTokenToId(m_tenantToken).c_str(), record.baseType.c_str()); return; diff --git a/lib/include/public/Enums.hpp b/lib/include/public/Enums.hpp index 49e081a40..0992e7602 100644 --- a/lib/include/public/Enums.hpp +++ b/lib/include/public/Enums.hpp @@ -565,6 +565,8 @@ namespace MAT_NS_BEGIN DROPPED_REASON_SERVER_DECLINED_5XX, DROPPED_REASON_SERVER_DECLINED_OTHER, DROPPED_REASON_RETRY_EXCEEDED, + DROPPED_REASON_TEARDOWN_TIMEOUT, + DROPPED_REASON_LATENCY_DISABLED_BY_PROFILE, DROPPED_REASON_COUNT }; diff --git a/lib/include/public/IOfflineStorage.hpp b/lib/include/public/IOfflineStorage.hpp index 79f076fde..85c2b4655 100644 --- a/lib/include/public/IOfflineStorage.hpp +++ b/lib/include/public/IOfflineStorage.hpp @@ -329,6 +329,18 @@ namespace MAT_NS_BEGIN { /// Number of records virtual size_t GetRecordCount(EventLatency latency = EventLatency_Unspecified) const = 0; + /// + /// Returns the total number of records that should be considered as + /// "still pending" at shutdown for teardown-timeout reporting. + /// + /// + /// Default implementation returns GetRecordCount(), which is correct for + /// persistent storages (e.g. SQLite, Room) where in-flight records are + /// still counted in the on-disk total. Volatile storages (e.g. memory) + /// must override to also include their in-flight / reserved records. + /// + virtual size_t GetRemainingRecordCountForShutdown() const { return GetRecordCount(); } + /// /// Get Vector of records from DB /// diff --git a/lib/offline/MemoryStorage.cpp b/lib/offline/MemoryStorage.cpp index 09142f8b4..1d4ec5664 100644 --- a/lib/offline/MemoryStorage.cpp +++ b/lib/offline/MemoryStorage.cpp @@ -520,5 +520,26 @@ namespace MAT_NS_BEGIN { return m_reserved_records.size(); } + /// + /// Memory storage does not include in-flight (reserved) records in + /// GetRecordCount(), so add them here for accurate shutdown reporting. + /// + /// + /// Take both locks (reserved first, then records — matching the order used + /// by Shutdown() and GetAndReserveRecords()) so the returned count is an + /// atomic snapshot. Without this, a record moving between the active queue + /// and the reserved map (e.g. via GetAndReserveRecords / ReleaseRecords) + /// could be double-counted or missed. + /// + size_t MemoryStorage::GetRemainingRecordCountForShutdown() const + { + LOCKGUARD(m_reserved_lock); + LOCKGUARD(m_records_lock); + size_t records = 0; + for (unsigned lat = EventLatency_Off; lat <= EventLatency_Max; lat++) + records += m_records[lat].size(); + return records + m_reserved_records.size(); + } + } MAT_NS_END diff --git a/lib/offline/MemoryStorage.hpp b/lib/offline/MemoryStorage.hpp index b94cfb1fd..8a378dc5d 100644 --- a/lib/offline/MemoryStorage.hpp +++ b/lib/offline/MemoryStorage.hpp @@ -67,6 +67,8 @@ namespace MAT_NS_BEGIN { virtual size_t GetRecordCount(EventLatency latency = EventLatency_Unspecified) const override; + virtual size_t GetRemainingRecordCountForShutdown() const override; + virtual size_t GetReservedCount(); virtual std::vector GetRecords(bool shutdown = false, EventLatency minLatency = EventLatency_Unspecified, unsigned maxCount = 0) override; @@ -88,7 +90,7 @@ namespace MAT_NS_BEGIN { /// Contains reserved (aka in-flight) records. /// Current storage interface API requires deletion and release by StorageRecordId. /// - std::mutex m_reserved_lock; + mutable std::mutex m_reserved_lock; std::map m_reserved_records; size_t m_size; diff --git a/lib/offline/OfflineStorageHandler.cpp b/lib/offline/OfflineStorageHandler.cpp index 41b211d97..9049339c4 100644 --- a/lib/offline/OfflineStorageHandler.cpp +++ b/lib/offline/OfflineStorageHandler.cpp @@ -150,6 +150,16 @@ namespace MAT_NS_BEGIN { return count; } + size_t OfflineStorageHandler::GetRemainingRecordCountForShutdown() const + { + size_t count = 0; + if (m_offlineStorageMemory != nullptr) + count += m_offlineStorageMemory->GetRemainingRecordCountForShutdown(); + if (m_offlineStorageDisk != nullptr) + count += m_offlineStorageDisk->GetRemainingRecordCountForShutdown(); + return count; + } + void OfflineStorageHandler::Flush() { if (!m_logManager.StartActivity()) { diff --git a/lib/offline/OfflineStorageHandler.hpp b/lib/offline/OfflineStorageHandler.hpp index eb17ac505..e7bdce4cb 100644 --- a/lib/offline/OfflineStorageHandler.hpp +++ b/lib/offline/OfflineStorageHandler.hpp @@ -50,6 +50,8 @@ namespace MAT_NS_BEGIN { virtual size_t GetSize() override; virtual size_t GetRecordCount(EventLatency latency = EventLatency_Unspecified) const override; + virtual size_t GetRemainingRecordCountForShutdown() const override; + virtual std::vector GetRecords(bool shutdown, EventLatency minLatency = EventLatency_Unspecified, unsigned maxCount = 0) override; virtual bool ResizeDb() override; diff --git a/lib/offline/OfflineStorage_Room.cpp b/lib/offline/OfflineStorage_Room.cpp index 423aecde3..72a04d0ed 100644 --- a/lib/offline/OfflineStorage_Room.cpp +++ b/lib/offline/OfflineStorage_Room.cpp @@ -1169,6 +1169,7 @@ namespace MAT_NS_BEGIN DebugEvent evt(DebugEventType::EVT_DROPPED); evt.param1 = dropped; + evt.param2 = static_cast(DROPPED_REASON_OFFLINE_STORAGE_OVERFLOW); evt.size = dropped; m_manager.DispatchEvent(evt); diff --git a/lib/offline/OfflineStorage_SQLite.cpp b/lib/offline/OfflineStorage_SQLite.cpp index 9b10ebfa2..0ce28278e 100644 --- a/lib/offline/OfflineStorage_SQLite.cpp +++ b/lib/offline/OfflineStorage_SQLite.cpp @@ -954,24 +954,27 @@ namespace MAT_NS_BEGIN { LOG_TRACE("DB is too big, deleting..."); Execute("DELETE FROM " TABLE_NAME_EVENTS); Execute("VACUUM"); - return true; + eventsDropped = count; } - - SqliteStatement trimStmt(*m_db, m_stmtTrimEvents_percent); - if (!trimStmt.execute(25)) + else { - // If something went wrong with trimming 25%, try more radical measure - LOG_TRACE("Evict all non-critical"); - Execute("DELETE FROM " TABLE_NAME_EVENTS " WHERE persistence=1"); + SqliteStatement trimStmt(*m_db, m_stmtTrimEvents_percent); + if (!trimStmt.execute(25)) + { + // If something went wrong with trimming 25%, try more radical measure + LOG_TRACE("Evict all non-critical"); + Execute("DELETE FROM " TABLE_NAME_EVENTS " WHERE persistence=1"); + } + eventsDropped = count - GetRecordCountUnsafe(EventLatency::EventLatency_Unspecified); + LOG_TRACE("Db resized, events dropped: %zu", eventsDropped); + trimStmt.reset(); } - eventsDropped = count - GetRecordCountUnsafe(EventLatency::EventLatency_Unspecified); - LOG_TRACE("Db resized, events dropeed: %d", eventsDropped); - trimStmt.reset(); } m_DbSizeEstimate = GetSize(); DebugEvent evt(DebugEventType::EVT_DROPPED); evt.param1 = eventsDropped; + evt.param2 = static_cast(DROPPED_REASON_OFFLINE_STORAGE_OVERFLOW); evt.size = eventsDropped; m_logManager.DispatchEvent(evt); diff --git a/lib/offline/StorageObserver.cpp b/lib/offline/StorageObserver.cpp index e81ebc66a..e0bcc341e 100644 --- a/lib/offline/StorageObserver.cpp +++ b/lib/offline/StorageObserver.cpp @@ -160,6 +160,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = EVT_DROPPED; evt.param1 = overallCount; + evt.param2 = static_cast(DROPPED_REASON_OFFLINE_STORAGE_OVERFLOW); evt.size = overallCount; DispatchEvent(evt); } @@ -180,6 +181,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = EVT_DROPPED; evt.param1 = overallCount; + evt.param2 = static_cast(DROPPED_REASON_RETRY_EXCEEDED); evt.size = overallCount; DispatchEvent(evt); } diff --git a/lib/offline/StorageObserver.hpp b/lib/offline/StorageObserver.hpp index 58f813972..4b10f1c77 100644 --- a/lib/offline/StorageObserver.hpp +++ b/lib/offline/StorageObserver.hpp @@ -62,6 +62,11 @@ namespace MAT_NS_BEGIN { return m_offlineStorage.GetRecordCount(); } + size_t GetRemainingRecordCountForShutdown() const + { + return m_offlineStorage.GetRemainingRecordCountForShutdown(); + } + RoutePassThrough start{ this, &StorageObserver::handleStart }; RoutePassThrough stop{ this, &StorageObserver::handleStop }; diff --git a/lib/stats/Statistics.cpp b/lib/stats/Statistics.cpp index 4abf09563..773dd4d13 100644 --- a/lib/stats/Statistics.cpp +++ b/lib/stats/Statistics.cpp @@ -160,6 +160,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = DebugEventType::EVT_DROPPED; evt.param1 = 1; + evt.param2 = static_cast(DROPPED_REASON_OFFLINE_STORAGE_SAVE_FAILED); OnDebugEvent(evt); return true; diff --git a/lib/system/TelemetrySystem.cpp b/lib/system/TelemetrySystem.cpp index 119d3f5e1..24ad34ba9 100644 --- a/lib/system/TelemetrySystem.cpp +++ b/lib/system/TelemetrySystem.cpp @@ -76,7 +76,20 @@ namespace MAT_NS_BEGIN { auto uploadTime = GetUptimeMs() - stopTimes[0]; if (uploadTime >= (1000L * timeoutInSec)) { - // Hard-stop if it takes longer than planned + // Hard-stop if it takes longer than planned. + // Emit an EVT_DROPPED carrying the remaining record count + // so listeners can attribute teardown-timeout losses. + // Delegate the "remaining" calculation to the storage layer + // so each backend (memory vs SQLite/Room) can decide whether + // in-flight records need to be added on top of GetRecordCount(). + const size_t remaining = storage.GetRemainingRecordCountForShutdown(); + if (remaining > 0) + { + DebugEvent evt(DebugEventType::EVT_DROPPED, remaining, + static_cast(DROPPED_REASON_TEARDOWN_TIMEOUT)); + evt.size = remaining; + m_logManager.DispatchEvent(evt); + } LOG_TRACE("Shutdown timer expired, exiting..."); break; }