P0: preserve canonical SV TX fault evidence and make revocation burst-bounded

.github/workflows/ptp-core-ci.yml

@@ -131,6 +131,7 @@ jobs:
           Build-Target 'sv_ptp_clock_lease_tests'
           Build-Target 'sv_ptp_phase_authority_tests'
           Build-Target 'sv_tx_egress_evidence_tests'
+          Build-Target 'sv_tx_fault_burst_tests'
           Build-Target 'ptp_profile_analyze'
 
       - name: Run PTP regression and analyzer smoke tests on MSVC

apps/arstack_studio/qml/WorkflowBar.qml

@@ -282,7 +282,10 @@ SurfacePanel {
             return "SV still running · Studio session warning"
         if (ribbon.unexpectedSvStop) {
             if (ribbon.device.svStopReason === "LEASE_EXPIRED") return "SV stopped · USB control lease expired"
-            if (ribbon.device.svStopReason === "RUNTIME_FAULT") return "SV stopped · board runtime fault"
+            if (ribbon.device.svStopReason === "RUNTIME_FAULT")
+                return ribbon.device.svTxFaultDetail.length > 0
+                    ? "SV stopped · " + ribbon.device.svTxFaultDetail
+                    : "SV stopped · board runtime fault"
             return "SV stopped · checking board cause"
         }
         if (smartSession.state === "WAITING FOR DEVICE") return "Connect device"

apps/arstack_studio/src/DeviceController.cpp

@@ -2261,6 +2261,37 @@ void DeviceController::processLine(const QString& rawLine) {
         }
     }
 
+    // Board-owned sticky Ethernet fault survives an SV START after fail-closed
+    // STOP. Never show an old burst as today's fault: bind to SVLIFE stop_seq
+    // and the verified board boot, while keeping the normal RUNNING state clear.
+    static const QRegularExpression kSvTxFault(
+        QStringLiteral("SVTXFAULT state=(RUNNING|STOPPED) svlife_stop_seq=(\\d+)"
+                       ".*last_stop_first_error=(-?\\d+) last_stop_last_error=(-?\\d+)"
+                       " last_stop_link_up=([01])"));
+    const auto txFault = kSvTxFault.match(line);
+    if (txFault.hasMatch()) {
+        bool seqOk = false;
+        bool errorOk = false;
+        const quint64 seq = txFault.captured(2).toULongLong(&seqOk);
+        const int lastError = txFault.captured(4).toInt(&errorOk);
+        if (seqOk && errorOk && seq > 0U && seq == lastSvStopSeq_) {
+            const QString errorName = lastError == 0x107
+                ? QStringLiteral("TIMEOUT")
+                : lastError == 0x103
+                    ? QStringLiteral("INVALID_STATE")
+                    : QStringLiteral("0x") +
+                      QString::number(static_cast<quint32>(lastError), 16).toUpper();
+            const QString detail = QStringLiteral("Ethernet TX %1 · link %2")
+                .arg(errorName, txFault.captured(5) == QStringLiteral("1")
+                    ? QStringLiteral("UP") : QStringLiteral("DOWN"));
+            if (svTxFaultSeq_ != seq || svTxFaultDetail_ != detail) {
+                svTxFaultSeq_ = seq;
+                svTxFaultDetail_ = detail;
+                emit svStopReasonChanged();
+            }
+        }
+    }
+
     auto match = kStateExpression.match(line);
     if (match.hasMatch()) {
         const bool observedRunning = match.captured(1).compare(QStringLiteral("RUNNING"), Qt::CaseInsensitive) == 0;
@@ -2513,11 +2544,13 @@ void DeviceController::processLine(const QString& rawLine) {
 void DeviceController::resetTelemetry() {
     operatorStopPending_ = false;
     lastSvStopSeq_ = 0;
+    const bool hadStopDetails = !svLastStopCause_.isEmpty() ||
+        !svTxFaultDetail_.isEmpty() || !svStopReason_.isEmpty();
     svLastStopCause_.clear();
-    if (!svStopReason_.isEmpty()) {
-        svStopReason_.clear();
-        emit svStopReasonChanged();
-    }
+    svStopReason_.clear();
+    svTxFaultDetail_.clear();
+    svTxFaultSeq_ = 0;
+    if (hadStopDetails) emit svStopReasonChanged();
     fps_ = QStringLiteral("—");
     missed_ = QStringLiteral("—");
     txFailures_ = QStringLiteral("—");

apps/arstack_studio/src/DeviceController.hpp

@@ -53,6 +53,7 @@ class DeviceController : public QObject {
     Q_PROPERTY(bool running READ running NOTIFY runningChanged)
     Q_PROPERTY(QString svStopReason READ svStopReason NOTIFY svStopReasonChanged)
     Q_PROPERTY(QString svLastStopCause READ svLastStopCause NOTIFY svStopReasonChanged)
+    Q_PROPERTY(QString svTxFaultDetail READ svTxFaultDetail NOTIFY svStopReasonChanged)
     Q_PROPERTY(bool scenarioApplying READ scenarioApplying NOTIFY scenarioStateChanged)
     Q_PROPERTY(QString portName READ portName NOTIFY connectedChanged)
     Q_PROPERTY(QString lastError READ lastError NOTIFY lastErrorChanged)
@@ -151,6 +152,10 @@ class DeviceController : public QObject {
     [[nodiscard]] bool running() const noexcept;
     [[nodiscard]] QString svStopReason() const { return svStopReason_; }
     [[nodiscard]] QString svLastStopCause() const { return svLastStopCause_; }
+    [[nodiscard]] QString svTxFaultDetail() const {
+        return svTxFaultSeq_ != 0U && svTxFaultSeq_ == lastSvStopSeq_
+            ? svTxFaultDetail_ : QString{};
+    }
     [[nodiscard]] bool scenarioApplying() const noexcept;
     [[nodiscard]] QString portName() const;
     [[nodiscard]] QString lastError() const;
@@ -452,6 +457,8 @@ class DeviceController : public QObject {
     QString svStopReason_;
     QString svLastStopCause_;
     quint64 lastSvStopSeq_{0};
+    quint64 svTxFaultSeq_{0};
+    QString svTxFaultDetail_;
     bool connected_{false};
     bool discovering_{false};
     bool deviceVerified_{false};

embedded/esp32p4_smv_injector/main/app_main.cpp

@@ -128,6 +128,9 @@ constexpr std::uint16_t kSampleModeSamplesPerSecond = 1U;
 constexpr std::uint16_t kDiagnosticEtherType = 0x88B5U;
 constexpr std::uint32_t kStatsMergeEvery = 400U;
 constexpr std::uint32_t kTransportFailureStopThreshold = 400U; // 100 ms at 4 kHz
+// Only one publisher task writes this recorder. Status/serial may read it on CPU0.
+// Capture real first/last ESP-IDF TX errors before a fail-closed STOP.
+ar::iec61850::sampled_values::SvTxFaultBurst g_sv_tx_faults{};
 constexpr std::uint32_t kTimerResolutionHz = 1000000U;
 #if AR_SV_PTP_PHASE_SUPPORTED
 // Separate clock authority from SV TX session state. During SV STOP the
@@ -1223,6 +1226,7 @@ void publisher_task(void* argument) {
             schedule_anchored = false;
             consecutive_canonical_tx_failures = 0U;
             last_canonical_tx_error = ESP_OK;
+            g_sv_tx_faults.new_session();
             expected_schedule.reset(active_profile.publisher_rate_hz);
 #if AR_SV_TX_EGRESS_SUPPORTED
             zero_epoch_frames_seen = 0U;
@@ -1461,12 +1465,21 @@ void publisher_task(void* argument) {
             ++local.canonical_fail;
             last_canonical_tx_error = ESP_ERR_INVALID_STATE;
             ++consecutive_canonical_tx_failures;
+            const bool first_link_failure = g_sv_tx_faults.failure(
+                static_cast<std::int32_t>(ESP_ERR_INVALID_STATE), false);
 #if AR_SV_PTP_PHASE_SUPPORTED
-            g_sample_clock_context.publisher_fault_generation.fetch_add(
-                1U, std::memory_order_release);
-            g_sample_clock_context.phase_authority_valid.store(
-                false, std::memory_order_release);
-            revoke_sv_auto_claims(SvSyncLossReason::ethernet_link);
+            // The first failure revokes stale evidence immediately. Repeating
+            // the same invalidation 400 times in the 4-kHz hot path cannot
+            // improve safety and adds work while the MAC is already failing.
+            if (first_link_failure) {
+                g_sample_clock_context.publisher_fault_generation.fetch_add(
+                    1U, std::memory_order_release);
+                g_sample_clock_context.phase_authority_valid.store(
+                    false, std::memory_order_release);
+                revoke_sv_auto_claims(SvSyncLossReason::ethernet_link);
+            }
+#else
+            (void)first_link_failure;
 #endif
 #if CONFIG_AR_SMV_BROADCAST_MIRROR
             ++local.mirror_fail;
@@ -1651,18 +1664,27 @@ void publisher_task(void* argument) {
 #endif
             if (canonical_result == ESP_OK) {
                 ++local.canonical_ok;
+                if (consecutive_canonical_tx_failures != 0U)
+                    g_sv_tx_faults.recovered();
                 consecutive_canonical_tx_failures = 0U;
                 last_canonical_tx_error = ESP_OK;
             } else {
                 ++local.canonical_fail;
                 last_canonical_tx_error = canonical_result;
                 ++consecutive_canonical_tx_failures;
+                const bool first_tx_failure = g_sv_tx_faults.failure(
+                    static_cast<std::int32_t>(canonical_result),
+                    (xEventGroupGetBits(g_link_events) & kLinkUpBit) != 0U);
 #if AR_SV_PTP_PHASE_SUPPORTED
-                g_sample_clock_context.publisher_fault_generation.fetch_add(
-                    1U, std::memory_order_release);
-                g_sample_clock_context.phase_authority_valid.store(
-                    false, std::memory_order_release);
-                revoke_sv_auto_claims(SvSyncLossReason::canonical_tx);
+                if (first_tx_failure) {
+                    g_sample_clock_context.publisher_fault_generation.fetch_add(
+                        1U, std::memory_order_release);
+                    g_sample_clock_context.phase_authority_valid.store(
+                        false, std::memory_order_release);
+                    revoke_sv_auto_claims(SvSyncLossReason::canonical_tx);
+                }
+#else
+                (void)first_tx_failure;
 #endif
             }
 
@@ -1689,10 +1711,20 @@ void publisher_task(void* argument) {
             local, lateness_us, missed_slots, canonical_sample_count, signal_state.generation);
         g_sample_tick_total.fetch_add(1U, std::memory_order_relaxed);
         if (consecutive_canonical_tx_failures >= kTransportFailureStopThreshold) {
+            g_sv_tx_faults.stopped(
+                static_cast<std::uint32_t>(esp_timer_get_time() / 1000LL));
+            const auto fault = g_sv_tx_faults.snapshot();
             ESP_LOGE(kTag,
-                     "SV transport fault: %lu consecutive canonical TX failures; last=%s; forcing STOP",
+                     "SV transport fault: %lu consecutive TX failures; first=%s "
+                     "last=%s link_at_failure=%u burst=%lu total=%lu "
+                     "stop_seq=%lu; forcing STOP (fail closed)",
                      static_cast<unsigned long>(consecutive_canonical_tx_failures),
-                     esp_err_to_name(last_canonical_tx_error));
+                     esp_err_to_name(static_cast<esp_err_t>(fault.stop_first_error)),
+                     esp_err_to_name(last_canonical_tx_error),
+                     fault.stop_link_up ? 1U : 0U,
+                     static_cast<unsigned long>(fault.stop_streak),
+                     static_cast<unsigned long>(fault.failed_frames),
+                     static_cast<unsigned long>(fault.stop_sequence));
             merge_stats(local);
             live_control_force_stop();
             stop_sample_clock(sample_clock);
@@ -2269,6 +2301,13 @@ void telemetry_task(void*) {
 
 } // namespace
 
+namespace ar::esp32p4::smv {
+ar::iec61850::sampled_values::SvTxFaultSnapshot
+live_tx_fault_snapshot() noexcept {
+    return g_sv_tx_faults.snapshot();
+}
+} // namespace ar::esp32p4::smv
+
 extern "C" void app_main(void) {
     g_link_events = xEventGroupCreate();
     if (g_link_events == nullptr) {

embedded/esp32p4_smv_injector/main/live_control.cpp

@@ -591,6 +591,29 @@ void print_state() noexcept {
                  g_max_heartbeat_gap_ms.load(std::memory_order_acquire)),
              static_cast<unsigned long>(
                  g_slow_heartbeat_events.load(std::memory_order_acquire)));
+    // This is sticky historical TRANSPORT evidence, not the current STOP
+    // condition. A restarted publisher may be RUNNING while last_stop_seq>0.
+    const auto tx = live_tx_fault_snapshot();
+    ESP_LOGI(kTag,
+             "SVTXFAULT state=%s svlife_stop_seq=%lu burst_now=%lu failed_total=%lu bursts=%lu "
+             "recovered=%lu first_error=%ld last_error=%ld link_now_at_failure=%u "
+             "last_stop_seq=%lu last_stop_uptime_ms=%lu last_stop_burst=%lu "
+             "last_stop_first_error=%ld last_stop_last_error=%ld last_stop_link_up=%u",
+             g_running.load(std::memory_order_acquire) ? "RUNNING" : "STOPPED",
+             static_cast<unsigned long>(g_sv_stop_seq.load(std::memory_order_acquire)),
+             static_cast<unsigned long>(tx.current_streak),
+             static_cast<unsigned long>(tx.failed_frames),
+             static_cast<unsigned long>(tx.failure_bursts),
+             static_cast<unsigned long>(tx.recovered_bursts),
+             static_cast<long>(tx.first_error),
+             static_cast<long>(tx.last_error),
+             tx.last_link_up ? 1U : 0U,
+             static_cast<unsigned long>(tx.stop_sequence),
+             static_cast<unsigned long>(tx.stop_uptime_ms),
+             static_cast<unsigned long>(tx.stop_streak),
+             static_cast<long>(tx.stop_first_error),
+             static_cast<long>(tx.stop_last_error),
+             tx.stop_link_up ? 1U : 0U);
     for (std::size_t index = 0U; index < state.channels.size(); ++index) {
         const auto& channel = state.channels[index];
         ESP_LOGI(kTag,

embedded/esp32p4_smv_injector/main/live_control.hpp

@@ -3,6 +3,7 @@
 #pragma once
 
 #include "ariec61850/sampled_values/live_signal_state.hpp"
+#include "ariec61850/sampled_values/sv_tx_fault_burst.hpp"
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
 
@@ -17,6 +18,9 @@ void live_control_initialize(
 void live_control_bind_publisher_task(TaskHandle_t task) noexcept;
 void live_control_force_stop() noexcept;
 
+[[nodiscard]] ar::iec61850::sampled_values::SvTxFaultSnapshot
+live_tx_fault_snapshot() noexcept;
+
 [[nodiscard]] ar::iec61850::sampled_values::SvLiveSignalState
 live_signal_snapshot() noexcept;
 

include/ariec61850/sampled_values/sv_tx_fault_burst.hpp

@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <atomic>
+#include <cstdint>
+
+// Single publisher writer; control/diagnostics readers may run on another core.
+// This records observed Ethernet TX failures. It must never turn a failed
+// transmit into a successful sample or automatically restart the publisher.
+namespace ar::iec61850::sampled_values {
+
+struct SvTxFaultSnapshot final {
+    std::uint32_t failed_frames{};
+    std::uint32_t failure_bursts{};
+    std::uint32_t recovered_bursts{};
+    std::uint32_t current_streak{};
+    std::int32_t first_error{};
+    std::int32_t last_error{};
+    bool last_link_up{};
+    std::uint32_t stop_sequence{};
+    std::uint32_t stop_uptime_ms{};
+    std::uint32_t stop_streak{};
+    std::int32_t stop_first_error{};
+    std::int32_t stop_last_error{};
+    bool stop_link_up{};
+};
+
+class SvTxFaultBurst final {
+public:
+    // Returns true only on the first error of each contiguous TX failure burst.
+    // Caller can revoke sync authority once; it must still count each miss.
+    [[nodiscard]] bool failure(
+        const std::int32_t error, const bool link_up) noexcept {
+        failed_frames_.fetch_add(1U, std::memory_order_relaxed);
+        const auto was = current_streak_.fetch_add(1U, std::memory_order_acq_rel);
+        if (was == 0U) {
+            first_error_.store(error, std::memory_order_relaxed);
+            failure_bursts_.fetch_add(1U, std::memory_order_relaxed);
+        }
+        last_error_.store(error, std::memory_order_relaxed);
+        last_link_up_.store(link_up, std::memory_order_release);
+        return was == 0U;
+    }
+
+    // Call only following a successful canonical TX after a nonzero streak.
+    void recovered() noexcept {
+        if (current_streak_.exchange(0U, std::memory_order_acq_rel) != 0U)
+            recovered_bursts_.fetch_add(1U, std::memory_order_relaxed);
+    }
+
+    // START clears the active error streak, NOT the persistent last-stop record.
+    void new_session() noexcept {
+        current_streak_.store(0U, std::memory_order_release);
+    }
+
+    // Called immediately BEFORE fail-closed STOP, while the writer still owns
+    // the current burst. Keep the real ESP-IDF error across a manual restart.
+    void stopped(const std::uint32_t uptime_ms) noexcept {
+        stop_streak_.store(
+            current_streak_.load(std::memory_order_acquire),
+            std::memory_order_relaxed);
+        stop_first_error_.store(
+            first_error_.load(std::memory_order_relaxed),
+            std::memory_order_relaxed);
+        stop_last_error_.store(
+            last_error_.load(std::memory_order_relaxed),
+            std::memory_order_relaxed);
+        stop_link_up_.store(
+            last_link_up_.load(std::memory_order_acquire),
+            std::memory_order_relaxed);
+        stop_uptime_ms_.store(uptime_ms, std::memory_order_relaxed);
+        stop_sequence_.fetch_add(1U, std::memory_order_release);
+    }
+
+    [[nodiscard]] SvTxFaultSnapshot snapshot() const noexcept {
+        return {
+            failed_frames_.load(std::memory_order_relaxed),
+            failure_bursts_.load(std::memory_order_relaxed),
+            recovered_bursts_.load(std::memory_order_relaxed),
+            current_streak_.load(std::memory_order_acquire),
+            first_error_.load(std::memory_order_relaxed),
+            last_error_.load(std::memory_order_relaxed),
+            last_link_up_.load(std::memory_order_acquire),
+            stop_sequence_.load(std::memory_order_acquire),
+            stop_uptime_ms_.load(std::memory_order_relaxed),
+            stop_streak_.load(std::memory_order_relaxed),
+            stop_first_error_.load(std::memory_order_relaxed),
+            stop_last_error_.load(std::memory_order_relaxed),
+            stop_link_up_.load(std::memory_order_relaxed),
+        };
+    }
+
+private:
+    std::atomic<std::uint32_t> failed_frames_{0U};
+    std::atomic<std::uint32_t> failure_bursts_{0U};
+    std::atomic<std::uint32_t> recovered_bursts_{0U};
+    std::atomic<std::uint32_t> current_streak_{0U};
+    std::atomic<std::int32_t> first_error_{0};
+    std::atomic<std::int32_t> last_error_{0};
+    std::atomic<bool> last_link_up_{false};
+    std::atomic<std::uint32_t> stop_sequence_{0U};
+    std::atomic<std::uint32_t> stop_uptime_ms_{0U};
+    std::atomic<std::uint32_t> stop_streak_{0U};
+    std::atomic<std::int32_t> stop_first_error_{0};
+    std::atomic<std::int32_t> stop_last_error_{0};
+    std::atomic<bool> stop_link_up_{false};
+};
+
+} // namespace ar::iec61850::sampled_values