From 05a6028d15f0a84d411e6dc78f4502aefe23cdc0 Mon Sep 17 00:00:00 2001 From: Florian Sager Date: Tue, 17 Nov 2026 16:04:52 +0000 Subject: [PATCH 1/4] fix(radio): replace one-way noise-floor ratchet with median estimator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The noise-floor calibration sampled only RSSI values below the current floor + threshold, a one-way ratchet: it accepted ever-lower samples but never recovered upward, so _noise_floor drifted to the -120 clamp and stayed there. That left the RSSI-margin LBT (isChannelActive with interference_threshold) permanently over-sensitive — sends deferred even on a quiet channel. Independently reproduced on the bench (RAK4631, 250 zero-hop adverts @ 0.4 s: floor pinned at -120 within ~20-30 s). Replace the ratcheted block mean with the median of the 64-sample block: - accepts every idle (!isReceivingPacket) sample — no downward bias; - median rejects transient interference spikes (high and low outliers) and recovers in BOTH directions; - _noise_floor is written only after a full block, so the previous value stays valid while the next block is sampled — no reset-to-0 and thus no permissive LBT window (margin = RSSI - 0) during reconvergence; - samples are rate-limited to one per NOISE_FLOOR_SAMPLE_INTERVAL_MS (50 ms), so a block spans a real ~3.2 s window and the median gets temporal interference rejection instead of collapsing to a few ms of near-simultaneous readings (it also keeps the first samples off the AGC settle ramp right after startReceive()). A block median jumping more than NOISE_FLOOR_MAX_RISE_DB (15 dB) above the published floor is treated as activity-contaminated and held (the idle guard misses inter-packet energy), so the LBT keeps a meaningful idle reference under load. The hold is one-sided (quieter blocks publish at once) and bounded: after NOISE_FLOOR_MAX_HELD_BLOCKS (3) consecutive held blocks the median is accepted, so a genuine permanent floor rise cannot keep the floor stuck low. resetAGC no longer forces _noise_floor = 0 (the stuck-ratchet workaround); it only discards the in-progress block so a fresh one is measured after the analog frontend reset. All noise-floor debug prints (accept, hold, bounded release) sit behind MESH_DEBUG_NOISE_FLOOR, consistent with the value print on dev. Bench-validated in the PR thread across three iterations (RAK4631, Heltec V3 stimulus; ~75 % channel occupancy): no -120 clamp under load, published floor stays the idle floor during bursts, recovers within one block after the channel clears, and a genuine >15 dB rise releases after 3 held blocks. Co-Authored-By: Claude Code --- src/helpers/radiolib/RadioLibWrappers.cpp | 106 ++++++++++++++++------ src/helpers/radiolib/RadioLibWrappers.h | 16 +++- 2 files changed, 94 insertions(+), 28 deletions(-) diff --git a/src/helpers/radiolib/RadioLibWrappers.cpp b/src/helpers/radiolib/RadioLibWrappers.cpp index 0faeebac358..111c6513175 100644 --- a/src/helpers/radiolib/RadioLibWrappers.cpp +++ b/src/helpers/radiolib/RadioLibWrappers.cpp @@ -8,11 +8,22 @@ #define STATE_TX_DONE 4 #define STATE_INT_READY 16 -#define NUM_NOISE_FLOOR_SAMPLES 64 -#define SAMPLING_THRESHOLD 14 - static volatile uint8_t state = STATE_IDLE; +// In-place insertion sort of int16_t samples for the noise-floor median. Runs once per +// calibration block (64 elements, ~every 2 s of idle), so O(n^2) is irrelevant here. +static void sortInt16(int16_t* a, int n) { + for (int i = 1; i < n; i++) { + int16_t key = a[i]; + int j = i - 1; + while (j >= 0 && a[j] > key) { + a[j + 1] = a[j]; + j--; + } + a[j + 1] = key; + } +} + // this function is called when a complete packet // is transmitted by the module static @@ -40,7 +51,9 @@ void RadioLibWrapper::begin() { // start average out some samples _num_floor_samples = 0; - _floor_sample_sum = 0; + _floor_block_ready = false; + _last_floor_sample_at = 0; + _held_block_count = 0; } uint32_t RadioLibWrapper::getRngSeed() { @@ -61,9 +74,9 @@ void RadioLibWrapper::idle() { void RadioLibWrapper::triggerNoiseFloorCalibrate(int threshold) { _threshold = threshold; - if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES) { // ignore trigger if currently sampling + if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES) { // restart only once the current block is complete _num_floor_samples = 0; - _floor_sample_sum = 0; + _floor_block_ready = false; } } @@ -78,34 +91,73 @@ void RadioLibWrapper::resetAGC() { doResetAGC(); state = STATE_IDLE; // trigger a startReceive() - // Reset noise floor sampling so it reconverges from scratch. - // Without this, a stuck _noise_floor of -120 makes the sampling threshold - // too low (-106) to accept normal samples (~-105), self-reinforcing the - // stuck value even after the receiver has recovered. - _noise_floor = 0; + // Discard any in-progress noise-floor block: the analog frontend was just reset, so + // queued RSSI samples are stale. _noise_floor itself is left in place — the median + // estimator no longer drifts to -120 (the reason the old ratchet needed a hard + // _noise_floor = 0 reset), and forcing 0 here would create a brief permissive LBT + // window (margin = RSSI - 0) until the next block completes. _num_floor_samples = 0; - _floor_sample_sum = 0; + _floor_block_ready = false; + _held_block_count = 0; // contamination context is stale after an AFE reset } void RadioLibWrapper::loop() { if (state == STATE_RX && _num_floor_samples < NUM_NOISE_FLOOR_SAMPLES) { - if (!isReceivingPacket()) { - int rssi = getCurrentRSSI(); - if (rssi < _noise_floor + SAMPLING_THRESHOLD) { // only consider samples below current floor + sampling THRESHOLD - _num_floor_samples++; - _floor_sample_sum += rssi; - } + uint32_t now = millis(); + if (!isReceivingPacket() && now - _last_floor_sample_at >= NOISE_FLOOR_SAMPLE_INTERVAL_MS) { + // Accept every idle sample, spaced NOISE_FLOOR_SAMPLE_INTERVAL_MS apart so the block spans a real + // ~3.2 s window and the median rejects transient transmissions (not a few-ms snapshot). The old + // "rssi < floor + threshold" filter was a one-way ratchet: it only accepted samples below the + // current floor, so the block average drifted to the -120 clamp and never recovered — leaving + // _noise_floor stuck low and the RSSI-margin LBT permanently over-sensitive. + _floor_samples[_num_floor_samples++] = (int16_t)getCurrentRSSI(); + _last_floor_sample_at = now; } - } else if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES && _floor_sample_sum != 0) { - _noise_floor = _floor_sample_sum / NUM_NOISE_FLOOR_SAMPLES; - if (_noise_floor < -120) { - _noise_floor = -120; // clamp to lower bound of -120dBi + } else if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES && !_floor_block_ready) { + // Block complete: reduce to the median. The median rejects transient interference + // spikes (high and low outliers) and recovers in BOTH directions, unlike the ratcheted + // mean. _noise_floor is written only here, so the previous value stays valid while the + // next block is sampled — no reset-to-0, no permissive LBT window during reconvergence. + sortInt16(_floor_samples, NUM_NOISE_FLOOR_SAMPLES); + int16_t median = (int16_t)(((int32_t)_floor_samples[NUM_NOISE_FLOOR_SAMPLES / 2 - 1] + + (int32_t)_floor_samples[NUM_NOISE_FLOOR_SAMPLES / 2]) / 2); + // One-sided hold: a median jumping far ABOVE the published floor is activity-contaminated + // (inter-packet energy slips past the !isReceivingPacket() idle guard). Hold the old value so + // the RSSI-margin LBT stays meaningful under load; near-stable/quieter blocks publish at once. + // First block always publishes (_noise_floor=0 from begin()), so the hold binds only post-boot. + // + // Bounded: after NOISE_FLOOR_MAX_HELD_BLOCKS consecutive held blocks accept the median, else a real + // permanent rise is held forever (stuck-floor bug from the other direction). Count-based so the hold + // rides out load bursts (slow blocks) while a quiet rise releases in a few blocks. + if (median > _noise_floor + NOISE_FLOOR_MAX_RISE_DB) { + _held_block_count++; + if (_held_block_count >= NOISE_FLOOR_MAX_HELD_BLOCKS) { + _noise_floor = median; + if (_noise_floor < -120) { + _noise_floor = -120; // clamp to lower bound of -120dBi + } + _held_block_count = 0; + #ifdef MESH_DEBUG_NOISE_FLOOR + MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor = %d (accepted after %d held blocks, persistent rise)", + (int)_noise_floor, NOISE_FLOOR_MAX_HELD_BLOCKS); + #endif + } else { + #ifdef MESH_DEBUG_NOISE_FLOOR + MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor held at %d (block median %d contaminated, held %d/%d)", + (int)_noise_floor, (int)median, _held_block_count, NOISE_FLOOR_MAX_HELD_BLOCKS); + #endif + } + } else { + _held_block_count = 0; + _noise_floor = median; + if (_noise_floor < -120) { + _noise_floor = -120; // clamp to lower bound of -120dBi + } + #ifdef MESH_DEBUG_NOISE_FLOOR + MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor = %d (median)", (int)_noise_floor); + #endif } - _floor_sample_sum = 0; - - #ifdef MESH_DEBUG_NOISE_FLOOR - MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor = %d", (int)_noise_floor); - #endif + _floor_block_ready = true; } } diff --git a/src/helpers/radiolib/RadioLibWrappers.h b/src/helpers/radiolib/RadioLibWrappers.h index 02ce834390d..2917701fc9d 100644 --- a/src/helpers/radiolib/RadioLibWrappers.h +++ b/src/helpers/radiolib/RadioLibWrappers.h @@ -11,6 +11,17 @@ struct PacketMillis { uint32_t payloadMillis; // header-valid -> rx-done deadline }; +#define NUM_NOISE_FLOOR_SAMPLES 64 // RSSI samples reduced to a median per noise-floor calibration block +#define NOISE_FLOOR_MAX_RISE_DB 15 // block median jumping this far ABOVE the published floor is treated as + // activity-contaminated and held, so the RSSI-margin LBT keeps a meaningful + // (idle) reference while the channel is occupied +#define NOISE_FLOOR_SAMPLE_INTERVAL_MS 50 // min spacing between RSSI samples so a 64-sample block spans a real + // ~3.2 s window, giving the median temporal interference rejection + // instead of collapsing to a few ms of near-simultaneous readings +#define NOISE_FLOOR_MAX_HELD_BLOCKS 3 // after this many consecutive held blocks the median is accepted, so a + // permanent floor rise can't keep it stuck low. Count-based: rides out load + // bursts, a true rise releases in a few blocks. + class RadioLibWrapper : public mesh::Radio { protected: PhysicalLayer* _radio; @@ -19,7 +30,10 @@ class RadioLibWrapper : public mesh::Radio { int16_t _noise_floor, _threshold; bool _cad_enabled; uint16_t _num_floor_samples; - int32_t _floor_sample_sum; + int16_t _floor_samples[NUM_NOISE_FLOOR_SAMPLES]; + bool _floor_block_ready; // true once a full block has been reduced to a median (waits for trigger to restart) + uint32_t _last_floor_sample_at; // millis() of the last accepted RSSI sample (rate-limits block sampling) + uint8_t _held_block_count; // consecutive held blocks since the last published noise-floor value uint8_t _preamble_sf; void idle(); From 04934f09bd70fba88695f4c0a8da14511e9b75a5 Mon Sep 17 00:00:00 2001 From: Florian Sager Date: Thu, 3 Sep 2026 20:41:47 +0000 Subject: [PATCH 2/4] RX-desync watchdog: verify chip RX mode and self-heal a wedged receiver The `state` variable is firmware-side truth: if the SX126x silently leaves RX (supply dip during TX, SPI glitch, front-end upset) the RAM copy still says STATE_RX, so recvRaw() never re-arms and the 8s Dispatcher check (reading the same variable) stays quiet - the node goes deaf until reboot. Visible symptom on the median noise-floor branch: the Current-RSSI register freezes at the last energy seen, pinning the floor high (e.g. -113 -> -74). - RadioLibWrapper::loop() polls the chip's real operating mode every 10 s via new verifyRxChipMode() (SX126x GetStatus, chip-mode bits 6:4 == RX); plain SPI read, skipped while receiving, off for radios without a status register (base returns true) - after RX_DESYNC_CONFIRM_TICKS bad polls: re-arm (standby + startReceive); from the next poll on: full AFE reset (resetAGC: warm sleep + Calibrate 0x7F + image recal) - a streak that survives both (RX_DESYNC_FATAL_STREAK) is counted fatal and surfaced as ERR_EVENT_RX_DESYNC (stats error flags) - the noise-floor block is discarded on detection (its samples were reading a frozen RSSI register) - episode counters exposed via radio stats JSON ("rx_desync") Requires RadioLib >= 7.8.1 (MeshCore's pin): before that, SX126x::getStatus() returned 0 unconditionally (numBytes=0 bug, RadioLib #1872, fixed by PR #1876) and the watchdog treated every healthy radio as desynced, warm-sleep resetting it roughly every 10 s (found by agessaman in the PR review). Worst-case deaf window shrinks from "until reboot" to ~30 s; a healthy radio only ever sees one extra status read per 10 s. Validated on RAK4631/SX1262 in the PR review (ACETyr): zero false positives idle and under load, and a bench-injected standby wedge (chip out of RX behind the firmware's back) is detected, re-armed, and RX verified restored - packet reception before/after the fault identical. Co-Authored-By: Claude Code --- src/Dispatcher.cpp | 5 ++ src/Dispatcher.h | 6 +++ src/helpers/StatsFormatHelper.h | 7 +-- src/helpers/radiolib/CustomLLCC68Wrapper.h | 6 +++ src/helpers/radiolib/CustomSTM32WLxWrapper.h | 5 ++ src/helpers/radiolib/CustomSX1262Wrapper.h | 6 +++ src/helpers/radiolib/CustomSX1268Wrapper.h | 5 ++ src/helpers/radiolib/RadioLibWrappers.cpp | 49 ++++++++++++++++++++ src/helpers/radiolib/RadioLibWrappers.h | 25 ++++++++++ 9 files changed, 111 insertions(+), 3 deletions(-) diff --git a/src/Dispatcher.cpp b/src/Dispatcher.cpp index c0610b7f8a7..84eacfa1b48 100644 --- a/src/Dispatcher.cpp +++ b/src/Dispatcher.cpp @@ -82,6 +82,11 @@ void Dispatcher::loop() { if (!is_recv && _ms->getMillis() - radio_nonrx_start > 8000) { // radio has not been in Rx mode for 8 seconds! _err_flags |= ERR_EVENT_STARTRX_TIMEOUT; } + if (_radio->isRxDamaged()) { + // chip left RX behind the firmware's back and resisted the wrapper's recovery — + // the node is (still) deaf; a reboot restores it + _err_flags |= ERR_EVENT_RX_DESYNC; + } if (outbound) { // waiting for outbound send to be completed if (_radio->isSendComplete()) { diff --git a/src/Dispatcher.h b/src/Dispatcher.h index aad6cba3ec2..3020e573eb6 100644 --- a/src/Dispatcher.h +++ b/src/Dispatcher.h @@ -63,6 +63,11 @@ class Radio { virtual int getNoiseFloor() const { return 0; } + // RX-desync watchdog: true when the driver detected the chip left RX behind the + // firmware's back and its own recovery attempts failed (reboot needed). Radio + // implementations without an authoritative status register stay false. + virtual bool isRxDamaged() const { return false; } + virtual void triggerNoiseFloorCalibrate(int threshold) { } virtual void setCADEnabled(bool enable) { } @@ -110,6 +115,7 @@ typedef uint32_t DispatcherAction; #define ERR_EVENT_FULL (1 << 0) #define ERR_EVENT_CAD_TIMEOUT (1 << 1) #define ERR_EVENT_STARTRX_TIMEOUT (1 << 2) +#define ERR_EVENT_RX_DESYNC (1 << 3) // radio wedged out of RX and resisted the wrapper's recovery /** * \brief The low-level task that manages detecting incoming Packets, and the queueing diff --git a/src/helpers/StatsFormatHelper.h b/src/helpers/StatsFormatHelper.h index bf619133e93..98a469a7a15 100644 --- a/src/helpers/StatsFormatHelper.h +++ b/src/helpers/StatsFormatHelper.h @@ -24,13 +24,14 @@ class StatsFormatHelper { RadioDriverType& driver, uint32_t total_air_time_ms, uint32_t total_rx_air_time_ms) { - sprintf(reply, - "{\"noise_floor\":%d,\"last_rssi\":%d,\"last_snr\":%.2f,\"tx_air_secs\":%u,\"rx_air_secs\":%u}", + sprintf(reply, + "{\"noise_floor\":%d,\"last_rssi\":%d,\"last_snr\":%.2f,\"tx_air_secs\":%u,\"rx_air_secs\":%u,\"rx_desync\":%u}", (int16_t)radio->getNoiseFloor(), (int16_t)driver.getLastRSSI(), driver.getLastSNR(), total_air_time_ms / 1000, - total_rx_air_time_ms / 1000 + total_rx_air_time_ms / 1000, + driver.getRxDesyncEvents() ); } diff --git a/src/helpers/radiolib/CustomLLCC68Wrapper.h b/src/helpers/radiolib/CustomLLCC68Wrapper.h index b8b4096a6fe..6fd0f413fd6 100644 --- a/src/helpers/radiolib/CustomLLCC68Wrapper.h +++ b/src/helpers/radiolib/CustomLLCC68Wrapper.h @@ -43,6 +43,12 @@ class CustomLLCC68Wrapper : public RadioLibWrapper { void doResetAGC() override { sx126xResetAGC((SX126x *)_radio, getRxBoostedGainMode()); } bool supportsRxBoostedGain() const override { return true; } + + // RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX). + bool verifyRxChipMode() override { + return ((((CustomLLCC68 *)_radio)->getStatus() >> 4) & 0x07) == 0x05; + } + bool setRxBoostedGainMode(bool en) override { return ((CustomLLCC68 *)_radio)->setRxBoostedGainMode(en) == RADIOLIB_ERR_NONE; } diff --git a/src/helpers/radiolib/CustomSTM32WLxWrapper.h b/src/helpers/radiolib/CustomSTM32WLxWrapper.h index bc93de7584a..3e94444292e 100644 --- a/src/helpers/radiolib/CustomSTM32WLxWrapper.h +++ b/src/helpers/radiolib/CustomSTM32WLxWrapper.h @@ -41,4 +41,9 @@ class CustomSTM32WLxWrapper : public RadioLibWrapper { uint8_t getSpreadingFactor() const override { return ((CustomSTM32WLx *)_radio)->spreadingFactor; } void doResetAGC() override { sx126xResetAGC((SX126x *)_radio, getRxBoostedGainMode()); } + + // RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX). + bool verifyRxChipMode() override { + return ((((CustomSTM32WLx *)_radio)->getStatus() >> 4) & 0x07) == 0x05; + } }; diff --git a/src/helpers/radiolib/CustomSX1262Wrapper.h b/src/helpers/radiolib/CustomSX1262Wrapper.h index cf393200818..8961c552fab 100644 --- a/src/helpers/radiolib/CustomSX1262Wrapper.h +++ b/src/helpers/radiolib/CustomSX1262Wrapper.h @@ -55,4 +55,10 @@ class CustomSX1262Wrapper : public RadioLibWrapper { } void doResetAGC() override { sx126xResetAGC((SX126x *)_radio, getRxBoostedGainMode()); } + + // RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX). + // Plain SPI read, does not disturb reception. + bool verifyRxChipMode() override { + return ((((CustomSX1262 *)_radio)->getStatus() >> 4) & 0x07) == 0x05; + } }; diff --git a/src/helpers/radiolib/CustomSX1268Wrapper.h b/src/helpers/radiolib/CustomSX1268Wrapper.h index ba1a635e3c0..a74d3aa283f 100644 --- a/src/helpers/radiolib/CustomSX1268Wrapper.h +++ b/src/helpers/radiolib/CustomSX1268Wrapper.h @@ -52,4 +52,9 @@ class CustomSX1268Wrapper : public RadioLibWrapper { } void doResetAGC() override { sx126xResetAGC((SX126x *)_radio, getRxBoostedGainMode()); } + + // RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX). + bool verifyRxChipMode() override { + return ((((CustomSX1268 *)_radio)->getStatus() >> 4) & 0x07) == 0x05; + } }; diff --git a/src/helpers/radiolib/RadioLibWrappers.cpp b/src/helpers/radiolib/RadioLibWrappers.cpp index 111c6513175..5040e669cb8 100644 --- a/src/helpers/radiolib/RadioLibWrappers.cpp +++ b/src/helpers/radiolib/RadioLibWrappers.cpp @@ -54,6 +54,10 @@ void RadioLibWrapper::begin() { _floor_block_ready = false; _last_floor_sample_at = 0; _held_block_count = 0; + + _last_rx_sync_check = millis(); + _rx_desync_streak = 0; + n_rx_desync_events = n_rx_desync_fatals = 0; } uint32_t RadioLibWrapper::getRngSeed() { @@ -159,6 +163,51 @@ void RadioLibWrapper::loop() { } _floor_block_ready = true; } + + // --- RX-desync watchdog --- + // `state` is firmware-side truth. If the chip silently leaves RX (supply dip during + // TX, SPI glitch, front-end upset), the RAM copy still says STATE_RX, so recvRaw() + // never re-arms and the Dispatcher-side 8 s check — reading the same variable — + // stays quiet: the node goes deaf until reboot. (Visible symptom: the Current-RSSI + // register freezes at the last energy seen, pinning the noise floor high.) + // Here we ask the CHIP instead: verifyRxChipMode() reads its real operating mode. + // Radio types without an authoritative status register report true (watchdog off). + // Recovery: after RX_DESYNC_CONFIRM_TICKS bad polls, re-arm the receiver; if that + // doesn't take, escalate to a warm sleep (resets the modem state machine/AFE). + // A streak that survives both is counted fatal and surfaces as ERR_EVENT_RX_DESYNC. + if (state == STATE_RX) { + uint32_t now = millis(); + if (now - _last_rx_sync_check >= RX_DESYNC_CHECK_INTERVAL_MS) { + _last_rx_sync_check = now; + if (!isReceivingPacket() && !verifyRxChipMode()) { + if (_rx_desync_streak == 0) { + n_rx_desync_events++; + MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync - chip reports mode != RX"); + } + _rx_desync_streak++; + // samples taken while wedged read a frozen RSSI register: discard the block + _num_floor_samples = 0; + _floor_block_ready = false; + _held_block_count = 0; + if (_rx_desync_streak == RX_DESYNC_CONFIRM_TICKS) { + MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync confirmed - re-arming receiver"); + idle(); // standby, then a fresh startReceive() + startRecv(); + } else if (_rx_desync_streak > RX_DESYNC_CONFIRM_TICKS) { + if (_rx_desync_streak == RX_DESYNC_FATAL_STREAK) { + n_rx_desync_fatals++; + MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync survived AFE reset - radio damaged, reboot suggested"); + } + // warm sleep resets the analog frontend and modem state machine; + // resetAGC() re-arms from STATE_IDLE via recvRaw() on the next pass + resetAGC(); + } + } else if (_rx_desync_streak > 0) { + MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync resolved after %u ticks", _rx_desync_streak); + _rx_desync_streak = 0; + } + } + } } void RadioLibWrapper::startRecv() { diff --git a/src/helpers/radiolib/RadioLibWrappers.h b/src/helpers/radiolib/RadioLibWrappers.h index 2917701fc9d..41e7f6c1f5b 100644 --- a/src/helpers/radiolib/RadioLibWrappers.h +++ b/src/helpers/radiolib/RadioLibWrappers.h @@ -22,11 +22,23 @@ struct PacketMillis { // permanent floor rise can't keep it stuck low. Count-based: rides out load // bursts, a true rise releases in a few blocks. +// RX-desync watchdog: the `state` variable is firmware-side truth. If the chip silently +// leaves RX (supply dip during TX, SPI glitch, front-end upset) the RAM copy still says +// STATE_RX, so recvRaw() never re-arms and the Dispatcher-side check (reading the same +// variable) stays quiet — the node goes deaf until reboot, and the Current-RSSI register +// freezes at the last energy seen (the "noise floor pinned high" symptom). +#define RX_DESYNC_CHECK_INTERVAL_MS 10000 // cadence of the chip-mode verification poll +#define RX_DESYNC_CONFIRM_TICKS 2 // consecutive bad polls before recovery starts (debounces one misread, + // e.g. the first GetStatus that wakes the chip from warm sleep) +#define RX_DESYNC_FATAL_STREAK 5 // streak that survived sleep-level recovery — flag via ERR_EVENT_RX_DESYNC + class RadioLibWrapper : public mesh::Radio { protected: PhysicalLayer* _radio; mesh::MainBoard* _board; uint32_t n_recv, n_sent, n_recv_errors; + uint16_t n_rx_desync_events; // desync episodes detected (recovery was attempted for each) + uint16_t n_rx_desync_fatals; // episodes that survived sleep-level recovery (reboot needed) int16_t _noise_floor, _threshold; bool _cad_enabled; uint16_t _num_floor_samples; @@ -35,6 +47,8 @@ class RadioLibWrapper : public mesh::Radio { uint32_t _last_floor_sample_at; // millis() of the last accepted RSSI sample (rate-limits block sampling) uint8_t _held_block_count; // consecutive held blocks since the last published noise-floor value uint8_t _preamble_sf; + uint32_t _last_rx_sync_check; // millis() of the last chip-mode verification (RX-desync watchdog) + uint8_t _rx_desync_streak; // consecutive verifications that found the chip out of RX (0 = healthy) void idle(); void startRecv(); @@ -88,6 +102,17 @@ class RadioLibWrapper : public mesh::Radio { uint32_t getPacketsSent() const { return n_sent; } void resetStats() { n_recv = n_sent = n_recv_errors = 0; } + // RX-desync watchdog. verifyRxChipMode() reads the chip's real operating mode; + // base assumes RX (no authoritative status register on every radio type). + // Override in radio-specific wrappers that can check (SX126x GetStatus). + // The SX126x overrides need RadioLib >= 7.8.1 (MeshCore's pin): before the fix + // (RadioLib #1872 / PR #1876) SX126x::getStatus() returned 0 unconditionally, + // which made every healthy radio read as desynced. + virtual bool verifyRxChipMode() { return true; } + bool isRxDamaged() const override { return _rx_desync_streak >= RX_DESYNC_FATAL_STREAK; } + uint16_t getRxDesyncEvents() const { return n_rx_desync_events; } + uint16_t getRxDesyncFatals() const { return n_rx_desync_fatals; } + virtual float getLastRSSI() const override; virtual float getLastSNR() const override; From 8ccd32296c4bd0d3ca19bfb3784ae0304e8106ae Mon Sep 17 00:00:00 2001 From: Florian Sager Date: Tue, 17 Nov 2026 16:07:25 +0000 Subject: [PATCH 3/4] noise floor: don't sample during an RX-desync episode; publish first block after resolve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding (ACETyr, PR #2933): while the receiver is wedged out of RX, the block discard ran only on the 10 s verification polls, but a block spans just ~3.2 s — so up to three contaminated blocks published per poll interval, and they kept publishing after detection, because the chip keeps serving standby/frozen RSSI until the re-arm takes. After the resolve, the recovery then inverted: the correct floor read as a "contaminated rise" above the corrupted one, the one-sided hold rejected it, and the node ran an over-sensitive LBT reference for ~42-71 s — the stuck-floor failure this PR removes, arriving through its own watchdog. Two changes: - pause floor sampling entirely while a desync streak is open (_rx_desync_streak > 0). Nothing legitimate can be measured from a wedged receiver; the published value simply holds until recovery, instead of tracking the fault. - on resolve, set a one-shot bypass so the first completed block publishes unconditionally, skipping the contamination hold: the published floor is known-stale at that point, so the hold must not gate the recovery measurement. Co-Authored-By: Claude Code Bench-hardened in a second pass: a block that was mid-flight when the wedge hit could still publish its contaminated median before the next verification poll noticed (two -120 publishes 0.7 and 2.4 s after injection, both pre- detection). Block completion now verifies the chip mode first (one extra status read per ~4 s block); on a wedged chip the block is vetoed and the desync streak starts immediately, so detection latency collapses to the first completing block (~0.7 s on the bench) and no contaminated value reaches the LBT reference at all. Validation (Heltec V3 / SX1262, bench injection at t=90 s): baseline -105 /-104, ZERO noise_floor publishes between injection and the post-desync recovery publish, which lands at the pre-episode level one block after the resolve; normal median cadence resumes immediately after. Co-Authored-By: Claude Code --- src/helpers/radiolib/RadioLibWrappers.cpp | 44 +++++++++++++++++++++-- src/helpers/radiolib/RadioLibWrappers.h | 2 ++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/helpers/radiolib/RadioLibWrappers.cpp b/src/helpers/radiolib/RadioLibWrappers.cpp index 5040e669cb8..f5700567112 100644 --- a/src/helpers/radiolib/RadioLibWrappers.cpp +++ b/src/helpers/radiolib/RadioLibWrappers.cpp @@ -54,6 +54,7 @@ void RadioLibWrapper::begin() { _floor_block_ready = false; _last_floor_sample_at = 0; _held_block_count = 0; + _floor_hold_bypass = false; _last_rx_sync_check = millis(); _rx_desync_streak = 0; @@ -106,7 +107,11 @@ void RadioLibWrapper::resetAGC() { } void RadioLibWrapper::loop() { - if (state == STATE_RX && _num_floor_samples < NUM_NOISE_FLOOR_SAMPLES) { + // _rx_desync_streak == 0: while a desync is unresolved the RSSI register reads a + // wedged/frozen receiver, so sampling is paused entirely (the block in flight at + // detection was already discarded). Without this gate, blocks completed between the + // 10 s verification polls publish garbage (a block spans only ~3.2 s). + if (state == STATE_RX && _rx_desync_streak == 0 && _num_floor_samples < NUM_NOISE_FLOOR_SAMPLES) { uint32_t now = millis(); if (!isReceivingPacket() && now - _last_floor_sample_at >= NOISE_FLOOR_SAMPLE_INTERVAL_MS) { // Accept every idle sample, spaced NOISE_FLOOR_SAMPLE_INTERVAL_MS apart so the block spans a real @@ -118,6 +123,23 @@ void RadioLibWrapper::loop() { _last_floor_sample_at = now; } } else if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES && !_floor_block_ready) { + // Publish nothing sampled while the chip was out of RX: verify the chip mode at + // block completion (one extra status read per block). A wedge is then caught at + // the FIRST completing block instead of at the next verification poll — samples + // taken while wedged read a frozen/clamped RSSI register and must not reach the + // LBT reference. On veto the block is discarded and the desync streak starts, so + // the poll path re-arms the receiver on its next tick. + if (!verifyRxChipMode()) { + if (_rx_desync_streak == 0) { + n_rx_desync_events++; + MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync - block publish vetoed (chip reports mode != RX)"); + } + _rx_desync_streak++; + _num_floor_samples = 0; + _floor_block_ready = false; + _held_block_count = 0; + return; + } // Block complete: reduce to the median. The median rejects transient interference // spikes (high and low outliers) and recovers in BOTH directions, unlike the ratcheted // mean. _noise_floor is written only here, so the previous value stays valid while the @@ -133,7 +155,22 @@ void RadioLibWrapper::loop() { // Bounded: after NOISE_FLOOR_MAX_HELD_BLOCKS consecutive held blocks accept the median, else a real // permanent rise is held forever (stuck-floor bug from the other direction). Count-based so the hold // rides out load bursts (slow blocks) while a quiet rise releases in a few blocks. - if (median > _noise_floor + NOISE_FLOOR_MAX_RISE_DB) { + if (_floor_hold_bypass) { + // First block after a resolved RX-desync episode: publish unconditionally. + // The published floor survived the episode and is known-stale, so the + // contamination hold must not gate the recovery measurement — otherwise the + // correct value is rejected as a "contaminated rise" and the node keeps an + // LBT reference that is off by tens of dB for several more blocks. + _floor_hold_bypass = false; + _held_block_count = 0; + _noise_floor = median; + if (_noise_floor < -120) { + _noise_floor = -120; // clamp to lower bound of -120dBi + } + #ifdef MESH_DEBUG_NOISE_FLOOR + MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor = %d (post-desync recovery)", (int)_noise_floor); + #endif + } else if (median > _noise_floor + NOISE_FLOOR_MAX_RISE_DB) { _held_block_count++; if (_held_block_count >= NOISE_FLOOR_MAX_HELD_BLOCKS) { _noise_floor = median; @@ -205,6 +242,9 @@ void RadioLibWrapper::loop() { } else if (_rx_desync_streak > 0) { MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync resolved after %u ticks", _rx_desync_streak); _rx_desync_streak = 0; + // sampling resumes with the next pass; the first completed block publishes + // unconditionally (floor known-stale) + _floor_hold_bypass = true; } } } diff --git a/src/helpers/radiolib/RadioLibWrappers.h b/src/helpers/radiolib/RadioLibWrappers.h index 41e7f6c1f5b..89c81e4e0ef 100644 --- a/src/helpers/radiolib/RadioLibWrappers.h +++ b/src/helpers/radiolib/RadioLibWrappers.h @@ -46,6 +46,8 @@ class RadioLibWrapper : public mesh::Radio { bool _floor_block_ready; // true once a full block has been reduced to a median (waits for trigger to restart) uint32_t _last_floor_sample_at; // millis() of the last accepted RSSI sample (rate-limits block sampling) uint8_t _held_block_count; // consecutive held blocks since the last published noise-floor value + bool _floor_hold_bypass; // publish the next completed block unconditionally (floor known-stale after a + // resolved RX-desync episode; see loop()) uint8_t _preamble_sf; uint32_t _last_rx_sync_check; // millis() of the last chip-mode verification (RX-desync watchdog) uint8_t _rx_desync_streak; // consecutive verifications that found the chip out of RX (0 = healthy) From 59fb7798e72709d7c63f104ccfde767c3fe53082 Mon Sep 17 00:00:00 2001 From: Florian Sager Date: Tue, 17 Nov 2026 20:04:47 +0000 Subject: [PATCH 4/4] noise floor: settle-gate sampling after every RX (re)start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Right after a successful startReceive() the RSSI readout is not yet meaningful: the AGC needs a few milliseconds to re-converge, and current- RSSI reads as a deep-negative ramp toward the real level. Measured independently twice: ~-127 -> -116 -> -106 -> -86 -> real level over ~8 ms after resetAGC()/startReceive() on an SX1268 (PR review), and on an SX1262 repeater site every one of 7,4xx-7,9xx accepted samples <= -125 dBm in 95-minute windows landed < 5 ms after an RX start, none later — with CAD scans restarting RX every ~1.5 s, that fed enough ramp readings into the old ratcheted mean to pin the floor at -120 within ~45 min (reported for a downstream fork in m0ours/MeshCore#27, which references this PR). The median estimator already rejects a transient ramp sample, and the 50 ms sample spacing limits each restart to at most one such sample per block — but that first sample after a restart could still land on the ramp whenever the previous sample was more than 50 ms ago, which on a busy site is the common case. Gate floor sampling on NOISE_FLOOR_RX_START_SETTLE_MS (20 ms) after every successful startReceive(), covering packet re-arms, TX return-to-RX, watchdog re-arm and resetAGC transitions alike. Co-Authored-By: Claude Code --- src/helpers/radiolib/RadioLibWrappers.cpp | 5 ++++- src/helpers/radiolib/RadioLibWrappers.h | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/helpers/radiolib/RadioLibWrappers.cpp b/src/helpers/radiolib/RadioLibWrappers.cpp index f5700567112..888cd64bad0 100644 --- a/src/helpers/radiolib/RadioLibWrappers.cpp +++ b/src/helpers/radiolib/RadioLibWrappers.cpp @@ -53,6 +53,7 @@ void RadioLibWrapper::begin() { _num_floor_samples = 0; _floor_block_ready = false; _last_floor_sample_at = 0; + _last_rx_start_ms = 0; _held_block_count = 0; _floor_hold_bypass = false; @@ -113,7 +114,8 @@ void RadioLibWrapper::loop() { // 10 s verification polls publish garbage (a block spans only ~3.2 s). if (state == STATE_RX && _rx_desync_streak == 0 && _num_floor_samples < NUM_NOISE_FLOOR_SAMPLES) { uint32_t now = millis(); - if (!isReceivingPacket() && now - _last_floor_sample_at >= NOISE_FLOOR_SAMPLE_INTERVAL_MS) { + if (!isReceivingPacket() && now - _last_floor_sample_at >= NOISE_FLOOR_SAMPLE_INTERVAL_MS + && now - _last_rx_start_ms >= NOISE_FLOOR_RX_START_SETTLE_MS) { // Accept every idle sample, spaced NOISE_FLOOR_SAMPLE_INTERVAL_MS apart so the block spans a real // ~3.2 s window and the median rejects transient transmissions (not a few-ms snapshot). The old // "rssi < floor + threshold" filter was a one-way ratchet: it only accepted samples below the @@ -257,6 +259,7 @@ void RadioLibWrapper::startRecv() { int err = _radio->startReceive(); if (err == RADIOLIB_ERR_NONE) { state = STATE_RX; + _last_rx_start_ms = millis(); // floor sampling settles before the AGC has re-converged after a restart } else { MESH_DEBUG_PRINTLN("RadioLibWrapper: error: startReceive(%d)", err); } diff --git a/src/helpers/radiolib/RadioLibWrappers.h b/src/helpers/radiolib/RadioLibWrappers.h index 89c81e4e0ef..fe7769aea60 100644 --- a/src/helpers/radiolib/RadioLibWrappers.h +++ b/src/helpers/radiolib/RadioLibWrappers.h @@ -18,6 +18,10 @@ struct PacketMillis { #define NOISE_FLOOR_SAMPLE_INTERVAL_MS 50 // min spacing between RSSI samples so a 64-sample block spans a real // ~3.2 s window, giving the median temporal interference rejection // instead of collapsing to a few ms of near-simultaneous readings +#define NOISE_FLOOR_RX_START_SETTLE_MS 20 // after every RX (re)start the RSSI readout ramps up for a few ms + // while the AGC re-converges (measured -127 -> -116 -> -106 -> -86 -> + // real level over ~8 ms on SX1268, and <= -125 dBm for <5 ms after + // restart on SX1262); no floor sample is taken before this has passed #define NOISE_FLOOR_MAX_HELD_BLOCKS 3 // after this many consecutive held blocks the median is accepted, so a // permanent floor rise can't keep it stuck low. Count-based: rides out load // bursts, a true rise releases in a few blocks. @@ -45,6 +49,7 @@ class RadioLibWrapper : public mesh::Radio { int16_t _floor_samples[NUM_NOISE_FLOOR_SAMPLES]; bool _floor_block_ready; // true once a full block has been reduced to a median (waits for trigger to restart) uint32_t _last_floor_sample_at; // millis() of the last accepted RSSI sample (rate-limits block sampling) + uint32_t _last_rx_start_ms; // millis() of the last successful startReceive() (settle-gates floor sampling) uint8_t _held_block_count; // consecutive held blocks since the last published noise-floor value bool _floor_hold_bypass; // publish the next completed block unconditionally (floor known-stale after a // resolved RX-desync episode; see loop())