Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/Dispatcher.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,11 @@ void Dispatcher::loop() {
if (!is_recv && _ms->getMillis() - radio_nonrx_start > 8000) { // radio has not been in Rx mode for 8 seconds!
_err_flags |= ERR_EVENT_STARTRX_TIMEOUT;
}
if (_radio->isRxDamaged()) {
// chip left RX behind the firmware's back and resisted the wrapper's recovery —
// the node is (still) deaf; a reboot restores it
_err_flags |= ERR_EVENT_RX_DESYNC;
}

if (outbound) { // waiting for outbound send to be completed
if (_radio->isSendComplete()) {
Expand Down
6 changes: 6 additions & 0 deletions src/Dispatcher.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ class Radio {

virtual int getNoiseFloor() const { return 0; }

// RX-desync watchdog: true when the driver detected the chip left RX behind the
// firmware's back and its own recovery attempts failed (reboot needed). Radio
// implementations without an authoritative status register stay false.
virtual bool isRxDamaged() const { return false; }

virtual void triggerNoiseFloorCalibrate(int threshold) { }

virtual void setCADEnabled(bool enable) { }
Expand Down Expand Up @@ -110,6 +115,7 @@ typedef uint32_t DispatcherAction;
#define ERR_EVENT_FULL (1 << 0)
#define ERR_EVENT_CAD_TIMEOUT (1 << 1)
#define ERR_EVENT_STARTRX_TIMEOUT (1 << 2)
#define ERR_EVENT_RX_DESYNC (1 << 3) // radio wedged out of RX and resisted the wrapper's recovery

/**
* \brief The low-level task that manages detecting incoming Packets, and the queueing
Expand Down
7 changes: 4 additions & 3 deletions src/helpers/StatsFormatHelper.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,14 @@ class StatsFormatHelper {
RadioDriverType& driver,
uint32_t total_air_time_ms,
uint32_t total_rx_air_time_ms) {
sprintf(reply,
"{\"noise_floor\":%d,\"last_rssi\":%d,\"last_snr\":%.2f,\"tx_air_secs\":%u,\"rx_air_secs\":%u}",
sprintf(reply,
"{\"noise_floor\":%d,\"last_rssi\":%d,\"last_snr\":%.2f,\"tx_air_secs\":%u,\"rx_air_secs\":%u,\"rx_desync\":%u}",
(int16_t)radio->getNoiseFloor(),
(int16_t)driver.getLastRSSI(),
driver.getLastSNR(),
total_air_time_ms / 1000,
total_rx_air_time_ms / 1000
total_rx_air_time_ms / 1000,
driver.getRxDesyncEvents()
);
}

Expand Down
6 changes: 6 additions & 0 deletions src/helpers/radiolib/CustomLLCC68Wrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ class CustomLLCC68Wrapper : public RadioLibWrapper {
void doResetAGC() override { sx126xResetAGC((SX126x *)_radio, getRxBoostedGainMode()); }

bool supportsRxBoostedGain() const override { return true; }

// RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX).
bool verifyRxChipMode() override {
return ((((CustomLLCC68 *)_radio)->getStatus() >> 4) & 0x07) == 0x05;
}

bool setRxBoostedGainMode(bool en) override {
return ((CustomLLCC68 *)_radio)->setRxBoostedGainMode(en) == RADIOLIB_ERR_NONE;
}
Expand Down
5 changes: 5 additions & 0 deletions src/helpers/radiolib/CustomSTM32WLxWrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,9 @@ class CustomSTM32WLxWrapper : public RadioLibWrapper {
uint8_t getSpreadingFactor() const override { return ((CustomSTM32WLx *)_radio)->spreadingFactor; }

void doResetAGC() override { sx126xResetAGC((SX126x *)_radio, getRxBoostedGainMode()); }

// RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX).
bool verifyRxChipMode() override {
return ((((CustomSTM32WLx *)_radio)->getStatus() >> 4) & 0x07) == 0x05;
}
};
6 changes: 6 additions & 0 deletions src/helpers/radiolib/CustomSX1262Wrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,10 @@ class CustomSX1262Wrapper : public RadioLibWrapper {
}

void doResetAGC() override { sx126xResetAGC((SX126x *)_radio, getRxBoostedGainMode()); }

// RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX).
// Plain SPI read, does not disturb reception.
bool verifyRxChipMode() override {
return ((((CustomSX1262 *)_radio)->getStatus() >> 4) & 0x07) == 0x05;
}
};
5 changes: 5 additions & 0 deletions src/helpers/radiolib/CustomSX1268Wrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,9 @@ class CustomSX1268Wrapper : public RadioLibWrapper {
}

void doResetAGC() override { sx126xResetAGC((SX126x *)_radio, getRxBoostedGainMode()); }

// RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX).
bool verifyRxChipMode() override {
return ((((CustomSX1268 *)_radio)->getStatus() >> 4) & 0x07) == 0x05;
}
};
196 changes: 170 additions & 26 deletions src/helpers/radiolib/RadioLibWrappers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,22 @@
#define STATE_TX_DONE 4
#define STATE_INT_READY 16

#define NUM_NOISE_FLOOR_SAMPLES 64
#define SAMPLING_THRESHOLD 14

static volatile uint8_t state = STATE_IDLE;

// In-place insertion sort of int16_t samples for the noise-floor median. Runs once per
// calibration block (64 elements, ~every 2 s of idle), so O(n^2) is irrelevant here.
static void sortInt16(int16_t* a, int n) {
for (int i = 1; i < n; i++) {
int16_t key = a[i];
int j = i - 1;
while (j >= 0 && a[j] > key) {
a[j + 1] = a[j];
j--;
}
a[j + 1] = key;
}
}

// this function is called when a complete packet
// is transmitted by the module
static
Expand Down Expand Up @@ -40,7 +51,15 @@ void RadioLibWrapper::begin() {

// start average out some samples
_num_floor_samples = 0;
_floor_sample_sum = 0;
_floor_block_ready = false;
_last_floor_sample_at = 0;
_last_rx_start_ms = 0;
_held_block_count = 0;
_floor_hold_bypass = false;

_last_rx_sync_check = millis();
_rx_desync_streak = 0;
n_rx_desync_events = n_rx_desync_fatals = 0;
}

uint32_t RadioLibWrapper::getRngSeed() {
Expand All @@ -61,9 +80,9 @@ void RadioLibWrapper::idle() {

void RadioLibWrapper::triggerNoiseFloorCalibrate(int threshold) {
_threshold = threshold;
if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES) { // ignore trigger if currently sampling
if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES) { // restart only once the current block is complete
_num_floor_samples = 0;
_floor_sample_sum = 0;
_floor_block_ready = false;
}
}

Expand All @@ -78,34 +97,158 @@ void RadioLibWrapper::resetAGC() {
doResetAGC();
state = STATE_IDLE; // trigger a startReceive()

// Reset noise floor sampling so it reconverges from scratch.
// Without this, a stuck _noise_floor of -120 makes the sampling threshold
// too low (-106) to accept normal samples (~-105), self-reinforcing the
// stuck value even after the receiver has recovered.
_noise_floor = 0;
// Discard any in-progress noise-floor block: the analog frontend was just reset, so
// queued RSSI samples are stale. _noise_floor itself is left in place — the median
// estimator no longer drifts to -120 (the reason the old ratchet needed a hard
// _noise_floor = 0 reset), and forcing 0 here would create a brief permissive LBT
// window (margin = RSSI - 0) until the next block completes.
_num_floor_samples = 0;
_floor_sample_sum = 0;
_floor_block_ready = false;
_held_block_count = 0; // contamination context is stale after an AFE reset
}

void RadioLibWrapper::loop() {
if (state == STATE_RX && _num_floor_samples < NUM_NOISE_FLOOR_SAMPLES) {
if (!isReceivingPacket()) {
int rssi = getCurrentRSSI();
if (rssi < _noise_floor + SAMPLING_THRESHOLD) { // only consider samples below current floor + sampling THRESHOLD
_num_floor_samples++;
_floor_sample_sum += rssi;
// _rx_desync_streak == 0: while a desync is unresolved the RSSI register reads a
// wedged/frozen receiver, so sampling is paused entirely (the block in flight at
// detection was already discarded). Without this gate, blocks completed between the
// 10 s verification polls publish garbage (a block spans only ~3.2 s).
if (state == STATE_RX && _rx_desync_streak == 0 && _num_floor_samples < NUM_NOISE_FLOOR_SAMPLES) {
uint32_t now = millis();
if (!isReceivingPacket() && now - _last_floor_sample_at >= NOISE_FLOOR_SAMPLE_INTERVAL_MS
&& now - _last_rx_start_ms >= NOISE_FLOOR_RX_START_SETTLE_MS) {
// Accept every idle sample, spaced NOISE_FLOOR_SAMPLE_INTERVAL_MS apart so the block spans a real
// ~3.2 s window and the median rejects transient transmissions (not a few-ms snapshot). The old
// "rssi < floor + threshold" filter was a one-way ratchet: it only accepted samples below the
// current floor, so the block average drifted to the -120 clamp and never recovered — leaving
// _noise_floor stuck low and the RSSI-margin LBT permanently over-sensitive.
_floor_samples[_num_floor_samples++] = (int16_t)getCurrentRSSI();
_last_floor_sample_at = now;
}
} else if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES && !_floor_block_ready) {
// Publish nothing sampled while the chip was out of RX: verify the chip mode at
// block completion (one extra status read per block). A wedge is then caught at
// the FIRST completing block instead of at the next verification poll — samples
// taken while wedged read a frozen/clamped RSSI register and must not reach the
// LBT reference. On veto the block is discarded and the desync streak starts, so
// the poll path re-arms the receiver on its next tick.
if (!verifyRxChipMode()) {
if (_rx_desync_streak == 0) {
n_rx_desync_events++;
MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync - block publish vetoed (chip reports mode != RX)");
}
_rx_desync_streak++;
_num_floor_samples = 0;
_floor_block_ready = false;
_held_block_count = 0;
return;
}
} else if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES && _floor_sample_sum != 0) {
_noise_floor = _floor_sample_sum / NUM_NOISE_FLOOR_SAMPLES;
if (_noise_floor < -120) {
_noise_floor = -120; // clamp to lower bound of -120dBi
// Block complete: reduce to the median. The median rejects transient interference
// spikes (high and low outliers) and recovers in BOTH directions, unlike the ratcheted
// mean. _noise_floor is written only here, so the previous value stays valid while the
// next block is sampled — no reset-to-0, no permissive LBT window during reconvergence.
sortInt16(_floor_samples, NUM_NOISE_FLOOR_SAMPLES);
int16_t median = (int16_t)(((int32_t)_floor_samples[NUM_NOISE_FLOOR_SAMPLES / 2 - 1]
+ (int32_t)_floor_samples[NUM_NOISE_FLOOR_SAMPLES / 2]) / 2);
// One-sided hold: a median jumping far ABOVE the published floor is activity-contaminated
// (inter-packet energy slips past the !isReceivingPacket() idle guard). Hold the old value so
// the RSSI-margin LBT stays meaningful under load; near-stable/quieter blocks publish at once.
// First block always publishes (_noise_floor=0 from begin()), so the hold binds only post-boot.
//
// Bounded: after NOISE_FLOOR_MAX_HELD_BLOCKS consecutive held blocks accept the median, else a real
// permanent rise is held forever (stuck-floor bug from the other direction). Count-based so the hold
// rides out load bursts (slow blocks) while a quiet rise releases in a few blocks.
if (_floor_hold_bypass) {
// First block after a resolved RX-desync episode: publish unconditionally.
// The published floor survived the episode and is known-stale, so the
// contamination hold must not gate the recovery measurement — otherwise the
// correct value is rejected as a "contaminated rise" and the node keeps an
// LBT reference that is off by tens of dB for several more blocks.
_floor_hold_bypass = false;
_held_block_count = 0;
_noise_floor = median;
if (_noise_floor < -120) {
_noise_floor = -120; // clamp to lower bound of -120dBi
}
#ifdef MESH_DEBUG_NOISE_FLOOR
MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor = %d (post-desync recovery)", (int)_noise_floor);
#endif
} else if (median > _noise_floor + NOISE_FLOOR_MAX_RISE_DB) {
_held_block_count++;
if (_held_block_count >= NOISE_FLOOR_MAX_HELD_BLOCKS) {
_noise_floor = median;
if (_noise_floor < -120) {
_noise_floor = -120; // clamp to lower bound of -120dBi
}
_held_block_count = 0;
#ifdef MESH_DEBUG_NOISE_FLOOR
MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor = %d (accepted after %d held blocks, persistent rise)",
(int)_noise_floor, NOISE_FLOOR_MAX_HELD_BLOCKS);
#endif
} else {
#ifdef MESH_DEBUG_NOISE_FLOOR
MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor held at %d (block median %d contaminated, held %d/%d)",
(int)_noise_floor, (int)median, _held_block_count, NOISE_FLOOR_MAX_HELD_BLOCKS);
#endif
}
} else {
_held_block_count = 0;
_noise_floor = median;
if (_noise_floor < -120) {
_noise_floor = -120; // clamp to lower bound of -120dBi
}
#ifdef MESH_DEBUG_NOISE_FLOOR
MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor = %d (median)", (int)_noise_floor);
#endif
}
_floor_sample_sum = 0;
_floor_block_ready = true;
}

#ifdef MESH_DEBUG_NOISE_FLOOR
MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor = %d", (int)_noise_floor);
#endif
// --- RX-desync watchdog ---
// `state` is firmware-side truth. If the chip silently leaves RX (supply dip during
// TX, SPI glitch, front-end upset), the RAM copy still says STATE_RX, so recvRaw()
// never re-arms and the Dispatcher-side 8 s check — reading the same variable —
// stays quiet: the node goes deaf until reboot. (Visible symptom: the Current-RSSI
// register freezes at the last energy seen, pinning the noise floor high.)
// Here we ask the CHIP instead: verifyRxChipMode() reads its real operating mode.
// Radio types without an authoritative status register report true (watchdog off).
// Recovery: after RX_DESYNC_CONFIRM_TICKS bad polls, re-arm the receiver; if that
// doesn't take, escalate to a warm sleep (resets the modem state machine/AFE).
// A streak that survives both is counted fatal and surfaces as ERR_EVENT_RX_DESYNC.
if (state == STATE_RX) {
uint32_t now = millis();
if (now - _last_rx_sync_check >= RX_DESYNC_CHECK_INTERVAL_MS) {
_last_rx_sync_check = now;
if (!isReceivingPacket() && !verifyRxChipMode()) {
if (_rx_desync_streak == 0) {
n_rx_desync_events++;
MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync - chip reports mode != RX");
}
_rx_desync_streak++;
// samples taken while wedged read a frozen RSSI register: discard the block
_num_floor_samples = 0;
_floor_block_ready = false;
_held_block_count = 0;
if (_rx_desync_streak == RX_DESYNC_CONFIRM_TICKS) {
MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync confirmed - re-arming receiver");
idle(); // standby, then a fresh startReceive()
startRecv();
} else if (_rx_desync_streak > RX_DESYNC_CONFIRM_TICKS) {
if (_rx_desync_streak == RX_DESYNC_FATAL_STREAK) {
n_rx_desync_fatals++;
MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync survived AFE reset - radio damaged, reboot suggested");
}
// warm sleep resets the analog frontend and modem state machine;
// resetAGC() re-arms from STATE_IDLE via recvRaw() on the next pass
resetAGC();
}
} else if (_rx_desync_streak > 0) {
MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync resolved after %u ticks", _rx_desync_streak);
_rx_desync_streak = 0;
// sampling resumes with the next pass; the first completed block publishes
// unconditionally (floor known-stale)
_floor_hold_bypass = true;
}
}
}
}

Expand All @@ -116,6 +259,7 @@ void RadioLibWrapper::startRecv() {
int err = _radio->startReceive();
if (err == RADIOLIB_ERR_NONE) {
state = STATE_RX;
_last_rx_start_ms = millis(); // floor sampling settles before the AGC has re-converged after a restart
} else {
MESH_DEBUG_PRINTLN("RadioLibWrapper: error: startReceive(%d)", err);
}
Expand Down
Loading