From ce91664fe7b32ef6fa7ea206a4147279d9cceed3 Mon Sep 17 00:00:00 2001 From: Sean Mancini Date: Thu, 5 Nov 2026 23:36:23 -0500 Subject: [PATCH 1/2] fix(polling): clear a recovered device's failed row once handled (#737) thold_update_host_status() announces every device in plugin_thold_host_failed that is up again, but the table was only cleared and rebuilt after all recovered devices had been handled. Each device is handled synchronously (up to four SNMP gets, a blocking exec() of the up command, and mail), so with many recoveries the run can outlast the poller cycle. When it was cut short before the rebuild, every row stayed and the next cycle sent the same recovery notices again. Delete each recovered device's row, by its row id, as soon as its recovery has been handled. The end-of-run rebuild is unchanged and still records devices that are down or recovering. The in-memory $failed list it consults for recovering devices is unaffected. --- includes/polling.php | 12 ++ tests/Unit/RecoveryNoticeRepeatTest.php | 157 ++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 tests/Unit/RecoveryNoticeRepeatTest.php diff --git a/includes/polling.php b/includes/polling.php index 202ed2fd..f701a4be 100644 --- a/includes/polling.php +++ b/includes/polling.php @@ -732,6 +732,18 @@ function thold_update_host_status() { } } } + + /** + * This device's recovery has been handled, so drop its row now + * rather than in the rebuild at the end of the run. Each device + * can block on SNMP, an up command and mail; if the run is cut + * short before the rebuild, every device left in the table is + * announced as recovered again on the next cycle (#737). + */ + db_execute_prepared('DELETE + FROM plugin_thold_host_failed + WHERE id = ?', + [$fh['id']]); } } } diff --git a/tests/Unit/RecoveryNoticeRepeatTest.php b/tests/Unit/RecoveryNoticeRepeatTest.php new file mode 100644 index 00000000..c54e05bd --- /dev/null +++ b/tests/Unit/RecoveryNoticeRepeatTest.php @@ -0,0 +1,157 @@ + + */ + private function recoveredDevice($id) { + return [ + 'id' => $id, + 'hostname' => 'dev' . $id, + 'description' => 'Device ' . $id, + 'status' => HOST_UP, + 'snmp_community' => '', + 'snmp_username' => '', + 'snmp_version' => 0, + 'site_id' => 0, + 'location' => '', + 'notes' => '', + 'status_fail_date' => '0000-00-00 00:00:00', + 'status_rec_date' => '0000-00-00 00:00:00', + 'status_last_error' => '', + 'availability' => 100, + 'total_polls' => 1, + 'failed_polls' => 0, + 'cur_time' => 0, + 'avg_time' => 0, + 'thold_send_email' => '1', + 'thold_host_email' => 0, + ]; + } + + /** + * Two devices in the failed-host table, rows 5 and 6, both up again. The + * host lookup answers in turn from $devices; a Throwable stands for the run + * dying at that device. + * + * @param array $devices + * + * @return void + */ + private function runHostStatus(array $devices) { + CactiStubs::$configOptions['alert_deadnotify'] = 'on'; + CactiStubs::$configOptions['alert_email'] = 'ops@example.org'; + + CactiStubs::willReturnFor('db_fetch_assoc', 'plugin_thold_host_failed', [ + ['id' => 5, 'poller_id' => 1, 'host_id' => 42], + ['id' => 6, 'poller_id' => 1, 'host_id' => 43], + ]); + + CactiStubs::willReturnFor('db_fetch_row_prepared', 'FROM host WHERE id', function () use (&$devices) { + $device = array_shift($devices); + + if ($device instanceof Throwable) { + throw $device; + } + + return $device; + }); + + set_error_handler(static fn (): bool => true); + ob_start(); + + try { + thold_update_host_status(); + } finally { + ob_end_clean(); + restore_error_handler(); + } + } + + /** + * Failed-host rows deleted one at a time, by row id. + * + * @return array + */ + private function rowsDeleted() { + $ids = []; + + foreach (CactiStubs::callsTo('db_execute_prepared') as $call) { + if (preg_match('/DELETE\s+FROM plugin_thold_host_failed\s+WHERE id = \?/', $call['sql'])) { + $ids[] = $call['params'][0]; + } + } + + return $ids; + } + + /** + * The run dies at the second device. The first device's recovery was + * mailed, and its row is already gone, so the next cycle will not announce + * it again. + * + * @return void + */ + public function testAHandledRecoveryIsClearedBeforeTheNextDevice(): void { + try { + $this->runHostStatus([$this->recoveredDevice(42), new RuntimeException('poller run cut short')]); + + $this->fail('the run was expected to stop at the second device'); + } catch (RuntimeException $e) { + $this->assertSame('poller run cut short', $e->getMessage()); + } + + $this->assertCount(1, CactiStubs::$mail); + $this->assertSame([5], $this->rowsDeleted()); + } + + /** + * A full run clears each handled row as it goes. + * + * @return void + */ + public function testEveryHandledRecoveryIsCleared(): void { + $this->runHostStatus([$this->recoveredDevice(42), $this->recoveredDevice(43)]); + + $this->assertCount(2, CactiStubs::$mail); + $this->assertSame([5, 6], $this->rowsDeleted()); + } +} From a951795b29b83f38e5e68c86c073db4d461f5a68 Mon Sep 17 00:00:00 2001 From: Sean Mancini Date: Fri, 6 Nov 2026 01:10:51 -0500 Subject: [PATCH 2/2] fix(polling): drop the failed row once the recovery notice is sent (#737) The first change deleted a recovered device's row only after all of its work, including the blocking up command. Validating on a real poller showed a run cut short inside that command still repeated the device's recovery notice next cycle, which is the case #737 describes. Delete the row right after the recovery notice is sent, before the up command. A run that dies in the command no longer repeats the notice; a run that dies before the notice was sent (for example in the SNMP gets) keeps the row, so the notice is sent on the next cycle instead. Adds a test for a run cut short inside the up command; it fails with the previous placement. --- includes/polling.php | 28 ++++++++++--------- tests/Unit/RecoveryNoticeRepeatTest.php | 36 +++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/includes/polling.php b/includes/polling.php index f701a4be..b18b7d73 100644 --- a/includes/polling.php +++ b/includes/polling.php @@ -679,6 +679,22 @@ function thold_update_host_status() { thold_mail($alert_email, '', '', $subject, $msg, '', '', $notify_list_id, $host, '', 7, 'thold_uhost_mail'); } + /** + * The recovery notice has gone out, so drop this device's row + * now rather than in the rebuild at the end of the run (#737). + * Each device can block on SNMP gets, the up command below and + * mail; a run cut short before the rebuild used to announce + * every device left in the table again on the next cycle. + * Deleting before the up command means a run that dies inside + * the command still does not repeat the notice; a run that dies + * before the notice was sent keeps the row, so it is sent next + * cycle instead. + */ + db_execute_prepared('DELETE + FROM plugin_thold_host_failed + WHERE id = ?', + [$fh['id']]); + $command = read_config_option('thold_device_command'); if ($command != '') { @@ -732,18 +748,6 @@ function thold_update_host_status() { } } } - - /** - * This device's recovery has been handled, so drop its row now - * rather than in the rebuild at the end of the run. Each device - * can block on SNMP, an up command and mail; if the run is cut - * short before the rebuild, every device left in the table is - * announced as recovered again on the next cycle (#737). - */ - db_execute_prepared('DELETE - FROM plugin_thold_host_failed - WHERE id = ?', - [$fh['id']]); } } } diff --git a/tests/Unit/RecoveryNoticeRepeatTest.php b/tests/Unit/RecoveryNoticeRepeatTest.php index c54e05bd..8771ba3d 100644 --- a/tests/Unit/RecoveryNoticeRepeatTest.php +++ b/tests/Unit/RecoveryNoticeRepeatTest.php @@ -143,6 +143,42 @@ public function testAHandledRecoveryIsClearedBeforeTheNextDevice(): void { $this->assertSame([5], $this->rowsDeleted()); } + /** + * The run dies inside the first device's up command, as on a poller that + * outlasts its cycle while the command blocks. The notice already went + * out and the row is already gone, so the next cycle does not repeat it. + * (The queue path stands in for the command so nothing is executed.) + * + * @return void + */ + public function testARunThatDiesInTheUpCommandDoesNotRepeatTheNotice(): void { + CactiStubs::$configOptions['thold_device_command'] = '/bin/true'; + CactiStubs::$configOptions['thold_notification_queue'] = 'on'; + + // With the queue on, the recovery mail is queued first; the second + // insert is the up command. + $queued = 0; + + CactiStubs::willReturnFor('db_execute_prepared', 'INSERT INTO notification_queue', function () use (&$queued) { + if (++$queued == 2) { + throw new RuntimeException('poller run cut short in the up command'); + } + + return true; + }); + + try { + $this->runHostStatus([$this->recoveredDevice(42), $this->recoveredDevice(43)]); + + $this->fail('the run was expected to stop in the first up command'); + } catch (RuntimeException $e) { + $this->assertSame('poller run cut short in the up command', $e->getMessage()); + } + + $this->assertSame(2, $queued, 'the recovery mail and then the up command were queued'); + $this->assertSame([5], $this->rowsDeleted()); + } + /** * A full run clears each handled row as it goes. *