diff --git a/includes/polling.php b/includes/polling.php index 202ed2fd..b18b7d73 100644 --- a/includes/polling.php +++ b/includes/polling.php @@ -679,6 +679,22 @@ function thold_update_host_status() { thold_mail($alert_email, '', '', $subject, $msg, '', '', $notify_list_id, $host, '', 7, 'thold_uhost_mail'); } + /** + * The recovery notice has gone out, so drop this device's row + * now rather than in the rebuild at the end of the run (#737). + * Each device can block on SNMP gets, the up command below and + * mail; a run cut short before the rebuild used to announce + * every device left in the table again on the next cycle. + * Deleting before the up command means a run that dies inside + * the command still does not repeat the notice; a run that dies + * before the notice was sent keeps the row, so it is sent next + * cycle instead. + */ + db_execute_prepared('DELETE + FROM plugin_thold_host_failed + WHERE id = ?', + [$fh['id']]); + $command = read_config_option('thold_device_command'); if ($command != '') { diff --git a/tests/Unit/RecoveryNoticeRepeatTest.php b/tests/Unit/RecoveryNoticeRepeatTest.php new file mode 100644 index 00000000..8771ba3d --- /dev/null +++ b/tests/Unit/RecoveryNoticeRepeatTest.php @@ -0,0 +1,193 @@ + + */ + private function recoveredDevice($id) { + return [ + 'id' => $id, + 'hostname' => 'dev' . $id, + 'description' => 'Device ' . $id, + 'status' => HOST_UP, + 'snmp_community' => '', + 'snmp_username' => '', + 'snmp_version' => 0, + 'site_id' => 0, + 'location' => '', + 'notes' => '', + 'status_fail_date' => '0000-00-00 00:00:00', + 'status_rec_date' => '0000-00-00 00:00:00', + 'status_last_error' => '', + 'availability' => 100, + 'total_polls' => 1, + 'failed_polls' => 0, + 'cur_time' => 0, + 'avg_time' => 0, + 'thold_send_email' => '1', + 'thold_host_email' => 0, + ]; + } + + /** + * Two devices in the failed-host table, rows 5 and 6, both up again. The + * host lookup answers in turn from $devices; a Throwable stands for the run + * dying at that device. + * + * @param array $devices + * + * @return void + */ + private function runHostStatus(array $devices) { + CactiStubs::$configOptions['alert_deadnotify'] = 'on'; + CactiStubs::$configOptions['alert_email'] = 'ops@example.org'; + + CactiStubs::willReturnFor('db_fetch_assoc', 'plugin_thold_host_failed', [ + ['id' => 5, 'poller_id' => 1, 'host_id' => 42], + ['id' => 6, 'poller_id' => 1, 'host_id' => 43], + ]); + + CactiStubs::willReturnFor('db_fetch_row_prepared', 'FROM host WHERE id', function () use (&$devices) { + $device = array_shift($devices); + + if ($device instanceof Throwable) { + throw $device; + } + + return $device; + }); + + set_error_handler(static fn (): bool => true); + ob_start(); + + try { + thold_update_host_status(); + } finally { + ob_end_clean(); + restore_error_handler(); + } + } + + /** + * Failed-host rows deleted one at a time, by row id. + * + * @return array + */ + private function rowsDeleted() { + $ids = []; + + foreach (CactiStubs::callsTo('db_execute_prepared') as $call) { + if (preg_match('/DELETE\s+FROM plugin_thold_host_failed\s+WHERE id = \?/', $call['sql'])) { + $ids[] = $call['params'][0]; + } + } + + return $ids; + } + + /** + * The run dies at the second device. The first device's recovery was + * mailed, and its row is already gone, so the next cycle will not announce + * it again. + * + * @return void + */ + public function testAHandledRecoveryIsClearedBeforeTheNextDevice(): void { + try { + $this->runHostStatus([$this->recoveredDevice(42), new RuntimeException('poller run cut short')]); + + $this->fail('the run was expected to stop at the second device'); + } catch (RuntimeException $e) { + $this->assertSame('poller run cut short', $e->getMessage()); + } + + $this->assertCount(1, CactiStubs::$mail); + $this->assertSame([5], $this->rowsDeleted()); + } + + /** + * The run dies inside the first device's up command, as on a poller that + * outlasts its cycle while the command blocks. The notice already went + * out and the row is already gone, so the next cycle does not repeat it. + * (The queue path stands in for the command so nothing is executed.) + * + * @return void + */ + public function testARunThatDiesInTheUpCommandDoesNotRepeatTheNotice(): void { + CactiStubs::$configOptions['thold_device_command'] = '/bin/true'; + CactiStubs::$configOptions['thold_notification_queue'] = 'on'; + + // With the queue on, the recovery mail is queued first; the second + // insert is the up command. + $queued = 0; + + CactiStubs::willReturnFor('db_execute_prepared', 'INSERT INTO notification_queue', function () use (&$queued) { + if (++$queued == 2) { + throw new RuntimeException('poller run cut short in the up command'); + } + + return true; + }); + + try { + $this->runHostStatus([$this->recoveredDevice(42), $this->recoveredDevice(43)]); + + $this->fail('the run was expected to stop in the first up command'); + } catch (RuntimeException $e) { + $this->assertSame('poller run cut short in the up command', $e->getMessage()); + } + + $this->assertSame(2, $queued, 'the recovery mail and then the up command were queued'); + $this->assertSame([5], $this->rowsDeleted()); + } + + /** + * A full run clears each handled row as it goes. + * + * @return void + */ + public function testEveryHandledRecoveryIsCleared(): void { + $this->runHostStatus([$this->recoveredDevice(42), $this->recoveredDevice(43)]); + + $this->assertCount(2, CactiStubs::$mail); + $this->assertSame([5, 6], $this->rowsDeleted()); + } +}