Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

--- 5.0 ---

* issue#440: Remember only a successful table creation (not updates of existing tables) when avoiding repeated schema operations, so schema drift on an existing table, such as a narrowed owner column, is still detected and repaired within the same request
* issue#451: Treat the stale hostField value 'host_id' (shipped in config_local.php.dist since 2022) as the syslog_incoming 'host' column, and correct config_local.php.dist, so the device-rule mute/maintenance filter and host alerts no longer fail with "Unknown column 'syslog_incoming.host_id'" during alert processing
* issue#452, issue#453: Resolve remote collector hostnames from the Cacti poller table over the Cacti connection instead of joining it on the Syslog connection, and tolerate a failed query, so the Syslog Status and Dashboard tabs no longer throw a TypeError that disabled the plugin when the Syslog tables live in a separate database

* issue#440: Avoid repeating successful schema operations and seed the default dashboard after schema setup, preventing duplicate table creation during installation and upgrade

* feature: Add an install-time timestamp precision choice for the logtime column (issue #325). New MariaDB installs default to millisecond TIMESTAMP(3), which preserves sub-second ordering and still RANGE partition-prunes; MySQL installs are fixed to second precision because MySQL cannot prune a fractional partitioned column, and an explicit millisecond request on MySQL is rejected. Upgrades never migrate the schema automatically; switch an existing MariaDB install with the destructive cli/updatedb_convert_ms.php utility during a maintenance window. README documents forwarder configuration (rsyslog, Cisco, Juniper, Arista, syslog-ng) for delivering sub-second timestamps
Expand Down
2 changes: 1 addition & 1 deletion config_local.php.dist
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,6 @@ $syslog_install_options['id'] = 'syslog';
$syslog_incoming_config['timeField'] = 'logtime';
$syslog_incoming_config['priorityField'] = 'priority_id';
$syslog_incoming_config['facilityField'] = 'facility_id';
$syslog_incoming_config['hostField'] = 'host_id';
$syslog_incoming_config['hostField'] = 'host';
$syslog_incoming_config['textField'] = 'message';
$syslog_incoming_config['id'] = 'seq';
58 changes: 48 additions & 10 deletions includes/functions.php
Original file line number Diff line number Diff line change
Expand Up @@ -3388,7 +3388,7 @@ function syslog_get_removal_rule_fields($table = 'syslog_incoming', $prefix = ''

return [
'message' => ['column' => $column($syslog_incoming_config['textField']), 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'host' => ['column' => $column($syslog_incoming_config['hostField']), 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'host' => ['column' => $column(syslog_incoming_host_field()), 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'program' => ['column' => $column($syslog_incoming_config['programField']), 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'facility_id' => ['column' => $column($syslog_incoming_config['facilityField']), 'operators' => ['=', '!='], 'type' => 'integer'],
'priority_id' => ['column' => $column($syslog_incoming_config['priorityField']), 'operators' => ['=', '!='], 'type' => 'integer']
Expand Down Expand Up @@ -3715,7 +3715,7 @@ function syslog_rule_preview(array $rule, string $rule_type = 'alert', int $rows

$incoming_fields = [
'facility' => $syslog_incoming_config['facilityField'],
'host' => $syslog_incoming_config['hostField'],
'host' => syslog_incoming_host_field(),
'program' => $syslog_incoming_config['programField'],
];

Expand Down Expand Up @@ -5279,6 +5279,23 @@ function syslog_alert_maintenance_is_active(array $alert): bool {
|| syslog_alert_schedule_is_active(syslog_alert_maintenance_window((string) read_config_option('syslog_alert_maintenance_days'), (string) read_config_option('syslog_alert_maintenance_start'), (string) read_config_option('syslog_alert_maintenance_end')));
}

/**
* Return the column that holds the hostname in the syslog_incoming table.
*
* Releases before 4.11 shipped config_local.php.dist with hostField set to
* 'host_id', which is not a syslog_incoming column; the staging table has
* always stored the hostname in 'host'. Treat that stale value as 'host'.
*
* @return string
*/
function syslog_incoming_host_field(): string {
global $syslog_incoming_config;

$field = $syslog_incoming_config['hostField'] ?? 'host';

return ($field === '' || $field === 'host_id') ? 'host' : (string) $field;
}

/**
* Limit an alert query by active device-wide handling rules.
*
Expand All @@ -5290,7 +5307,7 @@ function syslog_alert_maintenance_is_active(array $alert): bool {
*/
function syslog_device_rule_sql(bool $maintenance_active): array {
global $syslogdb_default, $syslog_incoming_config;
$host_field = $syslog_incoming_config['hostField'] ?? 'host';
$host_field = syslog_incoming_host_field();
$priority_field = $syslog_incoming_config['priorityField'] ?? 'priority_id';
$now = time();
// Qualify the outer staging columns explicitly: an unqualified `host`
Expand Down Expand Up @@ -6241,7 +6258,7 @@ function syslog_get_alert_sql(&$alert, $max_seq, $processing_boundary = true, $c
}
$fields = [
'message' => ['column' => '`' . $syslog_incoming_config['textField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'host' => ['column' => '`' . $syslog_incoming_config['hostField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'host' => ['column' => '`' . syslog_incoming_host_field() . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'program' => ['column' => '`' . $syslog_incoming_config['programField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'facility_id' => ['column' => '`' . $syslog_incoming_config['facilityField'] . '`', 'operators' => ['=', '!='], 'type' => 'integer'],
'priority_id' => ['column' => '`' . $syslog_incoming_config['priorityField'] . '`', 'operators' => ['=', '!='], 'type' => 'integer']
Expand All @@ -6265,7 +6282,7 @@ function syslog_get_alert_sql(&$alert, $max_seq, $processing_boundary = true, $c
case 'messagee':
return ['sql' => "`{$syslog_incoming_config['textField']}` LIKE ?", 'params' => ['%' . $alert['message']]];
case 'host':
return ['sql' => "`{$syslog_incoming_config['hostField']}` = ?", 'params' => [$alert['message']]];
return ['sql' => '`' . syslog_incoming_host_field() . '` = ?', 'params' => [$alert['message']]];
case 'program':
return ['sql' => "`{$syslog_incoming_config['programField']}` = ?", 'params' => [$alert['message']]];
case 'sql':
Expand All @@ -6281,7 +6298,7 @@ function syslog_get_alert_sql(&$alert, $max_seq, $processing_boundary = true, $c
}
$fields = [
'message' => ['column' => '`' . $syslog_incoming_config['textField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'host' => ['column' => '`' . $syslog_incoming_config['hostField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'host' => ['column' => '`' . syslog_incoming_host_field() . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'program' => ['column' => '`' . $syslog_incoming_config['programField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']],
'facility_id' => ['column' => '`' . $syslog_incoming_config['facilityField'] . '`', 'operators' => ['=', '!='], 'type' => 'integer'],
'priority_id' => ['column' => '`' . $syslog_incoming_config['priorityField'] . '`', 'operators' => ['=', '!='], 'type' => 'integer']
Expand Down Expand Up @@ -6347,7 +6364,7 @@ function syslog_get_alert_sql(&$alert, $max_seq, $processing_boundary = true, $c
} elseif ($alert['type'] == 'host') {
$sql = "SELECT *
FROM `$syslogdb_default`.`syslog_incoming`
WHERE `{$syslog_incoming_config['hostField']}` = ?
WHERE `" . syslog_incoming_host_field() . "` = ?
AND `status` = 1
AND `seq` <= ?";

Expand Down Expand Up @@ -7039,12 +7056,33 @@ function syslog_replication_collector_status(): array {
return [];
}

return syslog_db_fetch_assoc("SELECT rc.source_poller_id, p.hostname,
$collectors = syslog_db_fetch_assoc("SELECT rc.source_poller_id,
rc.last_batch_count, UNIX_TIMESTAMP(rc.last_received) AS last_received
FROM `$syslogdb_default`.`syslog_replication_collectors` AS rc
LEFT JOIN poller AS p
ON p.id = rc.source_poller_id
ORDER BY rc.last_received DESC, rc.source_poller_id ASC", false);

if (!is_array($collectors) || !cacti_sizeof($collectors)) {
return [];
}

// The poller table lives in the Cacti database, which may differ from the
// dedicated Syslog database, so resolve hostnames over the Cacti connection.
$poller_ids = array_values(array_unique(array_map('intval', array_column($collectors, 'source_poller_id'))));
$hostnames = [];
$pollers = db_fetch_assoc_prepared('SELECT id, hostname FROM poller WHERE id IN (' . implode(',', array_fill(0, cacti_sizeof($poller_ids), '?')) . ')', $poller_ids, false);

if (is_array($pollers)) {
foreach ($pollers as $poller) {
$hostnames[(int) $poller['id']] = $poller['hostname'];
}
}

foreach ($collectors as &$collector) {
$collector['hostname'] = $hostnames[(int) $collector['source_poller_id']] ?? null;
}
unset($collector);

return $collectors;
}

/** Conservative online delivery limit. Payloads may contain 2KiB messages. */
Expand Down
15 changes: 10 additions & 5 deletions includes/schema.php
Original file line number Diff line number Diff line change
Expand Up @@ -60,26 +60,31 @@ function syslog_db_update_table($table, $data, $removecolumns = false, $log = tr

static $applied = [];

// Core caches missing tables even after CREATE. Do not repeat an identical
// successful schema operation in this request (connect and upgrade can overlap).
// Core caches missing tables even after CREATE, so a second db_update_table()
// in this request would try to create the table again. Only a successful
// creation is remembered; updates of existing tables always run so schema
// drift is still repaired (connect and upgrade can overlap).
$connection = is_object($db_conn) ? $db_conn : $syslog_cnn;
$key = (is_object($connection) ? spl_object_hash($connection) : 'default') . ':' . $table;
$definition = [$connection, $data, $removecolumns];
$exists = syslog_db_table_exists($table, false);

if (isset($applied[$key]) && $applied[$key] === $definition && syslog_db_table_exists($table, false)) {
if ($exists && isset($applied[$key]) && $applied[$key] === $definition) {
return true;
}

if (isset($data['primary']) && is_array($data['primary'])
&& !syslog_cacti_db_array_primary_safe()
&& !syslog_db_table_exists($table, false)) {
&& !$exists) {
$data['primary'] = implode('`,`', $data['primary']);
}

$result = db_update_table($table, $data, $removecolumns, $log, $db_conn);

if ($result) {
if ($result && !$exists) {
Comment thread
TheWitness marked this conversation as resolved.
Resolved
$applied[$key] = $definition;
} else {
unset($applied[$key]);
}

return $result;
Expand Down
15 changes: 15 additions & 0 deletions tests/Unit/DeviceRulePolicyTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,21 @@
->toContain("syslog_replace_data(\$table, \$rows, \$data['rcnn_id'], \$syslog_cnn)");
});

it('maps the legacy host_id hostField to the incoming host column', function () {
syslog_load_plugin_source('includes/functions.php');
$GLOBALS['syslogdb_default'] = 'syslog';
$GLOBALS['syslog_incoming_config'] = ['hostField' => 'host_id', 'priorityField' => 'priority_id'];

expect(syslog_incoming_host_field())->toBe('host');

$filter = syslog_device_rule_sql(true);
expect($filter['sql'])->toContain('dr.host = `syslog_incoming`.`host`')
->not->toContain('`host_id`');

$GLOBALS['syslog_incoming_config'] = ['hostField' => 'src_host'];
expect(syslog_incoming_host_field())->toBe('src_host');
});

it('correlates device suppression and maintenance exceptions with the incoming message host', function () {
syslog_load_plugin_source('includes/functions.php');
$GLOBALS['syslogdb_default'] = 'syslog';
Expand Down
38 changes: 33 additions & 5 deletions tests/Unit/ReplicationDeliveryTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -53,17 +53,45 @@
$GLOBALS['config'] = ['poller_id' => 1];
$GLOBALS['syslogdb_default'] = 'cacti';
$sql = '';
$poller_sql = '';
test_override('syslog_db_table_exists', fn ($table) => $table === 'syslog_replication_collectors');
test_override('syslog_db_fetch_assoc', function ($statement) use (&$sql) { $sql = $statement; return [[
'source_poller_id' => 2, 'hostname' => 'remote-poller.example.com', 'last_batch_count' => 100, 'last_received' => 1758801600,
'source_poller_id' => 2, 'last_batch_count' => 100, 'last_received' => 1758801600,
]]; });
test_override('db_fetch_assoc_prepared', function ($statement) use (&$poller_sql) { $poller_sql = $statement; return [['id' => 2, 'hostname' => 'remote-poller.example.com']]; });
syslog_load_plugin_source('includes/functions.php');

expect(syslog_replication_collector_status())->toHaveCount(1)
$collectors = syslog_replication_collector_status();

expect($collectors)->toHaveCount(1)
->and($collectors[0]['hostname'])->toBe('remote-poller.example.com')
->and($sql)->toContain('syslog_replication_collectors')
->toContain('LEFT JOIN poller AS p')
->toContain('p.hostname')
->not->toContain('`.`syslog`');
->not->toContain('poller AS p')
->not->toContain('`.`syslog`')
->and($poller_sql)->toContain('FROM poller');
});

it('returns no collectors instead of a TypeError when the receipt query fails', function () {
$GLOBALS['config'] = ['poller_id' => 1];
$GLOBALS['syslogdb_default'] = 'syslog_db';
test_override('syslog_db_table_exists', fn () => true);
test_override('syslog_db_fetch_assoc', fn () => false);
syslog_load_plugin_source('includes/functions.php');

expect(syslog_replication_collector_status())->toBe([]);
});

it('keeps collectors listed when the poller hostname lookup fails', function () {
$GLOBALS['config'] = ['poller_id' => 1];
$GLOBALS['syslogdb_default'] = 'syslog_db';
test_override('syslog_db_table_exists', fn () => true);
test_override('syslog_db_fetch_assoc', fn () => [['source_poller_id' => 3, 'last_batch_count' => 5, 'last_received' => 1]]);
test_override('db_fetch_assoc_prepared', fn () => false);
syslog_load_plugin_source('includes/functions.php');

$collectors = syslog_replication_collector_status();

expect($collectors)->toHaveCount(1)->and($collectors[0]['hostname'])->toBeNull();
});

it('treats an unset poller ID as the Main Collector for receipt telemetry', function () {
Expand Down
21 changes: 21 additions & 0 deletions tests/Unit/SchemaPrimaryKeyCompatTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -118,3 +118,24 @@ function schema_primary_capture(string $cacti_version, bool $table_exists): arra
expect(syslog_db_update_table('syslog_repeat_probe', $definition, false, true, $connection))->toBeTrue();
expect($calls)->toBe(4);
});


it('always reaches db_update_table() for repeated updates of an existing table and propagates failures', function () {
syslog_load_plugin_source('includes/schema.php');
$GLOBALS['config']['cacti_version'] = '1.2.32';
$connection = new stdClass();
$results = [true, true, false];
$calls = 0;
test_override('syslog_db_table_exists', fn () => true);
test_override('db_update_table', function () use (&$calls, &$results) {
$calls++;
return array_shift($results);
});
$definition = ['columns' => [['name' => 'user', 'type' => 'varchar(50)']], 'primary' => ['id']];

expect(syslog_db_update_table('syslog_existing_probe', $definition, false, true, $connection))->toBeTrue();
expect(syslog_db_update_table('syslog_existing_probe', $definition, false, true, $connection))->toBeTrue();
expect($calls)->toBe(2);
expect(syslog_db_update_table('syslog_existing_probe', $definition, false, true, $connection))->toBeFalse();
expect($calls)->toBe(3);
});
Loading