diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index d854645..490ac76 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -33,6 +33,7 @@ flowview/ # Repository root (install to plugins/fl ├── themes/ # CSS theme overlays ├── arrays.php # Configuration arrays ├── config.php.dist # Template for dedicated flowview database config +├── config.php # Customer copy of config.php.dist (required to complete install; not shipped, whitelisted) ├── database.php # flowview_db_* wrapper (dual-database support) ├── flowview.php # Main NetFlow viewer UI ├── flowview_devices.php # Device/exporter administration @@ -178,7 +179,7 @@ api_plugin_register_realm('flowview', 'flowview_devices.php,flowview_schedules.p ``` ### Required Configuration Before Install -`plugin_flowview_install()`/`plugin_flowview_check_config()` require either `config.php` or `config_local.php` to exist (derived from `config.php.dist`) before the database is set up; otherwise installation is blocked with `raise_message(..., MESSAGE_LEVEL_ERROR)`. +`plugin_flowview_install()`/`plugin_flowview_check_config()` require either `config.php` or `config_local.php` to exist (derived from `config.php.dist`) before the database is set up; otherwise installation is blocked with `raise_message(..., MESSAGE_LEVEL_ERROR)`. To complete a normal install, customers MUST copy `config.php.dist` to `config.php` (the defaults use the Cacti database) in the plugin directory before enabling the plugin. Both files are expected on a normal install: `config.php.dist` is the shipped template, and the copied `config.php` is customer-owned configuration. `config.php` is therefore not committed to git and is listed under `whitelist` in `manifest.json` so upgrade pruning never touches it (and the manifest validator never requires it on disk). ### Flow Collector Service `flow_collector.php` runs as a long-lived daemon (see `service/`) that decodes NetFlow v5/v9/v10 packets and writes them via the `flowview_db_*` layer; be mindful of performance (avoid per-packet queries) when touching this file. @@ -250,4 +251,4 @@ existing code or adding new code, not just in dedicated cleanup passes: ## File manifest & upgrade pruning -The plugin ships a root `manifest.json` with three arrays: `tombstones` (files/directories older versions shipped that have since moved or been removed), `expected` (the top-level files and directories that ship today, directories written with a trailing `/`), and `whitelist` (paths holding user data that must never be touched). Keep `expected` current: CI runs `tests/bin/validate-manifest.php`, which fails on any drift between `expected` and the real top-level tree (it ignores `tests/`, `phpunit.xml`, `.git*`, `.md*`, and whitelisted paths). Custom customer CSS/theme files belong in `expected`, and stylesheets live in `css/` (not `themes/`). On upgrade, `flowview_prune_files()` deletes the tombstoned paths, the dev-only `tests/` tree, and the `phpunit.xml` test config, leaves `whitelist`, `.git*`, and `.md*` alone, and logs (without removing) any top-level entry the manifest does not account for. As a safety measure it refuses any tombstone that resolves outside the plugin directory (a tampered manifest.json) and logs a warning for any file or directory it cannot remove. When you move or delete a shipped file, add its old path to `tombstones` and update `expected` in the same change. +The plugin ships a root `manifest.json` with three arrays: `tombstones` (files/directories older versions shipped that have since moved or been removed), `expected` (the top-level files and directories that ship today, directories written with a trailing `/`), and `whitelist` (paths holding user data that must never be touched, e.g. the customer-copied `config.php`). Keep `expected` current: CI runs `tests/bin/validate-manifest.php`, which fails on any drift between `expected` and the real top-level tree (it ignores `tests/`, `phpunit.xml`, `.git*`, `.md*`, and whitelisted paths). Custom customer CSS/theme files belong in `expected`, and stylesheets live in `css/` (not `themes/`). On upgrade, `flowview_prune_files()` deletes the tombstoned paths, the dev-only `tests/` tree, and the `phpunit.xml` test config, leaves `whitelist`, `.git*`, and `.md*` alone, and logs (without removing) any top-level entry the manifest does not account for. As a safety measure it refuses any tombstone that resolves outside the plugin directory (a tampered manifest.json) and logs a warning for any file or directory it cannot remove. When you move or delete a shipped file, add its old path to `tombstones` and update `expected` in the same change. diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a3c134..0a5dcb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ --- develop --- +* issue#308: Launch the background database upgrade only once and raise the "upgrading in the background" notice a single time, instead of relaunching the CLI and repeating the notice on every Plugins page visit while the recorded version still lags the code. A crashed upgrade past its process timeout is relaunched, and the CLI clears the notice and records the new version when it finishes. Whitelist the customer-created config.php in manifest.json so upgrade pruning never touches it, and run flowview_upgrade.php via __DIR__ so it works when invoked from outside the Cacti root * issue#306: Match flows that start before and end after the report time window; flows are still read only from the partitions inside that window * dev: Run the CI poller with the shared runner group so RRD files and structured directories pass permission verification * issue: Remove the Listener and Stream from the report page. They are part of what a Filter materializes, so changing them there ran the report against data that was never materialized. They are now set only in the Filter editor, and the report page reads them straight from the saved Filter. The New button now follows Edit. diff --git a/flowview_upgrade.php b/flowview_upgrade.php index 15c5d9a..8ea5e8e 100644 --- a/flowview_upgrade.php +++ b/flowview_upgrade.php @@ -23,7 +23,7 @@ +-------------------------------------------------------------------------+ */ -chdir('../../'); +chdir(__DIR__ . '/../../'); require('./include/cli_check.php'); require_once('./plugins/flowview/includes/functions.php'); require_once('./plugins/flowview/setup.php'); @@ -98,6 +98,14 @@ flowview_upgrade($current_version, $old_version); +// Remove files tombstoned in manifest.json plus the dev-only tests/ tree. +flowview_prune_files(); + +// Clear the operator notice flag so a later upgrade can raise it again. The +// web hook (flowview_launch_background_upgrade()) stops relaunching this CLI +// once flowview_upgrade() records the new version in plugin_config. +set_config_option('flowview_db_upgrade_notified', 'off'); + unregister_process('flowview', 'upgrade', 1); exit(0); diff --git a/manifest.json b/manifest.json index 0db1888..ad98f92 100644 --- a/manifest.json +++ b/manifest.json @@ -44,5 +44,8 @@ "service/", "setup.php" ], - "whitelist": [] + "whitelist": [ + "config.php", + "config_local.php" + ] } diff --git a/setup.php b/setup.php index 9a65563..5299c02 100644 --- a/setup.php +++ b/setup.php @@ -162,17 +162,105 @@ function plugin_flowview_check_upgrade(bool $force = false): void { WHERE directory = ?', array('flowview')); if ($current != $old || $force) { - $php_binary = read_config_option('path_php_binary'); + flowview_launch_background_upgrade($force); + } +} - exec_background($php_binary, '-q ' .$config['base_path'] . '/plugins/flowview/flowview_upgrade.php'); +/** + * Launches the background upgrade CLI (flowview_upgrade.php) once and raises a + * single operator notice while it runs, rather than relaunching it and + * re-raising the notice on every page load. The CLI's register_process_start() + * is the authoritative double-run guard; this only avoids a needless relaunch + * and keeps the notice from repeating. The CLI records the new plugin version + * and clears the notice flag when it finishes, which stops this hook from + * relaunching it. + * + * @param bool $force Launch even when an upgrade run is already registered. + * + * @return void + * + * @global array $config Cacti global configuration array; used to locate the + * CLI script. + */ +function flowview_launch_background_upgrade(bool $force = false): void { + global $config; + + // Persist the notice flag before spawning the CLI, and raise the notice + // only once. A fast-finishing child clears this flag when it completes; + // writing it here first keeps this request from overwriting that completion + // write, which would leave the next upgrade without its operator notice. + if (read_config_option('flowview_db_upgrade_notified', true) != 'on') { + set_config_option('flowview_db_upgrade_notified', 'on'); raise_message('flowview_upgrade', __('Please be advised the Flowview plugins Tables are being upgraded in the background. This may take some time. Check the Cacti log for more information'), MESSAGE_LEVEL_INFO); + } - // Remove files tombstoned in manifest.json plus the dev-only tests/ tree. - flowview_prune_files(); + if ($force || !flowview_background_upgrade_running()) { + $php_binary = read_config_option('path_php_binary'); + + if ($php_binary != '') { + exec_background($php_binary, '-q ' . cacti_escapeshellarg($config['base_path'] . '/plugins/flowview/flowview_upgrade.php')); + } } } +/** + * Reports whether a background FlowView upgrade process is currently + * registered and is still alive. The web launcher uses it to avoid spawning a + * second copy and re-raising the operator notice on every page load. A run + * that outlived its timeout, or whose recorded PID has exited, is treated as + * not running so the upgrade is relaunched rather than blocked for the rest of + * the 24-hour timeout. Matches the registration flowview_upgrade.php makes with + * register_process_start('flowview', 'upgrade', 1, 86400). + * + * @return bool True when a non-stale, live upgrade process is registered. + */ +function flowview_background_upgrade_running(): bool { + if (!db_table_exists('processes')) { + return false; + } + + $process = db_fetch_row_prepared('SELECT pid, + UNIX_TIMESTAMP(started) AS started, + UNIX_TIMESTAMP() AS `now`, + timeout + FROM processes + WHERE tasktype = ? + AND taskname = ? + AND taskid = 1', + ['flowview', 'upgrade']); + + if (!cacti_sizeof($process)) { + return false; + } + + // A registration that has outlived its timeout is stale. + if (((int) $process['now'] - (int) $process['started']) > (int) $process['timeout']) { + return false; + } + + // A crashed CLI can leave a fresh registration for a full day, so probe the + // recorded PID: a dead worker is recovered immediately. The CLI's own + // register_process_start() guard replaces the registration, so the web + // request only decides whether to relaunch and must not delete it. + $pid = (int) $process['pid']; + + if ($pid <= 0 || $pid > (PHP_OS_FAMILY === 'Windows' ? 4294967295 : 2147483647)) { + return false; + } + + if (function_exists('posix_kill') && !posix_kill($pid, 0)) { + // Only ESRCH proves the process exited. EPERM can mean the web server + // does not own the CLI process, and an unavailable probe is + // inconclusive too, so both are treated as still running. + $esrch = defined('SOCKET_ESRCH') ? SOCKET_ESRCH : 3; + + return !function_exists('posix_get_last_error') || posix_get_last_error() !== $esrch; + } + + return true; +} + /** * @return mixed */ diff --git a/tests/Unit/FlowviewBackgroundUpgradeTest.php b/tests/Unit/FlowviewBackgroundUpgradeTest.php new file mode 100644 index 0000000..81f7fd5 --- /dev/null +++ b/tests/Unit/FlowviewBackgroundUpgradeTest.php @@ -0,0 +1,132 @@ +toBe(array( + array('/usr/bin/php', "-q '/var/www/cacti/plugins/flowview/flowview_upgrade.php'"), + )); + + expect($GLOBALS['__test_messages'])->toHaveCount(1) + ->and($GLOBALS['__test_messages'][0][0])->toBe('flowview_upgrade'); + + expect($GLOBALS['__test_config']['flowview_db_upgrade_notified'])->toBe('on'); +}); + +it('does not relaunch or re-notify while a non-stale run is registered', function () { + $GLOBALS['__test_config']['flowview_db_upgrade_notified'] = 'on'; + + // A registered run that is well within its timeout, owned by a live PID + // (this test process is guaranteed alive). + plugin_test_queue_db_result('db_fetch_row_prepared', array( + 'pid' => getmypid(), + 'started' => 1000, + 'now' => 1050, + 'timeout' => 86400, + )); + + flowview_launch_background_upgrade(); + + expect($GLOBALS['__test_exec'])->toBe(array()) + ->and($GLOBALS['__test_messages'])->toBe(array()); +}); + +it('relaunches when the registered run has outlived its timeout', function () { + // started long enough ago that now - started exceeds timeout => stale. + plugin_test_queue_db_result('db_fetch_row_prepared', array( + 'started' => 1000, + 'now' => 1000 + 86400 + 60, + 'timeout' => 86400, + )); + + flowview_launch_background_upgrade(); + + expect($GLOBALS['__test_exec'])->toHaveCount(1) + ->and($GLOBALS['__test_exec'][0][1])->toContain('flowview_upgrade.php') + ->and($GLOBALS['__test_messages'])->toHaveCount(1); +}); + +it('force launches even when a non-stale run is registered', function () { + // A live run is registered, but --force must still relaunch. + plugin_test_queue_db_result('db_fetch_row_prepared', array( + 'started' => 1000, + 'now' => 1050, + 'timeout' => 86400, + )); + + flowview_launch_background_upgrade(true); + + expect($GLOBALS['__test_exec'])->toHaveCount(1); +}); + +it('treats a missing processes table as not running', function () { + $GLOBALS['__test_tables']['processes'] = false; + + expect(flowview_background_upgrade_running())->toBeFalse(); +}); + +it('reports a non-stale registered run as running', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array( + 'pid' => getmypid(), + 'started' => 1000, + 'now' => 1050, + 'timeout' => 86400, + )); + + expect(flowview_background_upgrade_running())->toBeTrue(); +}); + +it('recovers a non-stale registration whose worker PID has exited', function () { + if (!function_exists('posix_kill')) { + $this->markTestSkipped('posix_kill is required to probe PID liveness'); + } + + // A registration well within its timeout but owned by a PID that is not + // running must be treated as not running so the upgrade is relaunched + // instead of being blocked for the rest of the 24-hour timeout. + plugin_test_queue_db_result('db_fetch_row_prepared', array( + 'pid' => 2147483646, + 'started' => 1000, + 'now' => 1050, + 'timeout' => 86400, + )); + + expect(flowview_background_upgrade_running())->toBeFalse(); +}); diff --git a/tests/bootstrap-unit.php b/tests/bootstrap-unit.php index d3f1589..d9f4821 100644 --- a/tests/bootstrap-unit.php +++ b/tests/bootstrap-unit.php @@ -88,6 +88,7 @@ $GLOBALS['__test_db_results'] = array(); $GLOBALS['__test_config'] = array(); $GLOBALS['__test_messages'] = array(); +$GLOBALS['__test_tables'] = array(); if (!function_exists('plugin_test_reset')) { function plugin_test_reset() { @@ -96,6 +97,7 @@ function plugin_test_reset() { $GLOBALS['__test_config'] = array(); $GLOBALS['__test_messages'] = array(); $GLOBALS['__test_exec'] = array(); + $GLOBALS['__test_tables'] = array(); $_GET = array(); $_POST = array(); $_REQUEST = array(); @@ -267,6 +269,12 @@ function read_config_option($name, $force = false) { } } +if (!function_exists('db_table_exists')) { + function db_table_exists($table, $log = true, $conn = false) { + return isset($GLOBALS['__test_tables'][$table]) ? (bool) $GLOBALS['__test_tables'][$table] : true; + } +} + if (!function_exists('set_config_option')) { function set_config_option($name, $value) { $GLOBALS['__test_config'][$name] = $value;