From cdbb53b48fb5c045cbf071e19059bbc3a7de95a0 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 29 Jul 2026 17:07:11 +0200 Subject: [PATCH] feat(graph_watchdog): param_drift detector (GRAPH_PARAM_DRIFT) Raises GRAPH_PARAM_DRIFT when a node parameter no longer matches its reference. The reference is either self-captured, the value the parameter had when the node armed, or pinned in config with expect. Only the pinned form catches a value that was already wrong at startup, because self-capture treats whatever it first sees as correct. The detector reports that a value moved and does not grade the consequence. Whether the new value breaks the robot is a question about the machine, not about the graph, so it raises at WARN. Unlike the detectors shipped so far, this one makes real service calls to other nodes instead of reading the local graph cache, so the sweep is bounded by max_reads_per_tick and walks the graph round-robin. That bounds the load on watched nodes and pays with coverage latency, in both directions: a drift is invisible until its node's turn comes, and so is a repair. The reads run on a thread the plugin owns, never on the gateway executor, so a node that stops answering stalls only this sweep. Also adds the config-plumbing e2e, three real gateway launches that prove nested per-detector config reaches a detector at all. Every C++ test calls configure() directly with hand-built JSON, so nothing else in the package covers the delivery path. The three launches cover the nested expect reader, mode: "off" as a string, and the bare mode: off that the ROS parser types as a YAML 1.1 boolean, which is the form operators write. Restores the healing-config section the README referred to twice without carrying, since it belongs with the first detector whose clear story depends on it. Raise the Pixi job timeout from 45 to 60 minutes. Its build step alone is about 32 minutes and the job has been finishing at 37-40 minutes for weeks, so it had no room left for a package that grows. --- .github/workflows/pixi.yml | 4 +- .github/workflows/quality.yml | 30 +- .../ros2_medkit_graph_watchdog/CMakeLists.txt | 114 +- .../ros2_medkit_graph_watchdog/README.md | 248 +- .../design/graph_watchdog.rst | 125 +- .../aggregated_fault.hpp | 7 +- .../ros2_medkit_graph_watchdog/detector.hpp | 23 +- .../param_drift_policy.hpp | 475 +++ .../ros2_medkit_graph_watchdog/package.xml | 7 + .../src/detectors/param_drift_detector.cpp | 1125 +++++ .../test/e2e/harness.py | 75 +- .../test/e2e/test_config_plumbing_e2e.test.py | 210 + .../test/e2e/test_param_drift_e2e.test.py | 207 + .../test/e2e/test_param_roundtrip_e2e.test.py | 658 +++ .../test/test_param_drift_integration.cpp | 3693 +++++++++++++++++ .../test/test_param_drift_policy.cpp | 648 +++ 16 files changed, 7602 insertions(+), 47 deletions(-) create mode 100644 src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/param_drift_policy.hpp create mode 100644 src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/param_drift_detector.cpp create mode 100644 src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_config_plumbing_e2e.test.py create mode 100644 src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_drift_e2e.test.py create mode 100644 src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_roundtrip_e2e.test.py create mode 100644 src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_integration.cpp create mode 100644 src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_policy.cpp diff --git a/.github/workflows/pixi.yml b/.github/workflows/pixi.yml index 7c16cad0c..b7e166863 100644 --- a/.github/workflows/pixi.yml +++ b/.github/workflows/pixi.yml @@ -16,7 +16,9 @@ jobs: runs-on: ubuntu-latest # Experimental - don't block development while RoboStack support matures continue-on-error: true - timeout-minutes: 45 + # The build step alone is ~32 min of this, and the job has been landing at 37-40 min for + # weeks with about 3 min of run-to-run variance, so 45 left no room for a package growing. + timeout-minutes: 60 defaults: run: shell: bash diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 7266c9ffb..df49ab2bb 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -309,9 +309,24 @@ jobs: - name: Extend test timeouts for ASan overhead run: | - find build/ -name "CTestTestfile.cmake" \ - -exec sed -i 's/TIMEOUT "60"/TIMEOUT "180"/g' {} + \ - -exec sed -i 's/TIMEOUT "120"/TIMEOUT "360"/g' {} + + # Multiply EVERY declared CTest timeout by three, whatever it is. The + # x3 factor is the one the two literal rewrites this replaced encoded + # (60 -> 180, 120 -> 360), so no test's effective budget changes here; + # what changes is that a test declared at any OTHER value is no longer + # silently left on its native budget under a sanitizer that makes it + # several times slower. A fixed pair of literals goes stale the moment + # a budget is raised for good reasons, and the failure it produces is + # the worst kind: ctest kills the process and the test's own output + # dies with it, so the run reports a timeout and no test name. + find build/ -name "CTestTestfile.cmake" -exec \ + perl -pi -e 's/\bTIMEOUT "(\d+)"/sprintf(q{TIMEOUT "%d"}, $1 * 3)/ge' {} + + # Print what the tests will actually run with, so a sanitizer job that + # is killed on time can be read against its real budgets. `cat` first + # so grep sees one stream: its exit status then means "the rewrite had + # nothing to act on", which under pipefail correctly fails the step, + # rather than "one -exec batch happened to hold no timeouts". + find build/ -name "CTestTestfile.cmake" -exec cat {} + \ + | grep -oE 'TIMEOUT "[0-9]+"' | sort -t'"' -k2 -n | uniq -c - name: Run unit + integration tests with ASan + UBSan timeout-minutes: 30 @@ -414,9 +429,12 @@ jobs: - name: Extend test timeouts for TSan overhead run: | - find build/ -name "CTestTestfile.cmake" \ - -exec sed -i 's/TIMEOUT "60"/TIMEOUT "180"/g' {} + \ - -exec sed -i 's/TIMEOUT "120"/TIMEOUT "360"/g' {} + + # Same generic x3 rewrite as the ASan job - see the comment there for + # why it is not a list of literal values. + find build/ -name "CTestTestfile.cmake" -exec \ + perl -pi -e 's/\bTIMEOUT "(\d+)"/sprintf(q{TIMEOUT "%d"}, $1 * 3)/ge' {} + + find build/ -name "CTestTestfile.cmake" -exec cat {} + \ + | grep -oE 'TIMEOUT "[0-9]+"' | sort -t'"' -k2 -n | uniq -c - name: Run unit + integration tests with TSan timeout-minutes: 30 diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt index 51779c26a..8c0fbdaac 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt @@ -28,6 +28,9 @@ find_package(ament_cmake REQUIRED) find_package(ros2_medkit_gateway REQUIRED) find_package(ros2_medkit_msgs REQUIRED) find_package(rclcpp REQUIRED) +# The reused gateway parameter transport includes rcl_interfaces message headers directly; +# it built only through rclcpp's transitive exports. +find_package(rcl_interfaces REQUIRED) find_package(nlohmann_json REQUIRED) find_package(lifecycle_msgs REQUIRED) find_package(tf2_msgs REQUIRED) @@ -82,7 +85,7 @@ add_library(ros2_medkit_graph_watchdog MODULE ${DETECTOR_SRCS}) target_include_directories(ros2_medkit_graph_watchdog PRIVATE include ${GATEWAY_SRC_INCLUDE_DIR}) medkit_target_dependencies(ros2_medkit_graph_watchdog - ros2_medkit_gateway rclcpp ros2_medkit_msgs lifecycle_msgs tf2_msgs geometry_msgs) + ros2_medkit_gateway rclcpp rcl_interfaces ros2_medkit_msgs lifecycle_msgs tf2_msgs geometry_msgs) target_link_libraries(ros2_medkit_graph_watchdog nlohmann_json::nlohmann_json) install(TARGETS ros2_medkit_graph_watchdog LIBRARY DESTINATION lib/${PROJECT_NAME}) @@ -121,6 +124,8 @@ if(BUILD_TESTING) find_package(ament_cmake_gtest REQUIRED) find_package(rosgraph_msgs REQUIRED) + # test_param_drift_integration.cpp includes rcutils/logging.h directly. + find_package(rcutils REQUIRED) ament_add_gtest(test_warmup_tracker test/test_warmup_tracker.cpp) target_include_directories(test_warmup_tracker PRIVATE include ${GATEWAY_SRC_INCLUDE_DIR}) @@ -187,12 +192,14 @@ if(BUILD_TESTING) target_link_libraries(test_reliability_gate nlohmann_json::nlohmann_json) medkit_set_test_domain(test_reliability_gate) - # LifecycleShutdownSuppressor: needs a REAL ReliabilityGate (departed_lifecycle_state_of() - # reads live from ctx.gate) - same real-Node/ROS_DOMAIN_ID rationale as test_reliability_gate - # above, unlike test_allowlist_suppressor (pure interface, no gate needed). + # Pure matching logic over hand-built inputs: no rclcpp::Node, so no domain is needed. ament_add_gtest(test_orphan_policy test/test_orphan_policy.cpp) target_include_directories(test_orphan_policy PRIVATE include) + ament_add_gtest(test_param_drift_policy test/test_param_drift_policy.cpp) + target_include_directories(test_param_drift_policy PRIVATE include) + target_link_libraries(test_param_drift_policy nlohmann_json::nlohmann_json) + ament_add_gtest(test_qos_policy test/test_qos_policy.cpp) target_include_directories(test_qos_policy PRIVATE include) medkit_target_dependencies(test_qos_policy rmw) # rmw is transitively found via rclcpp; no find_package needed @@ -202,9 +209,10 @@ if(BUILD_TESTING) medkit_target_dependencies(test_aggregated_fault ros2_medkit_gateway rclcpp ros2_medkit_msgs) target_link_libraries(test_aggregated_fault nlohmann_json::nlohmann_json) - # Pure suppressor-chain interface: no raise_fault/clear_fault calls (no ReliabilityGate - # sources needed), no live rclcpp::Node, so no ROS_DOMAIN_ID is required - same as - # test_qos_policy above. + # Real DDS endpoints read through the live graph API, so each of these takes a domain of its + # own via medkit_set_test_domain. The two that ALSO register a fake /fault_manager/report_fault + # service additionally share a RESOURCE_LOCK, because they spin real nodes for many seconds and + # running them concurrently on this box starves the discovery each one waits on. ament_add_gtest(test_qos_mismatch_integration test/test_qos_mismatch_integration.cpp src/detectors/qos_mismatch_detector.cpp @@ -232,6 +240,27 @@ if(BUILD_TESTING) medkit_set_test_domain(test_orphan_integration) set_tests_properties(test_orphan_integration PROPERTIES TIMEOUT 120 RESOURCE_LOCK graph_watchdog_integration_domain) + ament_add_gtest(test_param_drift_integration + test/test_param_drift_integration.cpp + src/detectors/param_drift_detector.cpp + src/detector_registry.cpp + src/reliability_gate.cpp + src/lifecycle_watcher.cpp + ${LIFECYCLE_STATE_READER_SOURCES} + ${PARAMETER_TRANSPORT_SOURCES}) + target_include_directories(test_param_drift_integration PRIVATE include ${GATEWAY_SRC_INCLUDE_DIR}) + medkit_target_dependencies(test_param_drift_integration + ros2_medkit_gateway rclcpp rcl_interfaces rcutils ros2_medkit_msgs lifecycle_msgs) + target_link_libraries(test_param_drift_integration nlohmann_json::nlohmann_json) + medkit_set_test_domain(test_param_drift_integration) + # 300 s, not 120: several cases here are paced off the detector's own attempt and tick horizons + # (sixty failed reads, sixty unarmed ticks, a read that answers past the per-read bound) and each + # of those costs real seconds that no budget knob can compress away. A timeout the suite can + # outgrow kills the process before gtest prints which case failed, which is the one thing a + # timeout must never do to a diagnostic suite. + set_tests_properties(test_param_drift_integration PROPERTIES + TIMEOUT 300 RESOURCE_LOCK graph_watchdog_integration_domain) + # === QoS-mismatch e2e (launch_testing) === # # Acceptance gate: proves GRAPH_QOS_MISMATCH raises and clears through the @@ -257,6 +286,77 @@ if(BUILD_TESTING) ENVIRONMENT "GATEWAY_TEST_PORT=19130;ROS_DOMAIN_ID=89;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" RESOURCE_LOCK graph_watchdog_e2e_domain) + # param_drift e2e: the detector's DEFAULT mode. The config-plumbing scenarios below all set + # baseline: false and none of them asserts a clear, so without this the self-capturing behaviour + # operators actually get, and the whole healing story the README documents, had no e2e at all. + # 300 s, not 240: the file's own deadlines are now the arming gate (60), the silent-capture + # window (15), the raise poll (60), the still-raised check (30) and the heal poll (60), and 240 + # no longer covers their sum. A budget shorter than the internal deadlines is killed by ctest + # before any of the messages those deadlines exist to print reaches the log, which turns every + # one of them into "timed out". + add_launch_test(test/e2e/test_param_drift_e2e.test.py TIMEOUT 300) + set_tests_properties(test_e2e_test_param_drift_e2e.test.py PROPERTIES + LABELS "integration;e2e" + ENVIRONMENT "GATEWAY_TEST_PORT=19170;ROS_DOMAIN_ID=89;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" + RESOURCE_LOCK graph_watchdog_e2e_domain) + + # Round-trip cost e2e: what ONE parameter read really costs the node that serves it, counted + # at a node that answers the parameter services by hand. param_drift's read budget is charged + # in round trips (2 / 3 / 2), every other test in this package proves the arithmetic GIVEN + # those numbers, and the stand-in transport they use counts CALLS - so nothing else here can + # notice a round trip being added to or removed from Ros2ParameterTransport. Deliberately runs + # with param_drift OFF: the reads are driven over the configurations endpoint so each count + # belongs to exactly one transport call. + # 300 s, not 180: this test's own retry deadlines (discovery, first contact against each of the + # two probes) plus the single-shot measurements between them can legitimately add up to more + # than 180 on a slow-discovery run - which is the run it exists to diagnose. A timeout shorter + # than the internal budgets kills the process before any of those diagnostics print, so the + # failure reads as "test timed out" and says nothing about round trips. + add_launch_test(test/e2e/test_param_roundtrip_e2e.test.py TIMEOUT 300) + set_tests_properties(test_e2e_test_param_roundtrip_e2e.test.py PROPERTIES + LABELS "integration;e2e" + ENVIRONMENT "GATEWAY_TEST_PORT=19180;ROS_DOMAIN_ID=89;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" + RESOURCE_LOCK graph_watchdog_e2e_domain) + + # === Config-plumbing e2e (launch_testing) === + # + # Proves nested per-detector config reaches a live detector through the REAL gateway + # (extract_plugin_config -> detector_config.hpp -> ParamDriftConfig::from_json). No C++ test + # above can give that proof: they all call configure() directly with hand-built JSON and + # never touch the delivery path. + # + # One source file, three CTest targets. The plugin reads its config once, at set_context() + # time, so proving the nested `expect` reader and the nested `mode` reader needs separate + # gateway launches rather than separate test methods in one. WATCHDOG_E2E_SCENARIO selects + # which launch and which assertions run in each process. + add_launch_test(test/e2e/test_config_plumbing_e2e.test.py + TARGET test_config_plumbing_e2e_positive + TIMEOUT 180) + set_tests_properties(test_config_plumbing_e2e_positive PROPERTIES + LABELS "integration;e2e" + ENVIRONMENT "GATEWAY_TEST_PORT=19140;ROS_DOMAIN_ID=89;WATCHDOG_E2E_SCENARIO=positive;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" + RESOURCE_LOCK graph_watchdog_e2e_domain) + + add_launch_test(test/e2e/test_config_plumbing_e2e.test.py + TARGET test_config_plumbing_e2e_mode_off + TIMEOUT 180) + set_tests_properties(test_config_plumbing_e2e_mode_off PROPERTIES + LABELS "integration;e2e" + ENVIRONMENT "GATEWAY_TEST_PORT=19150;ROS_DOMAIN_ID=89;WATCHDOG_E2E_SCENARIO=mode_off;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" + RESOURCE_LOCK graph_watchdog_e2e_domain) + + # Third launch: `mode: off` written BARE, which the ROS parser types as a YAML 1.1 boolean + # rather than a string. That is the form operators write, and a string-only reader silently + # keeps the detector raising. + add_launch_test(test/e2e/test_config_plumbing_e2e.test.py + TARGET test_config_plumbing_e2e_mode_off_yaml_bool + TIMEOUT 180) + set_tests_properties(test_config_plumbing_e2e_mode_off_yaml_bool PROPERTIES + LABELS "integration;e2e" + ENVIRONMENT + "GATEWAY_TEST_PORT=19160;ROS_DOMAIN_ID=89;WATCHDOG_E2E_SCENARIO=mode_off_yaml_bool;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" + RESOURCE_LOCK graph_watchdog_e2e_domain) + ros2_medkit_relax_vendor_warnings() endif() diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md index 2225c130e..638af37fb 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md @@ -7,7 +7,8 @@ Detectors read the graph and raise faults through a `ReportFault` service client gateway node; the faults surface via FaultManager on the gateway `/faults` API. This package carries the plugin skeleton, the central reliability gate that holds raises -until the graph has quiesced, and two detectors, `qos_mismatch` and `orphan`. The remaining +until the graph has quiesced, and three detectors, `qos_mismatch`, `orphan` and +`param_drift`. The remaining silent-fault classes land in follow-up changes, each against its own issue; their fault codes are already reserved in the frozen `GRAPH_*` namespace (see "Fault codes"). @@ -72,6 +73,17 @@ differs only in its numeric field and the rule above already spares it. `/robot/scan`, is six edits apart, so no budget that is still specific will reach it. Do not rely on this detector for that class. +### `param_drift` keys + +| Key | Type | Default | Meaning | +|-----|------|---------|---------| +| `mode` | string \| bool | `raise` | As above. A bare `off` arrives as a YAML boolean; both forms disable the detector. | +| `baseline` | bool | `true` | Self-capture the value of every parameter not listed in `expect`, then flag later changes to it. `false` checks only what `expect` pins, and skips reading the full parameter list. | +| `expect.` | any | - | Absolute expected value for ``, checked on every node that has it. This is the only rule that catches a value that was wrong from the first tick, which self-capture cannot see. A node that does not declare `` answers NOT_FOUND, which is not a drift - so a misspelt name checks nothing at all. Once the sweep has covered every armed app - each one read, or given up on after repeated failed reads - and at least one of them answered, a pin that no app declares is warned about once, naming it. | +| `ignore` | string[] | `[]` | Parameter-name globs never flagged. Applies to the self-captured set only, so it cannot silence an `expect` pin. A pattern of nothing but `*` matches every name and suppresses self-capture wholesale, which is indistinguishable at runtime from finding nothing; that combination warns at startup. | +| `max_reads_per_tick` | int | `8` | Parameter-service ROUND TRIPS this detector may spend per tick, round-robin over nodes. Round trips, not calls: one call is several requests at the watched node, and the node pays for each of them. A `baseline` visit is charged 2 (a list, then one batched get) whatever the node's parameter count; each `expect` pin is charged 3 (a list, a get and a descriptor read), or 2 on a node that does not declare it - see the table under "Coverage latency" for what each of those really costs the node. So a self-capture visit spends 2 of the budget and pinning N parameters spends up to 3N. It is an AMORTISED rate, not a per-tick cap: there is no pacing INSIDE a visit, so a node's whole read set is issued back to back and the reader waits out the debt afterwards. Four pins against a budget of 8 fire 12 round trips inside one tick period and then idle for the rest of the next one; what the knob bounds is the sustained rate, not the burst. Accepted range 1..100000; anything else keeps the default and warns. | +| `prune_grace` | int | `60` | Consecutive sweeps an app may be missing before its finding is dropped: the finding survives `prune_grace` absences and goes on the next one. Injected for every detector at plugin scope and overridable per detector; a value outside 0..3600 is rejected with a warning and this detector falls back to 2, not to the plugin default. The captured BASELINE has a shorter horizon of its own, and that one is NOT configurable - it is the compile-time `kBaselineForgetGrace`: the baseline is dropped on the third consecutive missed sweep, the two the reliability gate absorbs plus one, so an ordinary dropped discovery poll cannot re-capture a node on its already-drifted value. Setting `prune_grace` below 2 pulls that forward onto the sweep the finding itself is dropped on, so the two then go together. | + ### `qos_mismatch` keys | Key | Type | Default | Meaning | @@ -81,6 +93,39 @@ rely on this detector for that class. | `allowlist` | string[] | `[]` | Subscriber FQNs never reported. Exact match only. | +## Closing the loop: healing config is required + +Every detector here reports level-triggered: it re-raises FAILED while the condition holds +and emits PASSED on every clean sweep. That is what lets the fault_manager's debounce +counter walk from CONFIRMED back to HEALED. The fault_manager only counts PASSED events +toward healing when `healing_enabled` is true for that fault, and it defaults to `false`. +Without it, a `GRAPH_*` fault that has genuinely cleared stays CONFIRMED forever. Enable +healing globally on the fault_manager node: + +```bash +ros2 run ros2_medkit_fault_manager fault_manager_node --ros-args \ + -p healing_enabled:=true \ + -p healing_threshold:=1 # 2 consecutive clean sweeps to heal (confirmation_threshold defaults to -1) +``` + +or scope it to this plugin only, by longest-prefix match on `source_id`, which is always +`graph_watchdog`: + +```yaml +# entity_thresholds.yaml, loaded via the fault_manager's entity_thresholds.config_file param +graph_watchdog: + healing_enabled: true + healing_threshold: 1 # 2 consecutive clean sweeps to heal (confirmation_threshold defaults to -1) +``` + +Healing time does not depend on how long the fault was active. The debounce counter is +clamped to `[confirmation_threshold, healing_threshold]` on every event, so a FAILED event +can push it down to `confirmation_threshold` and no further; a long-running fault does not +dig a hole that later clean sweeps have to climb out of. After the fix, each clean sweep +steps the counter up by one, so healing takes at most +`healing_threshold - confirmation_threshold` consecutive clean sweeps - exactly that many once +the counter has walked down to the floor, fewer if the fault was raised only briefly. + ## Where GRAPH_* faults hang Every `GRAPH_*` fault is raised with `source_id: graph_watchdog`. That is an App the @@ -339,6 +384,204 @@ local graph-cache queries, so every topic is checked every tick. `/rosout` and `configure()`; both C++ tiers hand `configure()` a JSON object directly. +#### `param_drift` (GRAPH_PARAM_DRIFT) + +Watches ROS 2 node parameters and raises `GRAPH_PARAM_DRIFT` when a live value diverges +from its reference. It needs no configuration to be useful: it captures each parameter's +value once the node arms and flags later runtime changes. Pinning a parameter to an +absolute value with `expect` additionally catches the case self-capture cannot see, a value +that was already wrong when the node started. + +**A changed parameter is not the same as a broken robot.** The detector reports that a +value moved and does not grade the consequence. Whether the new value matters is a question +about the machine, not about the graph, so it raises at WARN and leaves the judgement to +whoever reads the fault. + +**Aggregated fault, not per-node.** All drift in the graph is one graph-level +`GRAPH_PARAM_DRIFT`, not a fault per owning node, for the same reason as the other +detectors: the fault_manager identifies a fault by `fault_code` alone, so per-node faults +would collide into a single record. The description enumerates every currently-drifted +`(node, parameter)` pair, up to a cap of 480 characters - past that the text is truncated. Each +app's own contribution is trimmed to 150 characters before that, so one node drifting on many +parameters cannot fill the 480 by itself, and the entries are ordered `expect` violations first, +then self-captured drift, with newly detected apps ahead of the ones already reported inside each +group, so the cap cannot hide the two things worth reading. It clears +(`EVENT_PASSED`) on every sweep where nothing drifts, so one node reverting is not enough while +another still drifts, subject to the same `healing_enabled` requirement described in "Closing the +loop" above. A clear is withheld while any app still has no usable read - one the sweep has not +reached, one whose reads keep failing, or one the reliability gate has not armed - because health +that was never measured is not reported as health. That withholding is bounded in both directions: +an app whose reads keep failing stops holding the clear back once it has been given up on (see +below), and a gate-denied app stops once the frozen hold has elapsed. An app whose reads keep +failing is named in the log once ten consecutive attempts have failed; one the sweep has not yet +attempted is not named, because nothing is known about it yet - instead, a clear that has been +withheld for a whole minute of ticks is logged once, saying how many apps are behind it and naming +one, so an operator watching a fault refuse to heal can tell a graph the sweep cannot cover from a +detector that is simply working. + +```yaml +plugins: + graph_watchdog: + detectors: + param_drift: + expect: + use_sim_time: false # flag any node running on sim time in production + ignore: ["*_stamp"] # only filters self-captured params, so keep baseline enabled +``` + +`baseline` and `expect` combine: an empty `expect` with `baseline: true` is pure +self-capture; a sparse `expect` with `baseline: true` is pins plus self-capture; +`baseline: false` checks only the pinned parameters and moves far less data, because it never +reads the full parameter list. It is not cheaper against the budget, though: a self-capture visit +is charged 2 round trips no matter how many parameters the node has, while a single pin the node +declares is charged 3. + +**Coverage latency.** The sweep is budgeted by service ROUND TRIPS at the watched nodes - not by +node count, and not by transport calls either. `max_reads_per_tick` bounds how many round trips +happen per tick period, and the reader paces itself to that rate rather than reading back to back. +A node's whole read set is read in one visit, and the visit is then charged in round trips rather +than in calls. What it is charged is not always what the node paid: + +| Visit | Round trips charged | What the node actually pays | +|-------|---------------------|-----------------------------| +| `baseline: true` | 2 | 2 - one list, one batched get, whatever the parameter count. 4 on the very first contact; see below. | +| one `expect` pin the node declares | 3 | 3 - a name-scoped list, a get, and a descriptor read the transport makes unconditionally on the success path | +| one `expect` pin the node does not declare | 2 | 1 - the name-scoped list comes back empty and the transport stops there. The charge is the higher of the two NOT_FOUND paths: a name the list DOES find and the get then returns no value for costs 2, and the error code cannot tell them apart, so the bound overstates rather than understates. | +| one `expect` pin the node declares but has not set | 3 | 3 - not a NOT_FOUND at all. rclcpp answers an unset parameter with a `PARAMETER_NOT_SET` value rather than an absent one, so the read succeeds, pays for the descriptor like any other resolved pin, and the pin fires: a node holding no value at all does not satisfy a pin that names one. It is reported as `got=`, not as `got=null`, because a NaN-valued parameter reads as `null` too and the two are different faults. | + +The right-hand column is measured rather than asserted: `test/e2e/test_param_roundtrip_e2e.test.py` +drives the real transport against a node that answers the parameter services by hand and counts +every request it is asked to serve. + +Visiting every node once therefore takes at least about + +- `ceil(2 * node_count / max_reads_per_tick)` tick periods with `baseline: true`, and +- with `baseline: false`, `ceil(3 * pin_count * node_count / max_reads_per_tick)` where every node + declares every pin, down to `ceil(2 * pin_count * node_count / max_reads_per_tick)` where none of + them does, + +and a drift on a node not yet visited is invisible until then. At the defaults (budget 8, 1 s +tick) a 20-node graph is about 5 ticks in self-capture mode; the same graph with four pins every +node declares is about 30, and about 20 if no node declares any of them. Treat that as a floor +rather than a bound: a read that overruns its slot is not compensated, so a few slow nodes stretch +the rotation further. + +In `baseline: true` mode the very FIRST read of a given node costs two round trips more than the +table charges, because `list_parameters` primes the transport's defaults cache before doing its own +read. That excess is deliberately not charged: the cache has no expiry, so the pair is paid once +per node for the life of the gateway, and charging it on every read would throttle the whole run to +cover one rotation. The practical effect is that the first rotation after startup puts up to twice +the budgeted load on the nodes in it, and every rotation after that stays within budget. +`baseline: false` has no cold surcharge at all: `get_parameter` never primes that cache, so the +first pinned read of a node costs the same three round trips as every later one. + +Coverage latency governs repairs as well as detections: a node that reverted but has not been read +again yet keeps its stale entry, so the aggregated fault can stay raised for up to one rotation +after the underlying fix. Raise `max_reads_per_tick`, or lower the tick interval, to shorten that +window on a big graph. + +There is one stale entry those two knobs do NOT shorten. A node that is present but has left the +reliability gate's `active` state is not read at all, so its finding is frozen rather than +re-derived - which is right for a pause, and wrong for good. `kFrozenHoldTicks`, a compile-time 60 +consecutive gate-denied ticks (a minute at the default tick period), bounds it: past that the +finding is released and the app also stops holding back a clear, and both come back from a fresh +read if it ever arms again. Because that horizon is counted in TICKS and not in rotations, raising +`max_reads_per_tick` does nothing to it and shortening `tick_interval_ms` shortens it only +incidentally, by making a tick shorter. + +What is NOT derived from the rotation is anything the detector says about an app that will not +answer. Both the log line and the point at which it stops holding back a clear rest primarily on +completed, FAILED read attempts and not on elapsed ticks, so an app that is merely waiting its turn +is never announced as silent: the log names an app once ten consecutive reads of it have failed, +and the detector stops letting it hold back a clear only once more than sixty have. Each of those +carries an additional FLOOR in ticks, of the same size, which the app must also have spent +unread - an attempt is not a duration, and without the floor `max_reads_per_tick: 100000` spends +sixty-one attempts in a couple of milliseconds and writes off a node that has merely not finished +being discovered. The floor is a conjunction, so it can only ever delay the two, never bring them +forward. An armed app the sweep has never ATTEMPTED blocks the clear for as long as that stays +true, however long the rotation takes; a cached read is not a substitute, so an app whose reads +start failing blocks the clear again from the first failure. + +The reads run on a thread the plugin owns, never on the gateway's ROS executor, so a node +that stops answering stalls only this detector's sweep and not the gateway's request +handling. + +**Getting back to clean after a deliberate change.** The detector cannot tell an +intentional runtime change from a misconfiguration; it only knows the value moved away from +what it last saw. Reverting the parameter always works: the aggregate clears on the first sweep +where nothing drifts and every app has either been read, been given up on after repeated failed +reads, or exhausted the frozen hold with the gate still shut. + +Restarting the node only works if the restart is slow enough to be SEEN. A captured baseline +is dropped on the third consecutive sweep its app is absent from (sooner only if +`prune_grace` is set below 2, see its row above), and only then does the node's return start from +a fresh capture. That window is deliberate. DDS discovery drops a +node from a single poll routinely, with no restart involved, so a one-sweep window would +re-baseline that node on its already-drifted value and heal a true positive that could then +never fire again, because the new baseline IS the wrong value. The cost runs the other way: a +node that comes back inside the window keeps its pre-restart baseline and goes on being +reported as drifted until someone reverts the change. `respawn` in a launch file defaults to +no delay, so the usual edit-the-YAML-and-restart flow is frequently never observed as an +absence at all: a respawn that completes inside one tick leaves the node present on every +sweep, and no window, however short, catches that. To make a restart re-baseline, keep the +node out of the graph long enough to miss three sweeps: a `respawn_delay`, or a stop, a pause, +then a start. Accepting a new value as the baseline without a restart does not exist today. + +**Test tiers:** + +1. **Unit** (`test/test_param_drift_policy.cpp`): the pure comparison, rendering and + configuration rules against hand-built values - the glob matching, the exact + integer-versus-float comparison and the NaN carve-out, the `baseline` and `expect` + combinations, the config range checks, the ASCII-only elided rendering the capped description + depends on, and the `read_gap` spacing the pacing budget is built on. +2. **Integration** (`test/test_param_drift_integration.cpp`): real `rclcpp` nodes with real + parameters, read over the real parameter service against a fake `ReportFault` service, plus a + scripted stand-in transport for the replies no live node can be made to produce on demand - a + read still in flight when its app is de-armed, a SUCCESS carrying no parameters at all, a read + that throws, a node that never answers. It covers the grace-based prune when a node vanishes, + the re-baseline after an absence long enough to be a restart together with the single dropped + poll that must NOT re-baseline, that the baseline forget is reachable at every accepted + `prune_grace`, that the baseline horizon is exactly three missed sweeps and not two or four, + the clear path, that a closing reliability gate does not heal a present drift, and every state + in which a clear must be withheld because nothing is known about an app: one the sweep has not + reached, one the gate has not armed, and one that answered once and then stopped. It also + covers that both unread warnings need FAILED attempts AND elapsed ticks rather than either + alone, that an app given up on stops blocking the clear, that a withheld clear says so in the + log once, that a read overrunning the per-read bound is abandoned, that an app re-bound to a + different node starts from a fresh baseline, the per-app share of the description and that + trimming it keeps both ends, and the ordering that decides what survives the capped + description: a newly drifted app ahead of the ones already reported, and an `expect` violation + ahead of self-captured drift - including the cross case, where a pinned violation already + reported still outranks brand-new self-captured drift. The pacing budget is covered in both + tiers - `read_gap`'s arithmetic in the unit tests above, and the rate the reader actually + achieves against a configured budget here. +3. **E2e** (`test/e2e/test_param_drift_e2e.test.py`, + `test/e2e/test_param_roundtrip_e2e.test.py`, + `test/e2e/test_config_plumbing_e2e.test.py`): five real gateway launches. + The first is the acceptance gate: it drives the detector's DEFAULT self-capturing mode end to + end - a real node whose parameter the test changes over the real parameter service, a real + fault_manager, and both the raise and the heal read back from the operator-visible + `GET /api/v1/faults`. + The second is what makes the round-trip table above a measurement rather than an assertion. It + runs with `param_drift` switched OFF and drives the same two transport functions the detector + uses over `GET /{entity}/configurations`, against a node that answers the parameter services by + hand and counts every request it is asked to serve: a cold and a warm `baseline` visit, a + resolved pin, both NOT_FOUND paths (the unlisted name that costs 1 and the listed one with no + value that costs 2, which is what the charge of 2 is sized for), a declared-but-unset pin that + is a success costing 3, and a first pinned read of an untouched node. Nothing else in the + package can notice a round trip being added to or removed from the TRANSPORT - every other test + proves the arithmetic GIVEN those numbers, and the stand-in transport they use counts CALLS. + It does not read the detector's charge constants either, though: those are held to the + measurement by the timing cases in the integration suite, which pace the reader off them. + The last three are the only proof that a nested SUB-OBJECT under a detector key survives the + delivery path: `expect.` arrives from the gateway as nested JSON and has to be flattened + back to the dotted parameter name. (Plain per-detector scalars and arrays are proven by the + orphan e2e.) Every C++ test above calls `configure()` directly with hand-built JSON and never + touches the delivery path. One launch proves the nested + `expect` reader, one proves `mode: "off"`, and one proves the bare `mode: off` that the + ROS parser types as a YAML 1.1 boolean rather than a string, which is the form operators + actually write. + ## Reliability (bringup-quiesce) Silent-fault detectors are prone to bringup noise: a node joining the graph, a @@ -413,7 +656,7 @@ itself. The two things a detector opts into explicitly are `/tf_static`, whose publishers latch so a late subscriber must match the durability to receive them). -## Adding a detector +## Adding a detector 1. Create `src/detectors/.cpp`. 2. Subclass `Detector`; implement `id()` and `tick(DetectorContext&)`. Read the @@ -434,4 +677,3 @@ Frozen in `include/ros2_medkit_graph_watchdog/graph_fault_codes.hpp`: `GRAPH_QOS_MISMATCH`, `GRAPH_ORPHAN`, `GRAPH_NODE_DISAPPEARED`, `GRAPH_TF_STALE`, `GRAPH_PARAM_DRIFT`, `GRAPH_LATENCY_BUDGET`, plus one extension of the frozen namespace for a new capability beyond the original six: `GRAPH_NODE_INACTIVE`. - diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst index 27dd162b3..2aed0cf00 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst @@ -89,8 +89,9 @@ Reliability core Detectors --------- -``qos_mismatch`` and ``orphan`` are the detectors this package ships so far. The -remaining silent-fault classes land in follow-up changes, each against its own issue. +``qos_mismatch``, ``orphan`` and ``param_drift`` are the detectors this package ships so +far. The remaining silent-fault classes land in follow-up changes, each against its own +issue. ``qos_mismatch`` raises ``GRAPH_QOS_MISMATCH``. It watches every topic's publisher/subscriber QoS pairs rather than parameter values: each @@ -129,11 +130,10 @@ are not checked. The checks match only concrete incompatible enum pairs, so ``SYSTEM_DEFAULT``/``UNKNOWN`` (never reported by a live endpoint, which always carries the resolved profile) never raise. -**Aggregation via the shared helper.** ``qos_mismatch`` is the first detector to use -the new ``AggregatedFault`` helper (``aggregated_fault.hpp``), factored out of -a shared helper so later detectors do not each reimplement the same -level-triggered raise/clear pattern. The rationale is -(see "Aggregation rationale" above): the fault_manager identifies a fault by +**Aggregation via the shared helper.** ``qos_mismatch`` was the first detector to use the +``AggregatedFault`` helper (``aggregated_fault.hpp``); ``orphan`` and ``param_drift`` go +through the same one, so no detector reimplements the level-triggered raise/clear +pattern. The rationale: the fault_manager identifies a fault by ``fault_code`` alone, so one ``GRAPH_QOS_MISMATCH`` per mismatched topic would collide into a single record under the shared code. The detector keeps one ``AggregatedFault`` instance for the whole graph and, each tick, hands it every currently-mismatched @@ -141,12 +141,17 @@ topic's description (keyed by topic name so a repeat mismatch on the same topic overwrites rather than duplicates); an empty map on a clean tick clears (``EVENT_PASSED``), level-triggered semantics (see "Closing the loop" in the README for the ``healing_enabled`` requirement to actually -reach HEALED). ``graph_source_id()``, also in ``aggregated_fault.hpp``, is the same -host-Component-id-or-literal-fallback logic - the detectors' -aggregated faults land under the same ``source_id``. +reach HEALED). ``graph_source_id()``, also in ``aggregated_fault.hpp``, returns the +constant ``kGraphWatchdogEntityId`` (``"graph_watchdog"``) and nothing else, so every +detector's aggregated faults land under the same ``source_id``. It deliberately does NOT +fall back to the host Component id: a runtime host Component built by ``HostInfoProvider`` +never sets ``external``, ``collect_component_app_fqns`` only puts a Component's bare id in +the fault scope set when it does, and a fault raised under it is therefore listed by no +entity endpoint at all. The ``ctx`` argument is unused and kept only because every call +site already has it in hand. -**Coverage is exhaustive, not budgeted.** Unlike a call-budgeted -round-robin sweep (bounded by a parameter-service transport call budget), +**Coverage is exhaustive, not budgeted.** Unlike a budgeted +round-robin sweep (bounded by a parameter-service round-trip budget), ``qos_mismatch`` reads no external service - ``get_topic_names_and_types()`` and the two ``get_*_info_by_topic()`` calls are local graph-cache queries, so every topic is checked every tick with no coverage-latency trade-off to configure or budget knob to @@ -221,9 +226,99 @@ so it also proves a string array and an integer survive the parameter path into ``configure()``, which no C++ tier can show. +``param_drift`` raises ``GRAPH_PARAM_DRIFT``. It reads node parameters over the +parameter service and reports a value that no longer matches its reference. The +reference is either self-captured, the value the parameter had when the node armed, or +pinned in config through ``expect``. A node absent for longer than the window the +reliability gate absorbs loses its captured baseline, so a restart - the documented way to apply +a new configuration - re-captures instead of reporting the change as drift. A shorter gap keeps +the baseline on purpose: DDS discovery drops a node from a single poll routinely, and re-capturing +on that would take the node's current, possibly already drifted, value as the new reference and +heal a real fault that could then never fire again. Only the pinned form can catch a value that was +already wrong at startup; self-capture by construction treats whatever it first sees as +correct. + +**A budgeted sweep, unlike every other detector here.** ``qos_mismatch`` and ``orphan`` +read the local graph cache and are therefore exhaustive every tick. This one makes real +service calls to other nodes, so it is bounded by ``max_reads_per_tick`` and sweeps the +graph round-robin. The budget is spent in service ROUND TRIPS at the watched node rather +than in transport calls, because one call is several requests there: a self-capture read +is a list plus a batched get, and an ``expect`` pin the node declares is a list, a get and a +descriptor read. A pin the node does NOT declare stops at the list, and is charged two rather +than the one it costs, because a second and rarer NOT_FOUND path does cost two and the error +code cannot tell them apart - a load bound may overstate, never understate. That buys a bound +on the load the sweep puts on watched nodes and pays with +coverage latency: a drift is invisible until its node's turn comes, and so is a repair. +The reads run on a thread the plugin owns rather than the gateway executor, so a node +that stops answering stalls this sweep only. + +**Three horizons for a stale entry.** A node missing from one sweep is not forgotten +immediately. Its captured BASELINE is dropped on the third consecutive sweep without it - the +two the warmup tracker's forget grace absorbs, plus one - so that a one-tick discovery gap does +not silently reset the baseline and hide a real drift behind a fresh capture. Its FINDING +survives ``prune_grace`` consecutive absences and is dropped on the next one, so that the same +gap does not clear the aggregated fault either. Only the finding horizon is configurable: +``prune_grace`` is a per-detector config key, while the baseline horizon is the compile-time +``kBaselineForgetGrace``. A ``prune_grace`` below 2 makes the finding the shorter of the two and +the baseline then goes on the same sweep, because the absence counter it rides on is dropped +with the finding. + +The third horizon is not about absence at all. A node that stays PRESENT but leaves the +reliability gate's ``active`` state is never re-evaluated (it is not armed) and never pruned (it +is still there), so its finding would otherwise be re-raised for ever - an operator who fixes +the parameter and leaves the node deactivated would watch ``GRAPH_PARAM_DRIFT`` stay CONFIRMED +naming a value that is no longer true. ``kFrozenHoldTicks`` (60 consecutive gate-denied ticks, +one minute at the default tick period) bounds that: past it the finding is released and the app +also stops holding back a clear, and both re-derive from a fresh read if it ever arms again. +Because it is a compile-time constant and counted in ticks rather than in rotations, neither +``max_reads_per_tick`` nor ``tick_interval_ms`` shortens it. + +**Severity is WARN, deliberately.** The detector knows a value moved; it does not know +whether that breaks the robot. Grading the consequence needs knowledge of the machine +that the graph does not carry. + +**Nothing is cleared that was not measured.** A clear asserts that nothing is drifting anywhere, +which is only true if every app was actually looked at, so the aggregate emits nothing at all while +any app still has no usable read. That covers three states which are all "nothing is known about +this app": the round-robin has not reached it, its reads keep failing, or the reliability gate has +not armed it - and the last of those is the ordinary state of every app for the first +``warmup_cycles`` ticks of a run, so without it a restart clears before issuing a single round trip. +A cached read counts as evidence only until a read of that app FAILS, after which the app is +unmeasured again. Each hold is bounded from the other side: an app is given up on after both sixty +failed attempts and sixty ticks unread, and a gate-denied one after the frozen hold. The bound on +attempts alone would not do, because an attempt is not a duration - the reader paces itself off +``max_reads_per_tick``, so at the accepted ceiling sixty-one attempts are spent in milliseconds and +a node still being discovered would be written off. Finally, a clear withheld for a minute of ticks +is logged once with the count of apps behind it, because from outside the process a correctly +withheld clear and a detector that has nothing to report look exactly the same. + +**Test tiers.** ``test_param_drift_policy.cpp`` pins the pure comparison, rendering and +config rules. ``test_param_drift_integration.cpp`` drives real nodes with real parameters over +the real parameter service against a fake ``ReportFault`` service, including a node that +never answers, which must not hang the tick, and a node that answers past the per-read bound, +which must not be read at all. +``test/e2e/test_param_drift_e2e.test.py`` is the acceptance gate: one gateway launch carrying +the whole raise-and-clear story for the detector's default self-capturing mode, with a real +fault_manager and both ends read back from ``GET /api/v1/faults``. +``test/e2e/test_param_roundtrip_e2e.test.py`` measures what one parameter read really costs the +node that serves it, against a node that answers the parameter services by hand and counts every +request - so a round trip added to or removed from the TRANSPORT is caught, instead of quietly +invalidating the round-trip figures the budget is built on. Nothing else in the package can +notice that: every other test proves the arithmetic GIVEN those figures, and the stand-in +transport they use counts calls. It does not, however, read the detector's own charge constants +at all - it runs with ``param_drift`` switched off - so what holds those to the measurement is +the timing suite in ``test_param_drift_integration.cpp``, which times the reader against a +configured budget and therefore moves when a charge does. Cost and charge are pinned separately +and on purpose. +``test/e2e/test_config_plumbing_e2e.test.py`` drives three further gateway launches covering the +nested ``expect`` sub-object reader, ``mode: "off"``, and the bare ``mode: off`` that the +ROS parser types as a YAML 1.1 boolean; those three prove configuration delivery and +suppression only, and assert no clear. + + Status --------------- The plugin loads, ticks the graph, and shuts down cleanly. The reliability core is real -and already ticking. Two silent-fault detector classes raise through it today, -``qos_mismatch`` and ``orphan``. The remaining classes land in follow-up changes, each -against its own issue. +and already ticking. Three silent-fault detector classes raise through it today, +``qos_mismatch``, ``orphan`` and ``param_drift``. The remaining classes land in follow-up +changes, each against its own issue. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp index 3e2e088fb..7fda64c0c 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp @@ -63,9 +63,10 @@ class AggregatedFault { public: /// Cap on the generated description. One aggregated fault names every affected entity, so /// on a large graph the raw text grows without bound - an oversized ReportFault description - /// then travels into every /faults payload and the fault_manager's store. param_drift's - /// hand-rolled aggregation applies the same cap; this keeps every detector that uses the - /// helper consistent with it. + /// then travels into every /faults payload and the fault_manager's store. Every detector + /// aggregates through this helper, so the cap is applied in one place rather than copied per + /// detector. A detector may additionally trim what a SINGLE entity contributes before handing + /// it over (param_drift does), so that one entity cannot spend the whole budget by itself. static constexpr std::size_t kMaxDescriptionChars = 480; AggregatedFault(const char * code, std::uint8_t severity) : code_(code), severity_(severity) { diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp index c56062d6d..2db5b83de 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -28,7 +29,12 @@ namespace ros2_medkit_gateway { struct IntrospectionInput; -} +// Only named in the test-only transport factory signature below, so a declaration is enough. +// Including the gateway header here would pull it into every consumer of this one, and the test +// targets that do not add GATEWAY_SRC_INCLUDE_DIR would then resolve it from install/ while the +// plugin resolves it from the source tree - the split this package's own CMakeLists warns about. +class ParameterTransport; +} // namespace ros2_medkit_gateway namespace ros2_medkit_graph_watchdog { @@ -128,6 +134,21 @@ class Detector { virtual std::size_t tracked_count_for_test() const { return 0; } + + /// Test-only injection hook for detectors that read parameters over a transport. Returns false + /// when the detector does not use one, so a test can assert it reached the right detector rather + /// than silently doing nothing. + /// + /// It lives on the base because detectors are file-local and self-registering: a test only ever + /// holds a `Detector*` from the registry, so there is no derived type to cast to. The alternative + /// is leaving the behaviours that only a stand-in transport can produce - a read still in flight + /// when its app is de-armed, a SUCCESS response carrying no parameters, a transport that fails to + /// construct - untested, and none of them is reachable from a live ROS graph. + using ParameterTransportFactory = + std::function(rclcpp::Node *)>; + virtual bool set_parameter_transport_factory_for_test(const ParameterTransportFactory & /*factory*/) { + return false; + } }; } // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/param_drift_policy.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/param_drift_policy.hpp new file mode 100644 index 000000000..f026e3c28 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/param_drift_policy.hpp @@ -0,0 +1,475 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ros2_medkit_graph_watchdog/detector_config_keys.hpp" // collect_unknown_detector_keys + +namespace ros2_medkit_graph_watchdog { + +/// Minimal glob matcher supporting `*` (matches any run, including empty). No `?`, +/// no character classes - ROS param ignore patterns only need `*`. +inline bool glob_match(const std::string & pattern, const std::string & text) { + std::size_t p = 0; + std::size_t t = 0; + std::size_t star = std::string::npos; + std::size_t mark = 0; + while (t < text.size()) { + if (p < pattern.size() && pattern[p] == '*') { + star = p++; + mark = t; + } else if (p < pattern.size() && pattern[p] == text[t]) { + ++p; + ++t; + } else if (star != std::string::npos) { + p = star + 1; + t = ++mark; + } else { + return false; + } + } + while (p < pattern.size() && pattern[p] == '*') { + ++p; + } + return p == pattern.size(); +} + +/// The transport's own name for the type of a parameter a node DECLARED and never SET. +/// Ros2ParameterTransport::parameter_type_to_string maps rclcpp's PARAMETER_NOT_SET to this. +inline constexpr const char * kNotSetParameterType = "not_set"; + +/// Key of the out-of-band marker that stands in for such a parameter inside an observed param map. +inline constexpr const char * kUnsetParameterKey = "x-medkit-unset"; + +/// How that marker reads in a fault description. Unquoted on purpose: a STRING parameter whose +/// value happens to be `` renders WITH quotes, so no node-supplied value can produce this +/// exact text and an operator reading it is never looking at something a node wrote. +inline constexpr const char * kUnsetParameterRendering = ""; + +/// The stand-in carried through an observed param map for a parameter the node declared and +/// never set. +/// +/// A JSON OBJECT, and that is the whole of the safety argument. ROS parameter values are bool, +/// int, double, string and the 1-D array forms of those, so an object is a shape no parameter +/// value can take; and ParamDriftConfig::flatten_expect recurses THROUGH objects and only ever +/// stores the scalar or array it bottoms out on, so an operator's `expect` pin cannot hold one +/// either. Neither side of a comparison can therefore be mistaken for this. A string sentinel +/// would have neither property - a node is free to hold the string `` - and the pin an +/// operator wrote on that node would silently stop reporting. +inline nlohmann::json unset_param_value() { + return nlohmann::json{{kUnsetParameterKey, true}}; +} + +/// True for the marker above, and for nothing a node or an operator can write. +inline bool is_unset_param(const nlohmann::json & v) { + return v.is_object() && v.contains(kUnsetParameterKey); +} + +/// The value one transport parameter entry (`{"name","value","type"}`) contributes to an observed +/// param map, with a declared-but-never-set parameter mapped to the marker instead of a bare null. +/// +/// Two signals, because either alone leaves a way for `null` to reach a fault description. The +/// TYPE is the transport's explicit statement and the one a live node produces: rclcpp answers a +/// declared-but-unset name with a PARAMETER_NOT_SET value rather than an absent one, so the read +/// SUCCEEDS carrying null and `type` is `not_set`. The VALUE is the backstop: null is not a legal +/// encoding of any ROS parameter value, so a transport that omits the type still cannot put a bare +/// `null` in front of an operator. A NaN double is caught by NEITHER - it arrives as a float that +/// merely DUMPS as `null` - and that is exactly the distinction this exists to keep: after this, +/// `got=null` in a description means the value is NaN (or infinite) and nothing else. +inline nlohmann::json param_entry_value(const nlohmann::json & entry) { + const auto type = entry.find("type"); + if (type != entry.end() && type->is_string() && type->get() == kNotSetParameterType) { + return unset_param_value(); + } + const auto value = entry.find("value"); + if (value == entry.end() || value->is_null()) { + return unset_param_value(); + } + return *value; +} + +/// True when an integer and a float denote the SAME number, without routing the integer through +/// a double. nlohmann's own cross-type compare does exactly that cast, so beyond 2^53 two +/// different values compare equal and the drift is missed. Reachable from an `expect` pin written +/// with a decimal point against an integer parameter, or a parameter whose type flips between +/// INTEGER and DOUBLE. +inline bool int_equals_float_exactly(const nlohmann::json & i, const nlohmann::json & f) { + const double d = f.get(); + if (!std::isfinite(d)) { + return false; // NaN or infinity is never equal to an integer + } + // Integrality without an equality test on doubles: -Werror=float-equal forbids == and != here, + // and it is right to - the same flag caught a real sentinel bug in this package before. + double whole = 0.0; + const double frac = std::modf(d, &whole); + if (frac > 0.0 || frac < 0.0) { + return false; // a fractional value can never equal an integer + } + // 2^63 is not representable as int64_t; compare against the double bound instead. + if (d < -9223372036854775808.0 || d >= 9223372036854775808.0) { + return false; + } + if (i.is_number_unsigned()) { + // The sign has to be checked BEFORE the cast. Converting a negative double to uint64_t is + // undefined behaviour, and it does not merely produce a harmless mismatch: on x86-64 -1.0 + // lands on UINT64_MAX, so a parameter holding that value compares EQUAL to a pin of -1.0 and + // the drift is missed. Every positive integer arrives from the transport stored unsigned, so + // this branch is the ordinary path, not a corner of it. + return d >= 0.0 && i.get() == static_cast(d); + } + return i.get() == static_cast(d); +} + +/// Equality for drift detection. Three carve-outs sit in front of nlohmann's `operator==`: +/// +/// - A mixed integer/float pair goes through int_equals_float_exactly rather than `operator==`, +/// whose cross-type compare casts the integer to a double and so collapses int64s differing +/// beyond 2^53 into a MISSED drift. +/// - Two NaN doubles are equal, because IEEE `NaN != NaN` would otherwise make any NaN-valued +/// param (a common "unset/disabled" sentinel) a permanent false positive that never clears. +/// - Arrays recurse element-wise, so both carve-outs apply inside an array param as well (ROS +/// params are at most 1-D, so no deeper recursion is needed). +/// +/// Everything else defers to `operator==`, and the unset marker (see unset_param_value) is +/// deliberately one of them rather than a fourth carve-out: as a JSON object it compares unequal to +/// every value a node or an `expect` pin can hold, so a pin against a parameter the node never set +/// still reports - and equal to itself, so a parameter that is unset on every sweep does not drift +/// against its own captured baseline for ever. Both directions are pinned in the unit tests. +inline bool param_values_equal(const nlohmann::json & a, const nlohmann::json & b) { + if (a.is_number_float() && b.is_number_float() && std::isnan(a.get()) && std::isnan(b.get())) { + return true; + } + if (a.is_number_integer() && b.is_number_float()) { + return int_equals_float_exactly(a, b); + } + if (a.is_number_float() && b.is_number_integer()) { + return int_equals_float_exactly(b, a); + } + if (a.is_array() && b.is_array()) { + if (a.size() != b.size()) { + return false; + } + for (std::size_t i = 0; i < a.size(); ++i) { + if (!param_values_equal(a[i], b[i])) { + return false; // recurse -> NaN[] handled element-wise (ROS params <= 1-D) + } + } + return true; + } + return a == b; +} + +/// Minimum spacing between two parameter reads so that no more than `max_reads_per_tick` +/// of them happen per tick period. This is the whole enforcement of that budget: the reader +/// takes one target per pass and waits out the rest of its slot, so a gap of zero means no +/// budget at all. +/// +/// Computed in microseconds on purpose. A budget larger than the tick period expressed in +/// milliseconds (2000 reads against a 1000 ms tick) truncates to zero under integer +/// millisecond division, which silently removes the bound instead of tightening it. +/// Microseconds are not enough on their own either: a budget above a million reads per second +/// truncates to zero again, and a zero gap is a reader that never waits. The floor keeps the +/// loop paced no matter what an operator asks for, and it does so SILENTLY: a budget inside the +/// accepted range is taken as written, and whether the floor bites depends on the tick period, +/// which from_json never sees. `max_reads_per_tick: 100000` (the ceiling) against +/// `tick_interval_ms: 1` therefore buys 1000 round trips per tick period, not 100000, with +/// nothing logged - the floor caps the effective budget BELOW the request, which is the safe +/// direction for a load bound, so this is a limit and not a defect. +/// +/// Rounded UP, because the achievable rate is `1 / gap` and not `reads / period`: flooring the +/// division makes the gap shorter than the budget allows and the rate correspondingly higher. +/// `tick_interval_ms: 1` with `max_reads_per_tick: 600` - both accepted configurations - floors +/// to a 1 us gap and 1000 round trips per tick period against a documented bound of 600. A knob +/// an operator lowers to protect a node must never be overshot; rounding up can only spend the +/// budget more slowly than asked, which is the safe direction for a bound. +inline std::chrono::microseconds read_gap(int tick_interval_ms, int max_reads_per_tick) { + const auto reads = static_cast(max_reads_per_tick > 0 ? max_reads_per_tick : 1); + const auto period_us = static_cast(tick_interval_ms) * 1000; + return std::chrono::microseconds(std::max(1, (period_us + reads - 1) / reads)); +} + +/// Parsed param-drift detector configuration. See the design doc for the model. +struct ParamDriftConfig { + bool baseline_capture = true; ///< self-capture drift on unlisted params + std::map expect; ///< param_name -> absolute expected value + std::vector ignore_globs; ///< param-name globs never flagged (self-capture set only) + int max_reads_per_tick = 8; ///< bound on blocking param reads per tick (coverage latency lever) + /// Upper bound on the configurable budget. A request beyond this is not a tuning choice, it is + /// a typo or a unit mix-up: even at the floor gap of one microsecond the reader cannot issue a + /// million service round-trips per tick, so accepting it would only mean pretending. + static constexpr std::int64_t kMaxReadsPerTickCeiling = 100000; + std::vector warnings; ///< non-fatal config problems, surfaced by the detector's logger + + /// Parse the per-detector config the plugin delivers (config["detectors"]["param_drift"], + /// see detector_config.hpp): keys `baseline` (bool), `expect` (object of param_name -> + /// expected value, flattened from the gateway's nested delivery), `ignore` (native JSON + /// string array). A present-but-wrong-typed `baseline`/`ignore`/`expect` is NOT silently + /// dropped: it records a warning (a mistyped `ignore` would otherwise re-enable drift alerts + /// on suppressed params with no diagnostic trail). + static ParamDriftConfig from_json(const nlohmann::json & detector_cfg) { + ParamDriftConfig cfg; + if (detector_cfg.contains("baseline")) { + if (detector_cfg["baseline"].is_boolean()) { + cfg.baseline_capture = detector_cfg["baseline"].get(); + } else { + cfg.warnings.push_back("'baseline' must be a boolean; ignoring it and keeping the default (true)"); + } + } + if (detector_cfg.contains("ignore")) { + if (detector_cfg["ignore"].is_array()) { + for (const auto & g : detector_cfg["ignore"]) { + if (g.is_string()) { + cfg.ignore_globs.push_back(g.get()); + } else { + cfg.warnings.push_back("'ignore' entries must be strings; skipping a non-string entry"); + } + } + } else { + cfg.warnings.push_back("'ignore' must be a string array (e.g. [\"*_stamp\"]); ignoring it"); + } + } + if (detector_cfg.contains("max_reads_per_tick")) { + // Range-check BEFORE narrowing. is_number_integer() is true across the whole 64-bit range + // and get() truncates, so 4294967297 would arrive as 1 and -4294967295 would arrive as + // 1 too - a NEGATIVE request silently passing a positive-only guard, and a budget quietly + // eight times smaller than the default. Both were accepted without a warning. + const auto & value = detector_cfg["max_reads_per_tick"]; + const std::int64_t wide = value.is_number_integer() ? value.get() : 0; + if (value.is_number_integer() && wide > 0 && wide <= kMaxReadsPerTickCeiling) { + cfg.max_reads_per_tick = static_cast(wide); + } else { + cfg.warnings.push_back("'max_reads_per_tick' must be an integer in 1.." + + std::to_string(kMaxReadsPerTickCeiling) + "; keeping the default (8)"); + } + } + if (detector_cfg.contains("expect")) { + if (detector_cfg["expect"].is_object()) { + // The gateway un-nests dotted config keys, so a pin on a dotted param name + // (e.g. expect.qos_overrides./scan.reliability) arrives as a nested object. + // Flatten it back to the full dotted param name so it matches the observed key. + flatten_expect(detector_cfg["expect"], "", cfg.expect, &cfg.warnings); + } else { + cfg.warnings.push_back("'expect' must be an object of param_name -> expected value; ignoring it"); + } + } + // A wrong TYPE on the four keys above already warns; a wrong NAME used to not, even + // though it fails the same way. `ignore_globs` (the struct's own field name) instead + // of `ignore` leaves the globs empty and GRAPH_PARAM_DRIFT firing on exactly the + // params the operator meant to suppress. + collect_unknown_detector_keys(detector_cfg, known_keys(), cfg.warnings); + // Configurations that parse cleanly and can never do anything. Silence here is the worst + // outcome: the detector looks configured and reports nothing, forever. + if (!cfg.baseline_capture && cfg.expect.empty()) { + cfg.warnings.push_back( + "'baseline' is false and 'expect' is empty, so this detector will never read a " + "parameter or report anything; set 'expect' or leave 'baseline' enabled"); + } + if (!cfg.baseline_capture && !cfg.ignore_globs.empty()) { + cfg.warnings.push_back( + "'ignore' only applies to self-captured parameters, and 'baseline' is false, so it has " + "no effect here"); + } + // A glob made of nothing but '*' matches every parameter name, so self-capture is suppressed + // wholesale. At runtime that is indistinguishable from a detector that is working and finding + // nothing - the silent failure this detector exists to rule out, reproduced in its own config. + const bool catch_all_ignore = + std::any_of(cfg.ignore_globs.begin(), cfg.ignore_globs.end(), [](const std::string & g) { + return !g.empty() && g.find_first_not_of('*') == std::string::npos; + }); + if (cfg.baseline_capture && catch_all_ignore) { + if (cfg.expect.empty()) { + cfg.warnings.push_back( + "'ignore' matches every parameter name and 'expect' is empty, so this detector will " + "never report anything; narrow the pattern"); + } else { + cfg.warnings.push_back( + "'ignore' matches every parameter name, so self-captured drift is fully suppressed " + "and only the 'expect' pins can report"); + } + } + return cfg; + } + + /// Config keys this detector reads. Anything else under detectors.param_drift is a typo. + static const std::set & known_keys() { + static const std::set keys{"baseline", "ignore", "max_reads_per_tick", "expect"}; + return keys; + } + + /// Flatten a (possibly nested) `expect` object back to dotted param-name keys. The gateway + /// splits dotted config keys into nested objects; ROS param values are scalars/arrays (never + /// objects), so the recursion stops at the expected value and the joined key is the full + /// param name (e.g. {"qos_overrides":{"/scan":{"reliability":v}}} -> "qos_overrides./scan.reliability"). + static void flatten_expect(const nlohmann::json & obj, const std::string & prefix, + std::map & out, + std::vector * warnings = nullptr) { + for (const auto & item : obj.items()) { + const std::string key = prefix.empty() ? item.key() : prefix + "." + item.key(); + if (item.value().is_object()) { + if (item.value().empty() && warnings != nullptr) { + // An empty object carries no expected value, so the pin the operator wrote silently + // becomes nothing at all. Say so rather than dropping it. + warnings->push_back("'expect." + key + "' has no value; ignoring that pin"); + } + flatten_expect(item.value(), key, out, warnings); + } else if (!out.emplace(key, item.value()).second && warnings != nullptr) { + // Two config paths flattened to the same parameter name (e.g. `a: {b: 1}` next to + // `a.b: 2`). emplace keeps the first, so the other pin is silently lost. + warnings->push_back("'expect." + key + "' is pinned twice; keeping the first value"); + } + } + } +}; + +/// Pure, ROS-free drift core. Holds per-app captured baselines and computes this app's +/// drifted param descriptors from an observed param map. One instance per detector. +/// Aggregation (raise/clear across apps) moves to the detector. +class ParamDriftPolicy { + public: + explicit ParamDriftPolicy(ParamDriftConfig cfg) : cfg_(std::move(cfg)) { + } + + const ParamDriftConfig & config() const { + return cfg_; + } + + /// Per-value budget inside the description. Small enough that several drifted entries fit under + /// AggregatedFault's cap, large enough to still show a recognisable value. Public so a test can + /// pin the budget rather than a loose descriptor length that would pass at twice the size. + static constexpr std::size_t kMaxRenderedValueChars = 64; + + /// Returns this app's drifted descriptors (empty = clean). Captures the app's baseline + /// the first time it is called with capture_allowed=true (so a not-yet-armed node is + /// never baselined pre-settle). expect rules are absolute and fire regardless. + /// `pinned_out`, when given, receives how many of the returned descriptors came from an + /// `expect` pin. Those are always the leading entries, so a caller can tell an + /// operator-declared violation from self-captured drift without parsing the text back. It + /// must not parse: a parameter's own value can contain any substring, including the markers + /// this function writes, so text sniffing would misclassify. + std::vector evaluate(const std::string & app_id, const std::map & observed, + bool capture_allowed, std::size_t * pinned_out = nullptr) { + std::vector drifted; + for (const auto & [name, expected] : cfg_.expect) { + const auto it = observed.find(name); + if (it != observed.end() && !param_values_equal(it->second, expected)) { + drifted.push_back(app_id + ":" + safe_name(name) + " expected=" + compact(expected) + + " got=" + compact(it->second)); + } + } + if (pinned_out != nullptr) { + *pinned_out = drifted.size(); + } + if (cfg_.baseline_capture && capture_allowed) { + auto & base = baselines_[app_id]; // creates empty on first sight + for (const auto & [name, value] : observed) { + if (cfg_.expect.count(name) > 0 || is_ignored(name)) { + continue; + } + const auto bit = base.find(name); + if (bit == base.end()) { + base.emplace(name, value); // new param -> capture silently (not a drift) + continue; + } + // No `first` check here: when this app had no baseline at all, every name takes the + // capture branch above, so reaching this line already means the baseline pre-existed. + if (!param_values_equal(value, bit->second)) { + drifted.push_back(app_id + ":" + safe_name(name) + " baseline=" + compact(bit->second) + + " got=" + compact(value)); + } + } + } + return drifted; + } + + /// App vanished from the graph: drop its baseline so a later reappearance re-captures + /// instead of drifting against a stale baseline. + void forget(const std::string & app_id) { + baselines_.erase(app_id); + } + + private: + bool is_ignored(const std::string & name) const { + for (const auto & g : cfg_.ignore_globs) { + if (glob_match(g, name)) { + return true; + } + } + return false; + } + + /// Render a parameter value for a fault description. Node-supplied values are arbitrary, and + /// this is the first detector to put them into fault text at all, so two things must hold. + /// + /// ASCII only. The aggregated description is capped by cutting at a BYTE offset, and a cut + /// through a multi-byte UTF-8 sequence produces a description the gateway's own DTO writer + /// cannot serialize - GET /faults then answers 500 instead of listing the fault. `ensure_ascii` + /// escapes non-ASCII, so any byte offset is a safe cut point. + /// + /// Never throws. A value whose bytes are not valid UTF-8 makes a plain dump() throw, and that + /// throw escapes into the plugin's per-detector guard, so the detector would neither raise nor + /// clear for as long as the value is present - a CONFIRMED fault that can never heal. The + /// replace handler substitutes U+FFFD instead. + /// + /// Long values are elided: one array-valued parameter can otherwise consume the whole + /// description budget and hide every other drifted entry. + /// + /// A parameter the node DECLARED and never SET is rendered from the marker rather than dumped. + /// Dumping it would print `null`, and a NaN double prints `null` too - so one description said + /// the same thing about "the node never set this parameter" and "the value is NaN", which are + /// different faults with different remedies. See unset_param_value(). + static std::string compact(const nlohmann::json & v) { + if (is_unset_param(v)) { + return kUnsetParameterRendering; + } + return elide(v.dump(-1, ' ', /*ensure_ascii=*/true, nlohmann::json::error_handler_t::replace)); + } + + /// Same treatment for a parameter NAME. Names come verbatim from the watched node's + /// ListParameters reply and are never validated, so they carry exactly the two hazards values do: + /// a name whose bytes are not valid UTF-8, or a valid multi-byte name split by the aggregated + /// description's byte-offset cut, makes the gateway's JSON writer throw. That answers 500 on + /// GET /faults and takes every OTHER fault down with it, since they share the response. + static std::string safe_name(const std::string & name) { + return elide(nlohmann::json(name).dump(-1, ' ', /*ensure_ascii=*/true, nlohmann::json::error_handler_t::replace)); + } + + /// Shorten to the budget while keeping BOTH ends. Head-only truncation renders two different + /// values identically whenever they share a long prefix - the common case for robot_description + /// and for nav2 footprint arrays, where only a late element differs - so the fault would fire + /// correctly and then prove nothing about what changed. + static std::string elide(std::string text) { + if (text.size() <= kMaxRenderedValueChars) { + return text; + } + const std::size_t keep = kMaxRenderedValueChars - 3; + const std::size_t head = (keep + 1) / 2; + return text.substr(0, head) + "..." + text.substr(text.size() - (keep - head)); + } + + ParamDriftConfig cfg_; + std::map> baselines_; // app_id -> (param -> value) +}; + +} // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/package.xml b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/package.xml index ee975482e..615d6b3b5 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/package.xml +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/package.xml @@ -13,6 +13,10 @@ ros2_medkit_gateway ros2_medkit_msgs rclcpp + + rcl_interfaces nlohmann-json-dev lifecycle_msgs tf2_msgs @@ -25,6 +29,9 @@ rosgraph_msgs std_msgs sensor_msgs + + rcutils launch_testing_ament_cmake diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/param_drift_detector.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/param_drift_detector.cpp new file mode 100644 index 000000000..6a9b378f6 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/param_drift_detector.cpp @@ -0,0 +1,1125 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ros2_medkit_gateway/core/providers/introspection_provider.hpp" +#include "ros2_medkit_gateway/core/transports/parameter_transport.hpp" +#include "ros2_medkit_gateway/ros2/transports/ros2_parameter_transport.hpp" +#include "ros2_medkit_graph_watchdog/aggregated_fault.hpp" +#include "ros2_medkit_graph_watchdog/detector.hpp" +#include "ros2_medkit_graph_watchdog/detector_config_keys.hpp" +#include "ros2_medkit_graph_watchdog/detector_registry.hpp" +#include "ros2_medkit_graph_watchdog/graph_fault_codes.hpp" +#include "ros2_medkit_graph_watchdog/param_drift_policy.hpp" + +namespace ros2_medkit_graph_watchdog { + +namespace { +// Fault severity scale: SEVERITY_WARN=1, SEVERITY_ERROR=2 (ros2_medkit_msgs/msg/Fault.msg). +// The detector reports the drift and does not grade its impact: whether a changed parameter +// matters is a question about the robot, not about the graph, so triage belongs to whoever +// reads the fault. WARN keeps it visible without claiming the run is broken. +constexpr std::uint8_t kParamDriftSeverity = ros2_medkit_msgs::msg::Fault::SEVERITY_WARN; +constexpr double kParamServiceTimeoutSec = 0.5; // per-read timeout, same as the lifecycle GetState read +constexpr double kNegativeCacheTtlSec = 10.0; // unreachable-node backoff +constexpr int kForgetGrace = 2; // missed-sweep grace before forgetting, mirrors WarmupTracker + +// What one transport call costs the WATCHED node, in parameter-service round trips. +// +// Nothing in the build ties these literals to the transport they describe. Compiling the gateway's +// Ros2ParameterTransport into the plugin (see PARAMETER_TRANSPORT_SOURCES in CMakeLists.txt) only +// guarantees that the constants and the code are the same VERSION - a round trip added inside the +// transport would leave every one of them stale and still compile. +// +// Two DIFFERENT tests are needed to hold them true, and neither can do the other's job: +// +// - test/e2e/test_param_roundtrip_e2e.test.py measures what the TRANSPORT really costs, by running +// it against a node that answers the parameter services by hand and counts every request it is +// asked to serve. It reads none of the literals below - it drives the two transport functions +// over the configurations endpoint with this detector switched OFF - so it catches a round trip +// being added to or removed from the transport, and it catches nothing about these numbers. Set +// kBaselineReadRoundTrips to 4 and it still passes. +// - the CHARGE, i.e. that these literals still match that measurement, is held by the timing cases +// in test/test_param_drift_integration.cpp (ABaselineVisitIsChargedTwoRoundTripsAndNotTheColdStartPair, +// TheBudgetIsSpentInRoundTripsNotCalls, AnUndeclaredPinIsChargedLessThanOneThatResolves). They +// time the reader against a configured budget, which is a function of what it charges, so a +// literal moved here moves the elapsed time and they go red. +// +// Every OTHER test in this package proves the detector's arithmetic GIVEN these numbers, and the +// stand-in transport they use counts CALLS. +// +// A baseline visit is one list_parameters(): a list request, then one batched get for every name +// it returned (ros2_parameter_transport.cpp:397 and :420). +constexpr int kBaselineReadRoundTrips = 2; +// The FIRST list_parameters() against a given node costs two MORE, because it primes the +// transport's defaults cache before doing its own read (:364 -> :1158 list + :1190 get). That +// cache has no TTL (:1163 - entries are only evicted under LRU pressure), so the extra pair is a +// one-off per node per gateway lifetime and is deliberately NOT charged here: this knob bounds a +// SUSTAINED sweep, and charging every read for a cost paid once per node would throttle the whole +// run to cover the first rotation. The README states the cold-contact excess instead. +// +// An expect visit is one get_parameter() per pinned name, and each of those is three round trips: +// a name-scoped list, a get, and a describe (:520, :553, :585). The describe is unconditional on +// the success path - the block at :584 has no enclosing guard, only the USE of its result at :586 +// is guarded - so it is part of the price whether or not the descriptor is wanted. There is no +// cold-contact pair here: get_parameter never calls cache_default_values, so a `baseline: false` +// sweep pays the same three on its first read of a node as on every later one. +constexpr int kExpectReadRoundTrips = 3; +// A pin the node does not declare costs ONE round trip: the name-scoped list comes back empty and +// the transport returns NOT_FOUND without reaching the get or the describe (:538). `expect` pins +// are documented as checked on every node that HAS the parameter, so on a mixed graph most nodes +// answer NOT_FOUND for most pins, and charging those the full success price would throttle the +// sweep three times harder than the load it creates. NOT_FOUND has a second, rarer path that costs +// two (:571): the name-scoped list DOES find the name and the get then comes back with no value +// for it, which the GetParameters contract allows, so the transport stops one round trip later. +// The error code is the same on both paths and the caller cannot tell which it took, so the CHARGE +// below is the higher of them and deliberately overstates the common case: a bound may overstate a +// cost, it must never understate one. Both are measured in test_param_roundtrip_e2e. +// +// A parameter the node DECLARES and has not set is not either of those. rclcpp answers one with a +// PARAMETER_NOT_SET value rather than an absent one, so the transport gets a parameter back, pays +// for the unconditional describe and returns SUCCESS carrying null - three round trips, charged as +// kExpectReadRoundTrips. Charging it two would understate what the node pays. +constexpr int kExpectNotFoundRoundTrips = 2; + +// Completed, FAILED read attempts against one armed app before we say so. Counted in attempts and +// never in elapsed ticks. An app the round-robin sweep has not reached yet has made no attempt at +// all, and on a large graph it waits many ticks for its turn: reporting that "no read has +// succeeded" for it would be a bringup warning about a perfectly healthy node, which is what an +// elapsed-tick threshold produced. Ten consecutive failures is past any single dropped reply and +// squarely into "this node does not answer". +constexpr int kUnreadReportAfterFailures = 10; +// A FLOOR in ticks, spent alongside the attempt counts and never instead of them. +// +// An attempt is not a duration. The reader paces itself at max_reads_per_tick round trips per tick +// period, so what an attempt costs in wall clock is read_gap x its charge - and the whole horizon +// therefore scales with the load knob. At the accepted ceiling of 100000 the gap floors at one +// microsecond and sixty-one failed attempts are spent in a couple of milliseconds, so the very +// first ticks of a run write off any node whose parameter service DDS has not finished discovering +// yet. The README's own remedy for a slow sweep - raise max_reads_per_tick - is exactly what +// collapses it. +// +// Ticks are the one quantity the budget does not scale, so they are the floor. Used as a +// CONJUNCTION with the attempt count this can only ever DELAY a report or a give-up, never bring +// one forward, so it does not reintroduce the failure mode that counting attempts was adopted to +// fix: a tick horizon that EXPIRES on a large graph. The values mirror the attempt counts, which at +// the default tick period is ten seconds before an app is named and a minute before it stops +// holding back a clear. +constexpr int kUnreadReportAfterTicks = kUnreadReportAfterFailures; +// Missed sweeps a node may be absent for before we treat it as restarted and drop its captured +// baseline. Deliberately the SAME window WarmupTracker absorbs (kDefaultForgetGrace): that header +// documents that DDS discovery routinely drops a node from a single poll without it restarting, so +// re-baselining on the first miss would re-capture the node's CURRENT - possibly already drifted - +// values on ordinary churn. A confirmed fault would then heal while the parameter is still wrong +// and could never fire again, because the new baseline IS the wrong value. The cost of waiting is +// that a respawn faster than this window keeps the old baseline and reports the operator's change +// as drift until they revert it or the node stays away longer. +constexpr int kBaselineForgetGrace = 2; +// Upper bound on the configurable prune grace. At the 1 s default tick this is already an hour of +// absence before a finding is dropped; beyond it the value is a typo or a unit mix-up, not a choice. +constexpr std::int64_t kMaxPruneGrace = 3600; +// Completed, FAILED read attempts after which an app stops holding back a clear. +// +// A clear asserts that nothing is drifting anywhere, and that is only true if we actually looked +// everywhere. An app the sweep has never ATTEMPTED therefore blocks the clear INDEFINITELY: no +// amount of elapsed time turns an unmeasured node into a measured one, and a horizon in ticks is +// guaranteed to expire on a graph whose rotation is longer than the horizon - the detector would +// then report health it never measured, on exactly the large graphs where the sweep is slowest. +// +// What does justify giving up is evidence: this many attempts were made against the node and every +// one of them failed. That is a node with no working parameter service (a non-rclcpp participant, +// parameter services disabled, a wedged executor), and there is no per-node opt-out for this +// detector, so without a bound one such node would keep a fixed drift reported as CONFIRMED for +// the lifetime of the gateway, with a description naming something that is no longer true. Past +// the bound the app is declared unmeasurable, logged once, and excluded from the blocking set. +// +// The count only grows while the app stays unread and is dropped the moment a read succeeds, so an +// app already given up on can never silently re-enter the blocking set - and one that does become +// readable leaves it through the front door, by being read. +constexpr int kUnmeasurableAfterFailures = 60; +// The tick floor for the give-up, spent alongside the attempt count. See kUnreadReportAfterTicks. +constexpr int kUnmeasurableAfterTicks = kUnmeasurableAfterFailures; +// Consecutive ticks a clear may be withheld before the reason is put in the log. +// +// Withholding is the right thing to do - health that was never measured is not health - but it is +// indistinguishable from the outside from a detector that is simply working and finding nothing: +// no fault is raised, no fault is cleared, and a GRAPH_PARAM_DRIFT already in the store just never +// heals. An operator has no way to tell "the graph is being covered" from "the graph has not been +// covered for the last twenty minutes and never will be", so the detector has to say which. Once +// per episode, and only past a horizon a healthy rotation on a large graph comfortably fits inside. +constexpr int kWithheldClearReportTicks = 60; +// Consecutive ticks an app may stay non-armed while holding a frozen finding. Freezing is right for +// a pause - the drift is still real and the gate only means "do not raise about an unsettled +// entity". But a node that leaves `active` for good is never re-evaluated (not armed) and never +// pruned (still present), so without this the aggregate re-raises its stale finding forever, even +// after an operator fixes the parameter. Mirrors the prune horizon. +constexpr int kFrozenHoldTicks = 60; +// Characters one app may contribute to the aggregated description. Without a per-app bound a single +// node drifting on many parameters fills the whole 480-char cap by itself, and every other app - +// including one whose violation an operator explicitly pinned - is cut regardless of the ordering. +// Ordering decides who goes first; this is what makes going second still mean something. Three apps +// fit comfortably, which is the case worth optimising for: the aggregate exists to say WHICH nodes +// drifted, and the per-parameter detail is a hint, not the record. +constexpr std::size_t kMaxAppDetailChars = 150; + +/// One app the reader has been told to read, tagged with the token of the arming EPISODE it +/// belongs to. +/// +/// The token, not the app id and not the fqn, is what the reader compares before publishing. Both +/// of those are stable across a de-arm and a re-arm, and tick() rewrites `targets` from scratch +/// every tick, so an app whose lifecycle gate closed and re-opened inside one (blocking, seconds +/// long) read window would pass an identity check and have a value sampled mid-transition published +/// as its next reading - which becomes its BASELINE, a false drift no later read can heal, and +/// exactly what the fence exists to prevent. A token is issued once per arming episode from a +/// monotonic counter and never reused, so an interrupted episode can never be mistaken for the one +/// that followed it. +struct ReadTarget { + std::string app_id; + std::string fqn; + std::uint64_t token = 0; +}; + +/// An armed app with no usable read, and the EVIDENCE we have about why. +/// +/// `failed_attempts` is the number of consecutive completed read attempts against this app that +/// failed. Zero means the round-robin sweep has not reached it yet - the app is waiting its turn, +/// nothing is known about it, and nothing may be said about it either. +struct UnreadApp { + std::string app_id; + int failed_attempts = 0; +}; + +/// One-shot latches for the two things we say about an app that will not answer, plus how long the +/// app has been in that state. Kept per app and dropped the moment the app leaves the unread set, +/// so a run of silence is reported once rather than every tick, and an app that starts answering +/// again can be reported afresh if it later stops. +struct UnreadLatch { + bool reported = false; ///< the "no read has succeeded" warning has been logged + bool gave_up = false; ///< the unmeasurable warning has been logged + /// Consecutive ticks this app has been armed with no usable read. The FLOOR half of both + /// horizons: see kUnreadReportAfterTicks for why an attempt count alone is not a duration. + int ticks = 0; +}; + +// Reading a node's parameters is a BLOCKING service round-trip (up to kParamServiceTimeoutSec +// each, and observed to hang PAST that on a real Nav2 node stuck mid-transition, since the +// underlying DDS recv does not always honour the spin timeout). Doing it inline on the tick +// thread stalls the WHOLE detection loop - node_death, tf_stale, everything - the moment ONE +// node's parameter service is slow. So the reads run on a dedicated background thread and the +// tick thread only ever reads the cache. State shared between the two threads lives here, +// behind a shared_ptr so a reader still inside a read keeps it alive. Teardown joins the +// thread rather than detaching it, and shuts the transport down first so a blocking read +// returns instead of holding the join (see the destructor); a detached reader would still be +// making service calls while rclcpp tears the context down, which aborts rather than throws. +struct SharedIo { + std::mutex mutex; + std::condition_variable cv; + bool stop = false; + bool reader_stopped = false; ///< set by the reader when it leaves the loop (no more rcl ops) + bool baseline_capture = true; ///< snapshot of policy config (set-once at reader start) + std::vector expect_names; ///< pinned params to read when baseline_capture is false + /// The ROS-neutral base, not the concrete ROS 2 transport. The detector builds the real one by + /// default; a test can inject a stand-in to control read latency, force an empty batch, or fail + /// construction outright - none of which is reachable through a live node. + std::unique_ptr transport; // reader-thread only + std::vector targets; // armed apps to read, this episode + std::uint64_t next_token = 1; ///< monotonic; never reused + std::map> observed; // app_id -> last usable read + std::set have; ///< app_ids with a usable cached read (distinct from "read empty") + /// app_id -> consecutive read attempts that COMPLETED and failed. Written by the reader under + /// this mutex, read by the tick thread under it. An app absent from this map and absent from + /// `have` has never been attempted at all, and that is the distinction the clear rests on: `have` + /// says "we looked and it answered", an entry here says "we looked and it did not", and neither + /// says "we have not looked yet". Erased on the first successful read. + std::map failed_reads; + /// Pacing for the reader thread (set-once at reader start, alongside baseline_capture). + /// `max_reads_per_tick` service round trips at the watched nodes per `tick_interval_ms` - one + /// transport call is several of those, see reader_loop(). + int max_reads_per_tick = 8; + int tick_interval_ms = kDefaultTickIntervalMs; +}; + +/// Blocking read of `fqn`'s parameters into `observed`. Runs ONLY on the reader thread (it is +/// the sole user of io.transport, so the transport needs no lock of its own here). Returns true +/// when `observed` is a usable, complete view; false when the node should be skipped (a non- +/// NOT_FOUND transport error, or a stop mid-read) so evaluate() never sees a partial view and +/// spuriously clears a real drift. Mirrors the original inline read_params semantics exactly. +/// +/// `round_trips` accumulates what this visit is CHARGED against the WATCHED node in +/// parameter-service round trips - not transport calls, which are 1-3 round trips each, and never +/// below what the node really paid (see kExpectNotFoundRoundTrips). reader_loop turns it into the +/// pacing debt for the visit, so the knob bounds the load a node really sees. +bool read_params_io(SharedIo & io, const std::string & fqn, std::map & observed, + int & round_trips) { + if (io.baseline_capture) { + round_trips += kBaselineReadRoundTrips; + const auto result = io.transport->list_parameters(fqn); // one list, then one batched get + if (!result.success || !result.data.is_array() || result.data.empty()) { + // An empty array is a SUCCESSFUL response that carries no view - the transport documents it + // as legitimate, e.g. when a parameter declaration races the list. Treating it as "nothing + // drifts here" is what the contract above exists to prevent: evaluate() would compare + // against no observations at all, find nothing, and the tick would clear a real drift. + // Every ROS 2 node declares use_sim_time, so a genuinely empty parameter set is not a case + // worth trading that away for. + return false; + } + for (const auto & p : result.data) { + if (p.contains("name") && p.contains("value")) { + // Through param_entry_value, not straight off "value": the entry also carries the TYPE, and + // for a parameter the node declared and never set that is the only thing separating it from + // a NaN double - both of which are `null` by the time a description is written. See + // param_entry_value() and unset_param_value(). + observed.emplace(p["name"].get(), param_entry_value(p)); + } + } + return true; + } + for (const auto & name : io.expect_names) { + { + std::lock_guard lk(io.mutex); + if (io.stop) { + return false; // shutting down mid-read: incomplete view -> skip (no spurious clear) + } + } + const auto result = io.transport->get_parameter(fqn, name); + // Charged from the OUTCOME rather than blind, because a name the node never declared costs it + // a fraction of a successful read (see kExpectNotFoundRoundTrips). A call that throws is not + // charged at all - reader_loop's own floor of one slot covers that, and a throw here means the + // context is being torn down, not that the sweep is running hot. + round_trips += result.error_code == ros2_medkit_gateway::ParameterErrorCode::NOT_FOUND ? kExpectNotFoundRoundTrips + : kExpectReadRoundTrips; + if (result.success && result.data.contains("value")) { + // Same reason as the baseline branch above: a SUCCESS carrying null is a parameter the node + // declared and never set, and it must not reach the description as the `null` a NaN prints. + observed.emplace(name, param_entry_value(result.data)); + } else if (result.error_code != ros2_medkit_gateway::ParameterErrorCode::NOT_FOUND) { + return false; // unreadable this tick -> skip, do not risk a spurious clear + } + } + return true; +} + +/// Background loop: round-robins the tick thread's current `targets`, doing one blocking read +/// at a time and publishing the result into the cache. A hung read stalls only THIS thread, not +/// the tick loop; on stop it shuts the transport down and exits, releasing the shared_ptr. +/// +/// PACED at max_reads_per_tick service ROUND TRIPS per tick period - the unit the watched node +/// actually pays in, not transport calls, which are 1-3 round trips each (see +/// kBaselineReadRoundTrips / kExpectReadRoundTrips / kExpectNotFoundRoundTrips). Without pacing +/// this loop issues round-trips +/// back to back for the whole life of the gateway - tick() re-arms `targets` with every present +/// app on every tick, so there is always work and the cv predicate is always satisfied. Every one +/// of those round trips is served on the customer node's own executor. The short-circuit paths +/// turn it into a pure busy spin: a node in the transport's negative cache returns +/// SERVICE_UNAVAILABLE immediately, and `baseline: false` with an empty `expect` does zero IO and +/// returns true, so with all targets in either state the thread burns a core doing nothing. The +/// budget is also what the documented coverage latency of +/// ceil(round_trips_per_visit * node_count / max_reads_per_tick) ticks is computed from. +/// +/// The cursor is POSITIONAL, not an identity. `idx` indexes a vector the tick thread rewrites from +/// scratch every tick, in sorted app-id order, so a target's slot is not stable: an app arriving +/// that sorts earlier shifts every later one down a slot, and one leaving shifts them up, which +/// re-reads an app just visited or jumps past the app that was due. Nobody is starved by that - +/// `idx` still advances by one per read, so every index of the CURRENT list is visited within +/// targets.size() reads - but the rotation is not a guaranteed round of exactly one read per app, +/// and under continuous node-name churn an individual app's turn can keep moving. Nothing the +/// detector SAYS about an app is allowed to rest on how long that app has gone unread; see +/// kUnreadReportAfterFailures and kUnmeasurableAfterFailures, which count failed ATTEMPTS. +/// +/// Every completed visit also records its OUTCOME against the app (see `failed_reads`), which is +/// what lets the tick thread tell "this node will not answer" from "the sweep has not got to it +/// yet" without appealing to how much time has passed. +void reader_loop(std::shared_ptr io) { + std::size_t idx = 0; + for (;;) { + ReadTarget target; + std::chrono::microseconds min_gap{0}; + { + std::unique_lock lk(io->mutex); + io->cv.wait(lk, [&] { + return io->stop || !io->targets.empty(); + }); + if (io->stop) { + break; + } + target = io->targets[idx % io->targets.size()]; + ++idx; + min_gap = read_gap(io->tick_interval_ms, io->max_reads_per_tick); + } + const auto read_started = std::chrono::steady_clock::now(); + std::map observed; + bool ok = false; + int round_trips = 0; + try { + ok = read_params_io(*io, target.fqn, observed, round_trips); + } catch (const std::exception &) { + // A transport/rclcpp call can throw when the context is invalidated during shutdown, or + // on a malformed service path. An escape would std::terminate the process (the reader has + // no outer handler unlike the tick loop), so swallow it: drop this read and let the loop + // re-check stop. Matches the guard the plugin puts around each detector->tick(). + ok = false; + } + { + std::unique_lock lk(io->mutex); + if (io->stop) { + break; + } + // Record the outcome only if this arming EPISODE is still current. The read ran with the + // lock released and can take seconds, during which the tick thread may have dropped this + // app - because its lifecycle gate closed, or because prune_vanished forgot it - and may + // even have re-armed it since. Publishing anyway is not merely stale, it is harmful in + // three ways: a value sampled mid-transition becomes the app's baseline when it re-arms, + // which is a false drift that can never heal; a read landing after the prune re-inserts an + // entry whose miss counter is gone, so no later prune can reach it and the maps grow + // without bound under node-name churn; and a failure charged to an episode that is over + // pushes an app that is now perfectly readable towards being given up on. + // + // Comparing the token rather than the app id is what closes the de-arm/re-arm case: the id + // and the fqn are both stable across it, so either would let a mid-transition sample land. + const bool current_episode = std::any_of(io->targets.begin(), io->targets.end(), [&target](const ReadTarget & t) { + return t.token == target.token; + }); + if (current_episode) { + if (ok) { + io->observed[target.app_id] = std::move(observed); + io->have.insert(target.app_id); + io->failed_reads.erase(target.app_id); + } else { + // An ATTEMPT that completed and failed. This - not elapsed time - is the only thing + // that ever justifies giving up on an app and letting a clear through without it. + ++io->failed_reads[target.app_id]; + // And the app stops counting as MEASURED. A cached read is evidence about the instant it + // was taken, not a standing licence: once a read fails, what is cached says nothing about + // the node's current parameters. Leaving the app in `have` kept it in the evaluated set + // for ever off a snapshot from before it went silent, so it never joined the unread set, + // never held back a clear, and neither unread warning could ever fire about it - the + // detector reported "nothing is drifting anywhere" on a node it had not read in hours. + // `observed` is deliberately kept: it is only ever read together with `have`, and the + // next successful read overwrites it. + io->have.erase(target.app_id); + } + } + // Hold the budget: wait out whatever is left of this visit's slots. The budget is spent in + // service ROUND TRIPS at the watched node - not in node visits, and not in transport calls + // either: one list_parameters() is two round trips there and one get_parameter() is three on + // a name the node declares, and an `expect` visit makes one of the latter per pinned name. + // Charging a slot per visit, + // or even per call, would let the sweep put several times the promised load on a node while + // the knob an operator lowered to protect it reports being honoured. Charge what was + // actually spent. Waiting on the cv (not sleeping) keeps stop responsive, so shutdown is not + // delayed by the pacing. + const auto owed = min_gap * std::max(1, round_trips); + const auto elapsed = std::chrono::steady_clock::now() - read_started; + if (elapsed < owed) { + io->cv.wait_for(lk, owed - elapsed, [&] { + return io->stop; + }); + } + if (io->stop) { + break; + } + } + } + // Signal that the reader has left the loop and will do no more rcl ops. A pre-shutdown + // callback waits on this (see tick()) so the thread is idle BEFORE rclcpp invalidates the + // context - a blocking rcl call caught mid-context-teardown would SIGABRT, not throw, so it + // cannot merely be try/catch'd. The transport itself is torn down later, in the destructor. + std::lock_guard lk(io->mutex); + io->reader_stopped = true; + io->cv.notify_all(); +} +} // namespace + +/// Self-capturing parameter-drift detector. See the package design doc. +/// Parameter reads run on a background thread (see SharedIo) so a slow/stuck node's blocking +/// read can never stall the tick loop; tick() only evaluates the cache. +class ParamDriftDetector : public Detector { + public: + ParamDriftDetector() : policy_(ParamDriftConfig{}) { + } + + bool set_parameter_transport_factory_for_test(const ParameterTransportFactory & factory) override { + transport_factory_ = factory; + return true; + } + ~ParamDriftDetector() override { + if (io_) { + // Deregister the pre-shutdown callback first (it holds a weak_ptr to io_, so it is safe + // either way, but leaving a stale callback on the context is untidy). No-op if it already + // ran during a normal shutdown. + if (context_) { + context_->remove_pre_shutdown_callback(pre_shutdown_handle_); + } + { + std::lock_guard lk(io_->mutex); + io_->stop = true; + } + io_->cv.notify_all(); + // Shut the transport down FIRST: it interrupts any in-flight blocking read (its spin has + // a bounded timeout, see Ros2ParameterTransport::shutdown) so the reader returns promptly. + // Then JOIN - never detach: a detached reader still doing a service call races rclcpp + // teardown and SIGABRTs. The join here runs before the node/context are torn down (the + // plugin destroys detectors before rclcpp shutdown), so the reader finishes cleanly. + if (io_->transport) { + io_->transport->shutdown(); + } + if (reader_thread_.joinable()) { + reader_thread_.join(); + } + } + } + ParamDriftDetector(const ParamDriftDetector &) = delete; + ParamDriftDetector & operator=(const ParamDriftDetector &) = delete; + ParamDriftDetector(ParamDriftDetector &&) = delete; + ParamDriftDetector & operator=(ParamDriftDetector &&) = delete; + + std::string id() const override { + return "param_drift"; + } + + /// Distinct app ids this detector holds ANY per-app bookkeeping for, across both its own maps + /// and the ones it shares with the reader thread. + /// + /// Every one of these is keyed by app id, and node names are not a bounded set: discovery has no + /// hidden-node filter, so each `ros2` CLI invocation appears once under a name nothing will ever + /// use again. A map that no prune path reaches therefore grows for the life of the process, and + /// nothing about a leak is visible in a fault, a log line or a description - which is why it + /// needs a number a test can assert on rather than a behaviour a test can observe. + std::size_t tracked_count_for_test() const override { + std::set ids; + const auto collect = [&ids](const auto & container) { + for (const auto & entry : container) { + ids.insert(entry.first); + } + }; + collect(misses_); + collect(drifted_); + collect(reported_); + collect(pinned_counts_); + collect(unread_latches_); + collect(unarmed_ticks_); + collect(bound_fqn_); + if (io_) { + std::lock_guard lk(io_->mutex); + collect(io_->observed); + collect(io_->failed_reads); + ids.insert(io_->have.begin(), io_->have.end()); + for (const auto & t : io_->targets) { + ids.insert(t.app_id); + } + } + return ids.size(); + } + + void configure(const nlohmann::json & config) override { + policy_ = ParamDriftPolicy{ParamDriftConfig::from_json(config)}; + // prune_grace is injected into EVERY detector's config by the plugin and is exempt from the + // unknown-key warning, so ignoring it here would accept the key with no warning and no + // effect - the exact silent failure the README says cannot happen. + own_warnings_.clear(); + prune_grace_ = kForgetGrace; + if (config.contains("prune_grace")) { + // Range-check on the WIDE value, like max_reads_per_tick. get() truncates first, so + // 4294967296 would arrive as 0 and drop findings on the first missed sweep - the flapping + // prune_grace exists to prevent - and 2^63-1 would arrive as -1 and be dropped in silence. + const auto & value = config["prune_grace"]; + const std::int64_t wide = value.is_number_integer() ? value.get() : -1; + if (wide >= 0 && wide <= kMaxPruneGrace) { + prune_grace_ = static_cast(wide); + } else { + own_warnings_.push_back("'prune_grace' must be an integer in 0.." + std::to_string(kMaxPruneGrace) + + "; keeping " + std::to_string(kForgetGrace)); + } + } + // Plugin-injected: the reader thread paces itself at max_reads_per_tick service round trips + // per tick period, so it needs to know how long a tick period is. + if (config.contains("tick_interval_ms") && config["tick_interval_ms"].is_number_integer() && + config["tick_interval_ms"].get() > 0) { + tick_interval_ms_ = config["tick_interval_ms"].get(); + } + warnings_logged_ = false; + } + + void tick(DetectorContext & ctx) override { + if (!ctx.gateway_node || !ctx.snapshot) { + return; + } + // Lazily build the shared IO + reader thread on the first tick (configure() has no node). + // + // `io_` is assigned LAST, once the transport and the thread both exist. Assigning it first + // and filling it in afterwards means any throw from this block - std::thread under resource + // pressure, an rclcpp throw while building the transport's hidden node, bad_alloc - leaves a + // half-built `io_` behind. The plugin's per-detector guard logs that throw once, and every + // later tick then sees a non-null `io_`, skips this block, and runs against a detector that + // has no reader at all: `have` stays empty forever. Nothing distinguishes that from a healthy + // detector at any call site. Leaving `io_` null instead means the next tick simply retries. + if (!io_) { + auto fresh = std::make_shared(); + fresh->baseline_capture = policy_.config().baseline_capture; + fresh->max_reads_per_tick = policy_.config().max_reads_per_tick; + fresh->tick_interval_ms = tick_interval_ms_; + for (const auto & [name, _] : policy_.config().expect) { + fresh->expect_names.push_back(name); + } + fresh->transport = transport_factory_ ? transport_factory_(ctx.gateway_node) + : std::make_unique( + ctx.gateway_node, kParamServiceTimeoutSec, kNegativeCacheTtlSec); + reader_thread_ = std::thread(reader_loop, fresh); + io_ = std::move(fresh); + // Register a pre-shutdown callback: it runs while the context is STILL valid (before + // rclcpp invalidates it, even on a SIGINT), signals the reader to stop and waits (bounded) + // for it to leave its loop. That guarantees the reader is not mid-blocking-rcl-call when + // the context dies - which would SIGABRT (rcl aborts, it does not throw, so it cannot be + // caught). The bounded wait keeps a genuinely hung read from hanging shutdown forever. + context_ = ctx.gateway_node->get_node_base_interface()->get_context(); + std::weak_ptr weak_io = io_; + pre_shutdown_handle_ = context_->add_pre_shutdown_callback([weak_io] { + auto io = weak_io.lock(); + if (!io) { + return; + } + std::unique_lock lk(io->mutex); + io->stop = true; + io->cv.notify_all(); + io->cv.wait_for(lk, std::chrono::seconds(2), [&] { + return io->reader_stopped; + }); + }); + } + + log_config_warnings_once(ctx); + + // Present apps (id + fqn) from the entity snapshot. + std::vector> apps; // (app.id, fqn) + for (const auto & a : ctx.snapshot->apps) { + const std::string fqn = a.effective_fqn(); // bound_fqn or derived; see app.hpp + if (!fqn.empty()) { + apps.emplace_back(a.id, fqn); + } + } + std::sort(apps.begin(), apps.end()); + + forget_rebound_apps(apps); + prune_vanished(apps); + + // Armed targets (fast, NO I/O): only capture/evaluate a node that is armed + lifecycle-ok + // The blocking read for each is done off-thread by reader_loop. + const std::string self_fqn = ctx.gateway_node->get_fully_qualified_name(); + std::vector> armed; + std::vector denied; // gate-denied apps still inside the frozen hold + for (const auto & [app_id, fqn] : apps) { + if (fqn == self_fqn) { + // The GATEWAY's own node, matched on its fully qualified name. Reading it would have the + // process interrogate itself through the hidden client node the transport spins, and its + // own parameters would then start reporting drift. + continue; + } + if (reliability_allows(ctx.gate, app_id)) { + unarmed_ticks_.erase(app_id); + armed.emplace_back(app_id, fqn); + } else { + if (++unarmed_ticks_[app_id] > kFrozenHoldTicks) { + // Held long enough that this is not a pause: release the finding so a fixed parameter can + // stop being reported. It re-raises from a fresh read if the app ever arms again. Past + // this point the app also stops holding back a clear: the hold is what bounds an app the + // gate will never open for, and without a bound one permanently inactive node would keep + // GRAPH_PARAM_DRIFT from ever healing, with no read it could ever make to get out of it. + drifted_.erase(app_id); + pinned_counts_.erase(app_id); + } else { + denied.push_back(app_id); + } + // Do NOT erase this app's drift. The aggregate raise is gated on the aggregate's own + // source, never on app_id, so an unarmed app's finding is not "suppressed anyway" - + // dropping it here is the only thing that would suppress it, and dropping it feeds the + // CLEAR path, which carries no gate at all. A routine lifecycle pause would then heal a + // drift that is still present. The entry stays frozen: not re-evaluated while unarmed, + // not removed either. Removal is the prune path's job, when the app actually goes away. + continue; + } + } + + // Hand the reader thread the current armed set, then evaluate drift from the CACHED reads. + std::map> cached; + std::vector unread; // armed apps with no usable read yet - see emit_aggregated + /// Gate-denied apps with no usable read either. The gate decides whether we may RAISE about an + /// entity; it says nothing about whether we have measured it, and an app that is denied is one + /// the reader is not even allowed to target. "The round-robin has not reached it" and "the gate + /// has not armed it" are the same state - nothing is known - and the clear may not be emitted + /// out of either. Without this, every app is denied for the first `warmup_cycles` ticks after a + /// gateway or plugin start, the armed set is empty, and the detector clears on every one of + /// those ticks before it has issued a single round trip. + std::vector denied_unmeasured; + { + std::lock_guard lk(io_->mutex); + // Carry an app's arming token forward only while it stays armed WITHOUT interruption. An app + // missing from the previous target list is starting a new episode and gets a fresh token, so + // a read still in flight from its last one cannot publish into it. The fqn is part of the + // key for the same reason: a re-bind to a different node under the same app id is a new + // episode too, even though the id never changed. + std::map, std::uint64_t> previous; + for (const auto & t : io_->targets) { + previous.emplace(std::make_pair(t.app_id, t.fqn), t.token); + } + std::vector next; + next.reserve(armed.size()); + for (const auto & [app_id, fqn] : armed) { + const auto pit = previous.find(std::make_pair(app_id, fqn)); + next.push_back(ReadTarget{app_id, fqn, pit != previous.end() ? pit->second : io_->next_token++}); + } + io_->targets = std::move(next); + for (const auto & [app_id, fqn] : armed) { + (void)fqn; + auto it = io_->observed.find(app_id); + if (it != io_->observed.end() && io_->have.count(app_id) > 0) { + cached[app_id] = it->second; // copy out under the lock; evaluate below without it held + } else { + const auto fit = io_->failed_reads.find(app_id); + unread.push_back(UnreadApp{app_id, fit != io_->failed_reads.end() ? fit->second : 0}); + } + } + for (const auto & app_id : denied) { + if (io_->have.count(app_id) == 0) { + denied_unmeasured.push_back(app_id); + } + } + } + io_->cv.notify_one(); + + // Advance the per-app unread bookkeeping BEFORE the blocking set is built: the give-up is a + // conjunction of failed attempts and elapsed ticks, so the tick count has to already exist when + // the set that depends on it is computed. + advance_unread_latches(unread); + + // Apps that hold back a clear: no usable read, and no grounds to stop waiting for one. An app + // the sweep has not attempted (failed_attempts == 0) is in here for as long as that stays true, + // however long the rotation takes - see kUnmeasurableAfterFailures. + std::vector blocking; + for (const auto & app : unread) { + if (!given_up_on(app)) { + blocking.push_back(app.app_id); + } + } + blocking.insert(blocking.end(), denied_unmeasured.begin(), denied_unmeasured.end()); + + for (const auto & [app_id, observed] : cached) { + std::size_t pinned = 0; + const auto d = policy_.evaluate(app_id, observed, /*capture_allowed=*/true, &pinned); + if (!d.empty()) { + drifted_[app_id] = d; + pinned_counts_[app_id] = pinned; + } else { + drifted_.erase(app_id); + pinned_counts_.erase(app_id); + } + } + + report_unmatched_pins_once(ctx, cached, blocking.empty()); + emit_aggregated(ctx, unread, blocking); + } + + private: + /// Name `expect` pins that no app declares, once each. A pin whose parameter name is misspelled + /// comes back NOT_FOUND from every read, and that is deliberately carved out as "not a drift" - + /// a node is not at fault for a parameter it never had. The cost is that such a pin is inert + /// forever, and at runtime inert is indistinguishable from a pin that is checked and passes + /// every time. That is the same silent failure this detector exists to rule out, so it has to + /// say something. + /// + /// Said only once the sweep has COVERED the graph, for the same reason a clear is: only one node + /// is visited per rotation slot, so a perfectly good pin is legitimately unseen for as long as + /// the sweep has not reached the node that declares it. Telling an operator to go and fix a name + /// that is already correct is the same defect as clearing a fault we never measured, and a + /// threshold in ticks fails here in exactly the same way - it expires on the large graphs where + /// the rotation is longest. + /// + /// `swept` is the clear's own condition: every armed app has either been read or been given up + /// on after repeated failures. `cached` non-empty is still required on top of it, because a graph + /// where every app was given up on is swept in name only and says nothing about any pin. + void report_unmatched_pins_once(const DetectorContext & ctx, + const std::map> & cached, + bool swept) { + if (cached.empty() || !swept) { + return; + } + for (const auto & [pin, expected] : policy_.config().expect) { + (void)expected; + const bool declared = std::any_of(cached.begin(), cached.end(), [&pin](const auto & entry) { + return entry.second.count(pin) != 0; + }); + if (declared) { + unmatched_pins_reported_.erase(pin); // a later disappearance is worth saying again + continue; + } + // Node check FIRST: latching without logging would silence the pin for the whole run. + if (!ctx.gateway_node || !unmatched_pins_reported_.insert(pin).second) { + continue; + } + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "param_drift: no app declares '%s', the parameter pinned under 'expect', so that " + "pin can never report a drift; check the name against the node's own parameters", + pin.c_str()); + } + } + + void log_config_warnings_once(const DetectorContext & ctx) { + if (warnings_logged_ || !ctx.gateway_node) { + return; + } + warnings_logged_ = true; + for (const auto & w : policy_.config().warnings) { + RCLCPP_WARN(ctx.gateway_node->get_logger(), "param_drift config: %s", w.c_str()); + } + // Keys the DETECTOR parses rather than the policy (prune_grace) surface here, so an operator + // finds them in the same place and a rejected value is never silent. + for (const auto & w : own_warnings_) { + RCLCPP_WARN(ctx.gateway_node->get_logger(), "param_drift config: %s", w.c_str()); + } + } + + /// Per-app unread bookkeeping for THIS tick: drop the entries of apps that have left the unread + /// set, and advance the tick counter of the ones still in it. + /// + /// The drop is load-bearing and is why this must run on every tick, including ticks where nothing + /// is unread. The latches are one-shot per RUN of failures, not once per process: an app that + /// goes quiet, recovers and goes quiet again has failed twice, and the second time is worth + /// saying to the operator who fixed it the first time. A latch left behind silences that app for + /// the life of the gateway, and silence is also what a working latch looks like. + void advance_unread_latches(const std::vector & unread) { + std::set still_unread; + for (const auto & app : unread) { + still_unread.insert(app.app_id); + } + for (auto it = unread_latches_.begin(); it != unread_latches_.end();) { + it = still_unread.count(it->first) == 0 ? unread_latches_.erase(it) : std::next(it); + } + for (const auto & app : unread) { + ++unread_latches_[app.app_id].ticks; + } + } + + /// Consecutive ticks `app_id` has been armed with no usable read. Zero when it is not in the + /// unread set at all. + int unread_ticks(const std::string & app_id) const { + const auto it = unread_latches_.find(app_id); + return it != unread_latches_.end() ? it->second.ticks : 0; + } + + /// Whether this app has stopped holding back a clear. BOTH counters have to be past the bound: + /// the attempts, because only evidence justifies giving up, and the ticks, because an attempt is + /// not a duration and the attempt horizon alone is scaled by the load knob (kUnreadReportAfterTicks). + /// Strictly past both, matching the give-up warning: at exactly the bound the app still blocks. + bool given_up_on(const UnreadApp & app) const { + return app.failed_attempts > kUnmeasurableAfterFailures && unread_ticks(app.app_id) > kUnmeasurableAfterTicks; + } + + /// Name the apps that are holding back a clear, once each, so the reason a GRAPH_PARAM_DRIFT + /// is not healing is readable instead of having to be inferred. + /// + /// Only apps we ATTEMPTED and failed to read are ever named. An app the round-robin has not + /// reached yet is silent for a reason that is entirely ours, and saying "no parameter read has + /// succeeded for it" would be a bringup warning about a healthy node on every graph whose + /// rotation outlasts the threshold. Reported once per app per run of failures; the latches are + /// created and dropped by advance_unread_latches, which tick() runs first. + void report_unread_once(const DetectorContext & ctx, const std::vector & unread) { + for (const auto & app : unread) { + if (app.failed_attempts <= 0) { + continue; // never attempted: there is nothing to report and nothing to give up on + } + const auto lit = unread_latches_.find(app.app_id); + if (lit == unread_latches_.end()) { + continue; // advance_unread_latches owns this map; it always has an entry for an unread app + } + auto & latch = lit->second; + if (app.failed_attempts >= kUnreadReportAfterFailures && latch.ticks >= kUnreadReportAfterTicks && + !latch.reported && ctx.gateway_node) { + latch.reported = true; + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "param_drift: %d consecutive parameter reads of '%s' have failed; its parameters " + "are unverified and GRAPH_PARAM_DRIFT will not clear while that is true", + app.failed_attempts, app.app_id.c_str()); + } + // Strictly past the bound, matching the blocking set built in tick(): at exactly the bound + // the app still blocks, so announcing that we gave up on it would be premature. + if (given_up_on(app) && !latch.gave_up && ctx.gateway_node) { + latch.gave_up = true; + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "param_drift: giving up on '%s' after %d failed parameter reads; it no longer " + "holds back GRAPH_PARAM_DRIFT from clearing, and its parameters stay unverified", + app.app_id.c_str(), kUnmeasurableAfterFailures); + } + } + } + + /// Say, once per episode, that a clear is being withheld and why. + /// + /// Withholding is correct - health that was never measured is not health - but from the outside + /// it is indistinguishable from a detector that is working and finding nothing: no raise, no + /// clear, and a GRAPH_PARAM_DRIFT already in the store simply never heals. The apps holding it + /// back are frequently ones report_unread_once may not name, because nothing has been ATTEMPTED + /// against them (they are waiting their turn, or the gate has not armed them), so without this + /// the log is silent about the one thing an operator needs in order to act. + void report_withheld_clear(const DetectorContext & ctx, const std::vector & blocking) { + ++withheld_ticks_; + if (withheld_ticks_ <= kWithheldClearReportTicks || withheld_reported_ || !ctx.gateway_node) { + return; + } + withheld_reported_ = true; + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "param_drift: nothing has been found drifting for %d consecutive ticks, but " + "GRAPH_PARAM_DRIFT is being withheld from clearing because %zu app(s) still have no " + "usable parameter read, starting with '%s'. Until every armed app has been read or " + "given up on, the graph has not been measured and no clear is emitted; raise " + "'max_reads_per_tick' or shorten 'tick_interval_ms' if the sweep is simply too slow " + "for the size of this graph", + withheld_ticks_, blocking.size(), blocking.front().c_str()); + } + + /// Drop everything an app's PREVIOUS binding left behind when its fqn changes underneath it. + /// + /// The reader already treats (app_id, fqn) as the identity of an arming episode, so a read in + /// flight from the old binding cannot publish into the new one - but nothing reset the state the + /// old binding had already published. The new node's first reading was then compared against a + /// baseline a DIFFERENT node happened to hold, which is a drift no later read can heal because + /// the reference itself belongs somewhere else, and it inherited the old node's failure count, + /// so a perfectly readable node could be given up on after a handful of its own attempts. + void forget_rebound_apps(const std::vector> & apps) { + for (const auto & [app_id, fqn] : apps) { + const auto it = bound_fqn_.find(app_id); + if (it == bound_fqn_.end()) { + bound_fqn_.emplace(app_id, fqn); + continue; + } + if (it->second == fqn) { + continue; + } + it->second = fqn; + policy_.forget(app_id); + drifted_.erase(app_id); + pinned_counts_.erase(app_id); + unread_latches_.erase(app_id); + if (io_) { + std::lock_guard lk(io_->mutex); + io_->observed.erase(app_id); + io_->have.erase(app_id); + io_->failed_reads.erase(app_id); + io_->targets.erase(std::remove_if(io_->targets.begin(), io_->targets.end(), + [&app_id](const ReadTarget & t) { + return t.app_id == app_id; + }), + io_->targets.end()); + } + } + } + + /// Two different horizons, on purpose. + /// + /// The BASELINE is dropped once the app has been absent from kBaselineForgetGrace + 1 + /// consecutive sweeps - long enough that the absence is no longer explained by the discovery + /// churn WarmupTracker documents. Dropping it on the FIRST missed sweep would re-capture the + /// node's current, possibly already drifted values on an ordinary dropped poll: the confirmed + /// fault would heal while the parameter is still wrong and could never fire again, because the + /// new baseline IS the wrong value. The cost is the other direction - a respawn faster than the + /// window keeps the pre-restart baseline and reports the operator's own change as drift until + /// they revert it. See kBaselineForgetGrace, and the README's recovery-path paragraph. + /// + /// The FINDING is dropped only after prune_grace consecutive absences. Removing it on the first + /// missed sweep would clear the aggregated fault on any one-tick discovery gap, which is + /// flapping, not healing. + /// + /// The two horizons are INDEPENDENTLY configurable and prune_grace can be the shorter one - `0` + /// and `1` are both inside the documented range - so the baseline forget cannot simply wait for + /// the churn window and hope the counter survives that long. Dropping the counter is what makes + /// any later cleanup unreachable: with prune_grace below the churn window the forget would never + /// run for any app, ever, leaving both maps growing without bound under node-name churn and every + /// returning node compared against a stale baseline. So the forget happens at the churn window OR + /// on the tick the counter is dropped, whichever comes first, and a latch keeps it one-shot. + void prune_vanished(const std::vector> & apps) { + std::set present; + for (const auto & [id, fqn] : apps) { + present.insert(id); + } + for (auto it = misses_.begin(); it != misses_.end();) { + if (present.count(it->first) > 0) { + it->second.misses = 0; + it->second.baseline_forgotten = false; + ++it; + continue; + } + ++it->second.misses; + const bool dropping = it->second.misses > prune_grace_; + if (!it->second.baseline_forgotten && (it->second.misses >= kBaselineForgetGrace + 1 || dropping)) { + // Past the churn-absorbing window (or out of ticks in which to do it at all), so treat this + // as a restart rather than a dropped poll: forget the baseline and the cached read so the + // node's return starts from a fresh capture. + it->second.baseline_forgotten = true; + policy_.forget(it->first); + if (io_) { + std::lock_guard lk(io_->mutex); + io_->observed.erase(it->first); + io_->have.erase(it->first); + io_->failed_reads.erase(it->first); + // And from targets, in the SAME critical section. tick() rewrites targets further down, + // so leaving the app there would let a read completing inside this tick republish the + // values just erased - they would become the baseline on return, which is the false, + // unhealable drift the reader's episode fence exists to prevent. + io_->targets.erase(std::remove_if(io_->targets.begin(), io_->targets.end(), + [&it](const ReadTarget & t) { + return t.app_id == it->first; + }), + io_->targets.end()); + } + } + if (dropping) { + drifted_.erase(it->first); + pinned_counts_.erase(it->first); + // The frozen-hold counter goes with the finding it holds. Nothing else erases it off the + // arm path, and discovery has no hidden-node filter, so every short-lived `ros2` CLI node + // would otherwise leave one entry behind for the life of the process. + unarmed_ticks_.erase(it->first); + // Same for the remembered binding: it is written for every present app and nothing else + // reaches it, so it leaks under node-name churn exactly as the counter above did. + bound_fqn_.erase(it->first); + it = misses_.erase(it); + } else { + ++it; + } + } + for (const auto & id : present) { + misses_.emplace(id, AbsenceCounter{}); + } + } + + /// Level-triggered aggregation: one graph-level fault carrying every app's drift, through the + /// same helper qos_mismatch and orphan use. Going through AggregatedFault rather than building + /// the text here is what keeps three properties this detector previously got wrong: the fault + /// is raised under graph_source_id() (a constant, owned entity - a host Component id would + /// match no entity scope at all, so the fault would be reachable from no endpoint), the + /// description stops being built once it exceeds the cap instead of concatenating the whole + /// graph first, and the cap itself is the helper's constant rather than a second copy of 480. + /// + /// `order` carries the priority: entries an operator explicitly pinned with `expect` come + /// first, then self-captured drift, and within each group the newly detected apps precede the + /// ones already reported. Without that, a capped description can hide exactly the two things + /// worth reading - the pin the operator declared, and whatever just changed. + void emit_aggregated(DetectorContext & ctx, const std::vector & unread, + const std::vector & blocking) { + // A clear asserts that nothing is drifting anywhere. That is only true if we actually LOOKED + // everywhere. drifted_ is built from cached reads alone, so an app whose parameter service + // never answers contributes nothing to it - and without this guard an empty drifted_ would + // be reported as health the detector never measured, every tick, forever. AggregatedFault + // decides raise-versus-clear from emptiness alone, so the guard has to live here: when there + // is nothing to report AND something we could not read, emit nothing at all. + // + // A raise is never withheld: a drift found on the apps that did answer is real either way. + // Every tick, including ticks with nothing unread: that call is what resets the latches. + report_unread_once(ctx, unread); + if (drifted_.empty() && !blocking.empty()) { + report_withheld_clear(ctx, blocking); + return; + } + withheld_ticks_ = 0; + withheld_reported_ = false; + std::map affected; + for (const auto & [app, items] : drifted_) { + std::string detail; + for (const auto & item : items) { + detail += (detail.empty() ? "" : "; ") + item; + } + if (detail.size() > kMaxAppDetailChars) { + // Keep both ends: the head names the app and its first drifted parameter, the tail keeps the + // count of what followed readable rather than ending mid-token. + const std::size_t keep = kMaxAppDetailChars - 3; + const std::size_t head = (keep + 1) / 2; + detail = detail.substr(0, head) + "..." + detail.substr(detail.size() - (keep - head)); + } + affected.emplace(app, detail); + } + aggregated_.emit_ordered(ctx, affected, emit_order()); + reported_ = drifted_; + } + + /// App ids in the order their details should appear in the description. Keys of `affected` + /// missing here are appended by the helper, so a gap degrades the ordering rather than + /// dropping an app. + std::vector emit_order() const { + std::vector pinned_new, pinned_old, baseline_new, baseline_old; + for (const auto & [app, items] : drifted_) { + (void)items; + const bool is_new = reported_.count(app) == 0; + const auto pit = pinned_counts_.find(app); + const bool pinned = pit != pinned_counts_.end() && pit->second > 0; + auto & bucket = pinned ? (is_new ? pinned_new : pinned_old) : (is_new ? baseline_new : baseline_old); + bucket.push_back(app); + } + std::vector order; + order.reserve(drifted_.size()); + for (const auto * bucket : {&pinned_new, &pinned_old, &baseline_new, &baseline_old}) { + order.insert(order.end(), bucket->begin(), bucket->end()); + } + return order; + } + + ParamDriftPolicy policy_; + int tick_interval_ms_ = kDefaultTickIntervalMs; // plugin-injected; paces the reader thread + std::shared_ptr io_; // shared with the background reader thread + std::thread reader_thread_; + rclcpp::Context::SharedPtr context_; // for the pre-shutdown callback + rclcpp::PreShutdownCallbackHandle pre_shutdown_handle_; + /// Consecutive missing sweeps for one app, plus the one-shot latch that keeps the baseline + /// forget from repeating once the churn window has passed. + struct AbsenceCounter { + int misses = 0; + bool baseline_forgotten = false; + }; + + std::map misses_; // app.id -> absence bookkeeping + std::map> drifted_; // app.id -> its drifted descriptors + std::map> reported_; // drifted_ as of the previous emit + std::map pinned_counts_; // app.id -> leading `expect` descriptors + std::map unread_latches_; // app.id -> which unread warnings were logged + std::set unmatched_pins_reported_; // expect pins already reported as declared by no app + std::map unarmed_ticks_; // app.id -> consecutive ticks the gate denied it + std::map bound_fqn_; // app.id -> the fqn it was last seen bound to + int withheld_ticks_ = 0; // consecutive ticks a clear has been held back + bool withheld_reported_ = false; // the withheld-clear warning fired for this episode + int prune_grace_ = kForgetGrace; // plugin-injected; absences before a finding is dropped + std::vector own_warnings_; // config problems the detector itself found + ParameterTransportFactory transport_factory_; // empty = build the real ROS 2 transport + AggregatedFault aggregated_{graph_fault_codes::kParamDrift, kParamDriftSeverity}; + bool warnings_logged_ = false; +}; + +REGISTER_DETECTOR(ParamDriftDetector, "param_drift") + +} // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py index 2198d4af3..1f65aea98 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py @@ -31,9 +31,13 @@ on PYTHONPATH once the workspace's install/setup.bash is sourced. """ +import json import os import time +from launch import LaunchDescription +from launch.actions import TimerAction +import launch_testing.actions import requests from ros2_medkit_test_utils.constants import get_test_port from ros2_medkit_test_utils.launch_helpers import ( @@ -42,16 +46,13 @@ create_gateway_node, ) -from launch import LaunchDescription -from launch.actions import TimerAction -import launch_testing.actions - API_BASE_PATH = '/api/v1' def create_watchdog_test_launch( *, detector_params=None, + extra_gateway_params=None, demo_nodes=None, port=None, demo_delay=2.0, @@ -66,6 +67,11 @@ def create_watchdog_test_launch( keys, e.g. ``'plugins.graph_watchdog.detectors.param_drift.mode'``), merged on top of the ``plugins``/``plugins.graph_watchdog.path`` parameters this factory always sets. + extra_gateway_params : dict or None + Extra ROS parameters for the gateway node itself rather than for the + plugin (e.g. ``'parameter_service_negative_cache_sec'``). Kept separate + from `detector_params` so a gateway-scope knob is not filed under a + plugin-scope name; both end up in the same node config. demo_nodes : list of str or None Node keys from ``ros2_medkit_test_utils.launch_helpers.DEMO_NODE_REGISTRY``. ``None`` means no demo nodes. @@ -95,6 +101,8 @@ def create_watchdog_test_launch( } if detector_params: params.update(detector_params) + if extra_gateway_params: + params.update(extra_gateway_params) gateway_node = create_gateway_node(port=port, extra_params=params) @@ -155,7 +163,7 @@ def graph_watchdog_plugin_path(): return path -def wait_until_watchdog_armed(port, timeout=60.0, interval=0.5): +def wait_until_watchdog_armed(port, timeout=60.0, interval=0.5, app_id=None): """Poll ``GET /api/v1/x-medkit-watchdog`` until the plugin is live and armed. Any assertion that a fault is ABSENT must gate on this first. Absence is the @@ -173,6 +181,28 @@ def wait_until_watchdog_armed(port, timeout=60.0, interval=0.5): ``global_state == 'armed'`` additionally proves the bringup grace elapsed, i.e. the gate would have permitted a raise during the window that follows. + `app_id` narrows that from "some app is armed" to "THIS app is armed", which + is what a test that is about to perturb one specific node needs. The gate is + what decides whether a detector may target an app at all (param_drift builds + its read set from ``reliability_allows`` - see its tick()), so an armed + entry is the precondition for the very first read of that node: gating on it + means a perturbation can no longer land before the detector was even + permitted to look, and a stack that comes up late fails HERE, naming the + node, instead of surfacing later as an unexplained missing fault. + + The check mirrors ``ReliabilityGate::allows_raise``: the payload's + per-entity ``state`` is ``armed`` only when the warmup has elapsed AND the + lifecycle watcher considers the node ok, which is exactly the conjunction + that permits a raise. The boolean ``armed`` field alone is only the warmup + half. + + What this does NOT prove is that a detector has already READ the app - no + e2e surface exposes that. The plugin's only route reports gate state, and a + detector's level-triggered PASSED for a fault that does not exist yet is + dropped by the fault_manager (see ``report_fault_event``), so a clear is + invisible until something has been raised. Callers that need a captured + baseline must still allow the sweep a window after this returns. + Parameters ---------- port : int @@ -181,26 +211,49 @@ def wait_until_watchdog_armed(port, timeout=60.0, interval=0.5): Maximum time to wait in seconds. interval : float Sleep between retries in seconds. + app_id : str or None + App id (as the gateway derives it from the ROS node name) that must be + present in ``entities`` with ``state == 'armed'``. ``None`` only + requires some entity plus a global ``armed`` state. Returns ------- bool - ``True`` once the plugin reports at least one entity and a global - ``armed`` state, ``False`` on timeout. + ``True`` once the plugin reports a global ``armed`` state together with + at least one entity (or with `app_id` armed, when given), ``False`` on + timeout. """ base = f'http://127.0.0.1:{port}{API_BASE_PATH}' deadline = time.monotonic() + timeout + last_seen = 'GET /x-medkit-watchdog was never answered at all' while time.monotonic() < deadline: try: response = requests.get(f'{base}/x-medkit-watchdog', timeout=5) - if response.status_code == 200: + if response.status_code != 200: + # Worth naming rather than folding into the generic message: the route is + # registered by the plugin itself, so a 404 from a gateway that is otherwise + # answering says the .so never loaded. + last_seen = f'HTTP {response.status_code} from GET /x-medkit-watchdog' + else: status = response.json().get('x-medkit-watchdog', {}) - if status.get('entities') and status.get('global_state') == 'armed': + last_seen = json.dumps(status) + entities = status.get('entities') or [] + if app_id is None: + armed = bool(entities) + else: + armed = any( + e.get('id') == app_id and e.get('state') == 'armed' for e in entities) + if armed and status.get('global_state') == 'armed': return True - except requests.exceptions.RequestException: - pass + except requests.exceptions.RequestException as exc: + last_seen = f'GET /x-medkit-watchdog failed: {exc}' time.sleep(interval) + # The caller turns this into an assertion failure, but only it knows what it + # was waiting for; the gate state itself is the evidence for WHY, and it is + # gone once the launch tears down. Print it while it still exists. + print(f'wait_until_watchdog_armed(app_id={app_id!r}) timed out after {timeout}s; ' + f'last watchdog status: {last_seen}') return False diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_config_plumbing_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_config_plumbing_e2e.test.py new file mode 100644 index 000000000..9ee26ead6 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_config_plumbing_e2e.test.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Config-plumbing e2e: proves NESTED plugin config reaches a live detector +through the REAL gateway. + +A C++ unit test cannot prove this - it hand-builds already-nested JSON and +calls a detector's configure() directly, bypassing the real config-delivery +path (extract_plugin_config -> detector_config.hpp -> +ParamDriftConfig::from_json) entirely. That bypass is exactly what hid the +pre-existing bug this test guards against a regression of: detector_config.hpp +used to read a FLAT dotted-key shape while the gateway delivers NESTED, and +even after fixing that, param_drift's own `expect` sub-object was read via a +flat "expect." prefix scan and silently dropped through the real gateway. + +Uses a param_drift `expect` rule, which fires on a STATIC graph (no runtime +drift needed - expect rules are absolute, see param_drift_policy.hpp and +test_param_drift_integration.cpp's ExpectRuleFlagsWrongFromStart). This +exercises BOTH config layers at once: the `expect` field (extract_detector_config +-> ParamDriftConfig::from_json) and `mode` (detector_config.hpp). + +Runs as THREE separate CTest targets (see CMakeLists.txt), each launching its +OWN gateway+fault_manager+demo-node stack (config is read once at plugin +set_context() time, so proving both layers needs two distinct launches, not +two test methods sharing one). The WATCHDOG_E2E_SCENARIO env var selects +which launch + assertion runs in this process: + +- "positive": plugins.graph_watchdog.detectors.param_drift.expect.use_sim_time + = True, with NO mode override. The demo node runs with its real default + use_sim_time=False, an immediate mismatch -> GRAPH_PARAM_DRIFT must raise. + If the `expect` nested reader (layer 2) were still broken, `expect` would + be silently dropped and this would never fire. +- "mode_off": same expect mismatch, PLUS + plugins.graph_watchdog.detectors.param_drift.mode = "off" -> GRAPH_PARAM_DRIFT + must never appear. If the `mode` nested reader (layer 1, detector_config.hpp) + were still broken, "off" would never be recognized and this detector would + keep raising despite the operator disabling it. +- "mode_off_yaml_bool": same, but `mode` is the YAML 1.1 boolean a BARE `off` + actually parses to. This is the form operators write, and it reaches the + plugin as a JSON bool, never a string. + +Both "off" scenarios assert an ABSENCE, so both first gate on the plugin's own +GET /x-medkit-watchdog route (harness.wait_until_watchdog_armed). Without that +gate the assertion passes just as well on a stack that never came up: the +gateway survives a plugin that fails to dlopen and a REST server that cannot +bind, both still exit 0, and poll_faults returns None on any transport error. +""" + +import os +import sys +import unittest + +import launch_testing + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from harness import ( # noqa: E402 + create_watchdog_test_launch, + poll_faults, + wait_until_watchdog_armed, +) + +from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES, get_test_port # noqa: E402 + +# No default on purpose. A default makes this file FAIL OPEN: if the CTest ENVIRONMENT property +# never reaches the process - an edit that drops the variable from one of the three targets, +# someone running launch_test on the file directly - then generate_test_description() also skips +# its `mode` override, so the launch and the assertions degrade together into a second copy of the +# positive scenario and CTest still reports 1/1 passed. A KeyError here is loud and immediate. +SCENARIO = os.environ['WATCHDOG_E2E_SCENARIO'] +PORT = get_test_port() + +# Fast tick cadence + short warmup so the whole story (gateway/fault_manager +# startup, demo-node discovery, per-app arm, global bringup grace - see the +# plugin README's "Reliability (bringup-quiesce)") comfortably fits inside +# the poll_faults() timeouts below without any hand-tuned pre-sleep: polling +# itself absorbs demo_delay + discovery + warmup, retrying every interval +# until the assertion's own timeout. +TICK_INTERVAL_MS = 200 +WARMUP_CYCLES = 3 + +# The two shapes `mode` actually arrives in. A QUOTED "off" in YAML stays a +# STRING; a BARE `off` is YAML 1.1, which rcl_yaml_param_parser types as BOOL +# False (as do `Off`, `OFF`, `no`, `n`). `off` bare is what operators write, so +# both shapes have to disable the detector - see parse_detector_mode(). +_MODE_OFF_STRING = 'off' +_MODE_OFF_YAML_BOOL = False + + +def generate_test_description(): + detector_params = { + 'plugins.graph_watchdog.tick_interval_ms': TICK_INTERVAL_MS, + 'plugins.graph_watchdog.warmup_cycles': WARMUP_CYCLES, + 'plugins.graph_watchdog.detectors.param_drift.baseline': False, + 'plugins.graph_watchdog.detectors.param_drift.expect.use_sim_time': True, + } + if SCENARIO == 'mode_off': + detector_params['plugins.graph_watchdog.detectors.param_drift.mode'] = _MODE_OFF_STRING + elif SCENARIO == 'mode_off_yaml_bool': + detector_params['plugins.graph_watchdog.detectors.param_drift.mode'] = _MODE_OFF_YAML_BOOL + + return create_watchdog_test_launch( + detector_params=detector_params, + demo_nodes=['temp_sensor'], + port=PORT, + ) + + +class TestConfigPlumbingPositive(unittest.TestCase): + """Positive control: the nested `expect` config reaches param_drift.""" + + def test_expect_rule_raises_from_nested_config(self): + fault = poll_faults(PORT, 'GRAPH_PARAM_DRIFT', timeout=60.0) + self.assertIsNotNone( + fault, + 'GRAPH_PARAM_DRIFT never raised - the nested `expect` config was ' + 'not delivered to param_drift through the real gateway', + ) + + +class TestConfigPlumbingModeOff(unittest.TestCase): + """Mode suppression: the nested `mode` config disables the detector.""" + + def test_mode_off_suppresses_detector(self): + # Gate on the plugin's own route BEFORE asserting absence. Without this the + # assertion is vacuous by construction: a stack that never came up produces + # exactly the same "no fault" result, and every failure mode on this launch + # (plugin fails to dlopen, REST server cannot bind) still exits 0. This runs + # in its own process against its own gateway on its own port, so nothing in + # the positive scenario covers it either. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed gate - the plugin did not load ' + 'or its tick loop never ran, so an absent fault proves nothing', + ) + # Proving absence means waiting out the full window: a detector that + # (regression) ignored `mode` would have raised well within it. + fault = poll_faults(PORT, 'GRAPH_PARAM_DRIFT', timeout=20.0) + self.assertIsNone( + fault, + 'GRAPH_PARAM_DRIFT raised despite ' + 'detectors.param_drift.mode="off" - the nested `mode` config was ' + 'not delivered to (or not honored by) the plugin', + ) + + +class TestConfigPlumbingModeOffYamlBool(unittest.TestCase): + """A BARE `off` disables the detector, not just a quoted one.""" + + def test_yaml_boolean_off_suppresses_detector(self): + # `mode: off` written bare in YAML never reaches the plugin as a string: + # rcl_yaml_param_parser applies YAML 1.1 and types it BOOL False. A reader + # that only accepts strings therefore leaves the detector in its `raise` + # default, and the detector the operator just disabled keeps pushing + # GRAPH_PARAM_DRIFT at the robot with nothing in the log to explain it. + # This is the shape operators actually write, so it gets its own launch. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed gate - the plugin did not load ' + 'or its tick loop never ran, so an absent fault proves nothing', + ) + fault = poll_faults(PORT, 'GRAPH_PARAM_DRIFT', timeout=20.0) + self.assertIsNone( + fault, + 'GRAPH_PARAM_DRIFT raised despite a bare ' + 'detectors.param_drift.mode: off - the YAML boolean form of "off" ' + 'was not honored by the plugin', + ) + + +# Each CTest target launches this file with one scenario, so only that scenario's case may +# run. Marking the other two as skipped would report them as skipped on every single run, +# which is exactly how a scenario that has genuinely stopped working would look. Removing +# them from the module instead means each run reports one case, and a missing result is a +# real failure rather than an expected line of output. +_SCENARIO_CASES = { + 'positive': 'TestConfigPlumbingPositive', + 'mode_off': 'TestConfigPlumbingModeOff', + 'mode_off_yaml_bool': 'TestConfigPlumbingModeOffYamlBool', +} +if SCENARIO not in _SCENARIO_CASES: + raise RuntimeError( + f'WATCHDOG_E2E_SCENARIO={SCENARIO!r} is not one of {sorted(_SCENARIO_CASES)}; ' + 'the CTest target and this file disagree about which scenarios exist') +for _scenario, _case_name in _SCENARIO_CASES.items(): + if _scenario != SCENARIO: + del globals()[_case_name] + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + """Verify the gateway/fault_manager/demo stack exits cleanly.""" + + def test_exit_codes(self, proc_info): + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'Process {info.process_name} exited with {info.returncode}', + ) diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_drift_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_drift_e2e.test.py new file mode 100644 index 000000000..cd527d003 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_drift_e2e.test.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""param_drift e2e for the mode operators actually get: self-capturing BASELINE drift. + +The three config-plumbing scenarios all set ``baseline: false`` and assert only that a fault +appears or does not, so until this file existed the detector's DEFAULT and headline behaviour - +capture a parameter's value, notice when it changes at runtime - had no end-to-end coverage, and +no scenario asserted a clear at all. The whole "Closing the loop" section of the README rests on +the clear reaching the fault_manager and healing there. + +What this drives, all real: a gateway process with the plugin ``.so`` loaded, a real +fault_manager, a real ROS node whose parameter this test changes over the real parameter service, +and the operator-visible ``GET /api/v1/faults`` surface. + +Unlike test_orphan_e2e.test.py, the node here MUST be spun: the detector reads its parameters over +a service, and an unspun node answers nothing, so the detector would report it as never read +instead of capturing a baseline. + +Everything below is anchored to the plugin's own GET /x-medkit-watchdog route +(harness.wait_until_watchdog_armed) rather than to wall-clock time from process start. This file +both asserts an ABSENCE and then perturbs a parameter, and each needs the gate for its own reason: +an absence proves nothing about a stack that never came up, and the detector may not read a node +before the gate arms it, so a perturbation that lands first turns the DRIFTED value into the +baseline and no fault can ever raise. +""" + +import os +import sys +import threading +import unittest + +import launch_testing +import rclpy +from rclpy.node import Node + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +# I100 as well as E402: `harness` is only importable because of the sys.path line above, so this +# import cannot be moved up to where the alphabetical order would put it. +from harness import ( # noqa: E402, I100 + create_watchdog_test_launch, + poll_cleared, + poll_faults, + wait_until_watchdog_armed, +) + +from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES, get_test_port # noqa: E402 + +PORT = get_test_port() + +# Fast tick + short warmup so the whole story fits inside the poll timeouts without a hand-tuned +# pre-sleep - same rationale as the other e2e files here. max_reads_per_tick is raised because the +# real graph carries the gateway, the fault_manager and this test's node, and the sweep is +# round-robin: at the default budget the node under test waits several ticks for its turn. +TICK_INTERVAL_MS = 200 +WARMUP_CYCLES = 3 +FAULT_CODE = 'GRAPH_PARAM_DRIFT' +# The ROS node name, which is also the App id the gateway derives for a node in the root +# namespace - so it is both what this test spins up and what the watchdog gate reports as an +# entity. Used for the arming gate and for the "the fault names the node" assertion, which must +# not be able to drift apart. +TARGET_NODE = 'pd_e2e_target' +PARAM_NAME = 'speed_limit' +BASELINE_VALUE = 1.0 +DRIFTED_VALUE = 0.2 + +# How long the detector gets to capture a baseline, measured from the moment the gate armed the +# target (NOT from process start - bringup is absorbed by the gate, which is the point of it). +# Sized to exceed the parameter transport's 10 s negative-cache TTL, so a first read that timed +# out and backed the node off is retried and still lands inside the window. The C++ integration +# suite sizes its own poll bound off the same constant for the same reason. +SILENT_CAPTURE_SEC = 15.0 + + +def generate_test_description(): + detector_params = { + 'plugins.graph_watchdog.tick_interval_ms': TICK_INTERVAL_MS, + 'plugins.graph_watchdog.warmup_cycles': WARMUP_CYCLES, + 'plugins.graph_watchdog.detectors.param_drift.max_reads_per_tick': 32, + } + return create_watchdog_test_launch( + detector_params=detector_params, + demo_nodes=None, + port=PORT, + # The clear must reach HEALED, i.e. leave the default active-fault query, which the + # fault_manager only does when healing is enabled - see harness.py and the README's + # "Closing the loop". + healing_enabled=True, + ) + + +class TestParamDriftE2e(unittest.TestCase): + """Baseline drift raises through the real stack, and restoring the value heals it.""" + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._node = Node(TARGET_NODE) + cls._node.declare_parameter(PARAM_NAME, BASELINE_VALUE) + cls._stop = threading.Event() + cls._spin = threading.Thread(target=cls._spin_until_stopped, daemon=True) + cls._spin.start() + + @classmethod + def _spin_until_stopped(cls): + # The node has to answer parameter-service calls for the detector to read it at all. + while not cls._stop.is_set() and rclpy.ok(): + rclpy.spin_once(cls._node, timeout_sec=0.1) + + @classmethod + def tearDownClass(cls): + cls._stop.set() + cls._spin.join(timeout=5.0) + cls._node.destroy_node() + rclpy.shutdown() + + def test_01_runtime_change_raises_and_names_the_parameter(self): + # Gate on the target being ARMED before anything below runs. Two distinct things go wrong + # without it, and neither is visible in the result: + # + # - The absence assertion is vacuous against a stack that never came up. A plugin that + # fails to dlopen, a REST server that cannot bind - the gateway survives both and exits + # 0, and poll_faults swallows every transport error and returns None, which is exactly + # what assertIsNone wants to see. + # - Worse, the flip below is only a drift if a baseline was captured first, and the + # detector may not target a node until the gate arms it. On a slow runner a fixed + # pre-window elapses during bringup, set_parameters lands before the first read, the + # DRIFTED value silently becomes the baseline, and no fault can ever raise - the run + # then dies at the 60 s poll below blaming the detector for a race in this test. + # + # Anchoring both to the gate makes bringup cost time here, where the failure message + # names the real cause, instead of eating the window that is supposed to cover the read. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} as an armed entity - the plugin did not ' + f'load, its tick loop never ran, or the gateway never discovered the node. The ' + f'detector reads only apps the gate arms, so nothing below would mean anything', + ) + + # Give the detector time to capture the baseline BEFORE changing anything. Capturing + # must be silent, so a fault appearing here at all would mean the detector reports its + # own first read as drift. + self.assertIsNone( + poll_faults(PORT, FAULT_CODE, timeout=SILENT_CAPTURE_SEC), + 'GRAPH_PARAM_DRIFT was raised before any parameter changed, so baseline capture is ' + 'not silent', + ) + + self._node.set_parameters( + [rclpy.parameter.Parameter(PARAM_NAME, rclpy.Parameter.Type.DOUBLE, DRIFTED_VALUE)]) + + fault = poll_faults(PORT, FAULT_CODE, timeout=60.0) + self.assertIsNotNone( + fault, + f'{FAULT_CODE} never raised after {PARAM_NAME} changed from {BASELINE_VALUE} to ' + f'{DRIFTED_VALUE} at runtime', + ) + description = fault.get('description', '') + self.assertIn(PARAM_NAME, description) + self.assertIn(TARGET_NODE, description) + + def test_02_restoring_the_value_heals_the_fault(self): + # "Cleared" is an absence too, and it is the default state of a fault that was never + # raised: on the stack test_01 just failed against, poll_cleared answers True on its + # first request. Confirm the fault is actually THERE first, so what this measures is a + # heal and not a fault that never happened. test_01 raised it and the drifted value is + # still set, so it is present now or the level-triggered raise is not level-triggered. + self.assertIsNotNone( + poll_faults(PORT, FAULT_CODE, timeout=30.0), + f'{FAULT_CODE} is not active at the start of the heal test, so there is nothing here ' + f'that could heal and the clear below would pass without measuring anything', + ) + + # This is the path the README tells operators to rely on: put the value back and the fault + # goes away on its own. It only works if the detector keeps emitting a level-triggered + # PASSED and the fault_manager counts those toward healing. + self._node.set_parameters( + [rclpy.parameter.Parameter(PARAM_NAME, rclpy.Parameter.Type.DOUBLE, BASELINE_VALUE)]) + + self.assertTrue( + poll_cleared(PORT, FAULT_CODE, timeout=60.0), + f'{FAULT_CODE} did not heal after {PARAM_NAME} was restored to {BASELINE_VALUE}', + ) + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + """Verify the gateway/fault_manager stack exits cleanly.""" + + def test_exit_codes(self, proc_info): + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'Process {info.process_name} exited with {info.returncode}', + ) diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_roundtrip_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_roundtrip_e2e.test.py new file mode 100644 index 000000000..26d30171e --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_roundtrip_e2e.test.py @@ -0,0 +1,658 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""What one parameter read really costs the node that serves it. + +``param_drift`` charges its ``max_reads_per_tick`` budget in parameter-service ROUND TRIPS +and not in transport calls, because one call is several round trips at the watched node: +``kBaselineReadRoundTrips`` (2), ``kExpectReadRoundTrips`` (3) and +``kExpectNotFoundRoundTrips`` (2) in ``src/detectors/param_drift_detector.cpp``. Every other +test in this package proves the detector's arithmetic GIVEN those numbers, and the stand-in +transport they use counts CALLS, so not one of them can tell whether the TRANSPORT still costs +what those numbers say. Add a fourth service call inside ``Ros2ParameterTransport`` and the +whole suite stays green while the load bound this feature sells quietly becomes a lie. + +This test measures the transport. It runs the REAL one - the gateway compiles the same +``ros2_parameter_transport.cpp`` the plugin does (see ``PARAMETER_TRANSPORT_SOURCES`` in +CMakeLists.txt) - against a node that answers the parameter services BY HAND and counts every +request it is asked to serve. The counts are what the node paid, not what anything claims it +paid. + +What it does NOT do, and must not be relied on for, is check the detector's own literals. It +never reads them: ``param_drift`` is switched OFF here and the reads are driven over HTTP, so +setting ``kBaselineReadRoundTrips`` to 4 and ``kExpectReadRoundTrips`` to 6 with the transport +untouched leaves this file green. What catches a CHARGE that has drifted away from the +measurement is the timing suite in ``test/test_param_drift_integration.cpp`` +(``ABaselineVisitIsChargedTwoRoundTripsAndNotTheColdStartPair``, +``TheBudgetIsSpentInRoundTripsNotCalls``, ``AnUndeclaredPinIsChargedLessThanOneThatResolves``), +which times the reader against a configured budget. The two halves are complementary and +neither is redundant: this file pins the cost, those pin the charge against it. + +The gateway is launched here with a 5 s per-call timeout and the negative cache disabled, while +the detector builds its transport with 0.5 s and a 10 s cache. That difference does not affect +anything measured below: a round-trip count on the SUCCESS path is the number of requests the +transport issues, which is the same whatever the deadline on each of them is. The generous +settings exist only so a slow first DDS discovery cannot turn a success into a TIMEOUT, or lock +a probe out for a minute after one early attempt, and so make the first-contact measurements +repeatable. + +Why the reads are driven over HTTP rather than by the detector. The constants describe two +functions, ``Ros2ParameterTransport::list_parameters`` and ``::get_parameter``, and +``GET /{entity}/configurations`` / ``GET /{entity}/configurations/{id}`` invoke exactly those, +once per backing node, synchronously, on demand (``ConfigurationManager`` delegates 1:1). That +is what makes a count attributable to ONE call. The detector reaches the same two functions +from a background round-robin sweep that never stops and exposes no visit boundary, so driving +it would only ever yield an increment pattern smeared over the sweep's pacing - a weaker claim +about the same code. ``param_drift`` is therefore switched OFF here, so the only parameter +traffic the probe node sees is the traffic this test asks for. + +The four costs pinned below, and how they relate to what the detector charges: + +* cold baseline visit = 4 - ``list_parameters`` primes the transport's defaults cache before + its own read (list + get, then list + get). That cache has no TTL, so this is a one-off per + node per gateway lifetime, and the detector deliberately does NOT charge it. +* warm baseline visit = 2 - matches ``kBaselineReadRoundTrips``. +* resolved ``expect`` pin = 3 - matches ``kExpectReadRoundTrips`` (name-scoped list, get, + describe; describe is unconditional on the success path). +* unresolved ``expect`` pin = 1 - the name-scoped list comes back empty and the transport + returns NOT_FOUND before the get. The detector charges 2 for this case ON PURPOSE: a second, + rarer NOT_FOUND path costs two round trips and the error code cannot tell the two apart, so + it charges the higher of them. A bound may overstate a cost, never understate one - so 1 here + is the real cost and is ALLOWED to be below the charge. Do not "fix" either number to match + the other. +* a listed name the get returns no value for = 2 - that second NOT_FOUND path, measured, so the + charge of 2 is anchored from above as well as below. +* a DECLARED BUT UNSET parameter = 3, and a SUCCESS rather than a NOT_FOUND. rclcpp answers one + with a ``PARAMETER_NOT_SET`` value, not with an absent one, so the transport gets a parameter + back and pays for the describe like any other resolved pin. It is charged 3, which is right. + This case is pinned separately because the two-round-trip path above was previously described + as being it. +* FIRST resolved ``expect`` pin against an untouched node = 3, the same as any later one. + ``get_parameter`` never primes the defaults cache, so the cold-contact pair is a property of + ``list_parameters`` and of ``baseline: true`` mode, not of reading a node for the first time. + Measured against a second probe that nothing has listed. +""" + +import os +import sys +import threading +import time +import unittest + +import launch_testing +from rcl_interfaces.msg import ( + ParameterDescriptor, + ParameterType, + ParameterValue, + SetParametersResult, +) +from rcl_interfaces.srv import ( + DescribeParameters, + GetParameters, + GetParameterTypes, + ListParameters, + SetParameters, + SetParametersAtomically, +) +import rclpy +from rclpy.executors import SingleThreadedExecutor +from rclpy.node import Node +import requests + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from harness import create_watchdog_test_launch # noqa: E402 + +from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES, get_test_port # noqa: E402 + +PORT = get_test_port() +API_BASE = f'http://127.0.0.1:{PORT}/api/v1' + +PROBE_NODE = 'pd_rt_probe' +# A SECOND probe, identical but never listed. The cold-contact pair belongs to +# list_parameters (which primes the transport's defaults cache) and not to get_parameter +# (which does not), so proving that needs a node whose cache is still unprimed when the pinned +# read reaches it - which the first probe stops being the moment test_02 lists it. +PIN_PROBE_NODE = 'pd_rt_pinprobe' +# Two declared parameters, so the batched get after a full list carries more than one name and +# a regression that silently drops the batch cannot pass by returning a single value. +DECLARED_PARAMS = {'probe_gain': 2.5, 'probe_margin': 0.5} +# A name the probe never declares: the name-scoped list answers empty and the transport must +# stop there. +UNDECLARED_PARAM = 'pd_rt_no_such_parameter' +# A name the probe lists and then returns NO VALUE for. This is the second, rarer NOT_FOUND path +# and the whole justification for charging NOT_FOUND two round trips rather than one: the +# name-scoped list answers, so the transport goes on to the get, and only the get finds there is +# nothing there. The error code is the same NOT_FOUND as the one-round-trip case and the caller +# cannot tell which it took, so the charge takes the higher of the two. +# +# The GetParameters contract does not require one value per requested name, so a reply that simply +# omits it is well formed - and it is the only reply that reaches this branch. See NOT_SET_PARAM +# for the case this was previously described as, which behaves quite differently. +NO_VALUE_PARAM = 'pd_rt_no_value_returned' +# A name the probe lists and answers with a PARAMETER_NOT_SET value: a parameter that is DECLARED +# BUT UNSET, which is what rclcpp's own parameter service replies for one. That reply is a SUCCESS +# carrying a null value, not a NOT_FOUND: the transport gets a parameter back, goes on to the +# describe, and returns it. It therefore costs 3 and is charged 3 like any other resolved pin. +# Pinned here because the 2-round-trip NOT_FOUND above used to be documented as this case, which +# would have made the charge for a resolved-but-null read an understatement of what the node pays. +# Measured on the SECOND probe, whose full list nothing asserts on; adding either name to the +# first would change what test_02 and test_03 read back. +NOT_SET_PARAM = 'pd_rt_declared_but_unset' + +# Per-request HTTP patience. Generous enough for a first request that races gateway startup, short +# enough that several of them still fit inside the ctest timeout with the retry deadlines below. +HTTP_TIMEOUT = 15 + +# The six services rclcpp's AsyncParametersClient requires before it reports ready. Only three +# of them are ever expected to carry traffic; the rest exist so wait_for_service() succeeds, +# and are counted anyway - a transport that started calling one of them would show up here +# rather than hide inside a total. +SERVICE_NAMES = ( + 'list_parameters', + 'get_parameters', + 'describe_parameters', + 'get_parameter_types', + 'set_parameters', + 'set_parameters_atomically', +) + + +def _double(value): + """Build a well-formed DOUBLE ``ParameterValue``.""" + return ParameterValue(type=ParameterType.PARAMETER_DOUBLE, double_value=float(value)) + + +class _RequestFailed: + """Stand-in for a response that never arrived. + + ``_measure`` is called from retry loops that decide what to do next from + ``response.status_code``, and the HTTP call it wraps is reachable in precisely the state + those loops exist for: a read that degenerates and outlives the client's patience. Letting + ``requests`` raise out of it aborts the retry with a traceback rather than retrying, and none + of the diagnostics the loop was written to print ever run - the failure report is then a stack + trace about a socket instead of a statement about round trips. + """ + + status_code = None + + def __init__(self, exc): + self.text = f'{type(exc).__name__}: {exc}' + + def json(self): + raise AssertionError(f'there is no response body to parse: {self.text}') + + +class ParameterRequestCounter(Node): + """A node that serves the parameter services by hand and counts what it is asked for. + + Its own rclcpp/rclpy parameter services are DISABLED: their traffic is unobservable from + the outside, and counting is the entire point. The hand-rolled replacements answer on the + names rclpy would have used, so the transport's ``AsyncParametersClient`` finds them + exactly as it finds a normal node's. + + Every reply is well formed. A malformed one would send the transport down an early-return + path, and the test would then be counting failures instead of the cost of a successful + read - a green run that proves nothing. + """ + + def __init__(self, node_name, listed_extras=()): + super().__init__(node_name, start_parameter_services=False) + self._lock = threading.Lock() + self._counts = dict.fromkeys(SERVICE_NAMES, 0) + # Names the list reports beyond DECLARED_PARAMS. NO_VALUE_PARAM is omitted from the get + # reply entirely; NOT_SET_PARAM is answered with a PARAMETER_NOT_SET value. Those two + # replies take the transport down different paths and cost the node different amounts. + self._listed_extras = tuple(listed_extras) + fqn = self.get_fully_qualified_name() + self._services = [ + self.create_service(ListParameters, f'{fqn}/list_parameters', self._on_list), + self.create_service(GetParameters, f'{fqn}/get_parameters', self._on_get), + self.create_service( + DescribeParameters, f'{fqn}/describe_parameters', self._on_describe), + self.create_service( + GetParameterTypes, f'{fqn}/get_parameter_types', self._on_types), + self.create_service(SetParameters, f'{fqn}/set_parameters', self._on_set), + self.create_service( + SetParametersAtomically, + f'{fqn}/set_parameters_atomically', + self._on_set_atomically), + ] + + def snapshot(self): + """Per-service request counts as of now.""" + with self._lock: + return dict(self._counts) + + def _bump(self, service): + with self._lock: + self._counts[service] += 1 + + def _on_list(self, request, response): + self._bump('list_parameters') + names = sorted(list(DECLARED_PARAMS) + list(self._listed_extras)) + if request.prefixes: + # rcl's prefix rule: a name matches when it IS the prefix or lives under it. The + # transport's name-scoped probe passes the bare parameter name, so an undeclared + # name matches nothing and the reply is legitimately empty. + names = [ + name for name in names + if any(name == p or name.startswith(f'{p}.') for p in request.prefixes) + ] + response.result.names = names + return response + + def _on_get(self, request, response): + self._bump('get_parameters') + # NO_VALUE_PARAM is omitted from the reply rather than answered NOT_SET. The service + # contract does not require one value per requested name, and an omitted value is the only + # reply that reaches the transport's second NOT_FOUND branch - the one that costs two round + # trips and that kExpectNotFoundRoundTrips is sized for. + response.values = [ + _double(DECLARED_PARAMS[name]) if name in DECLARED_PARAMS + else ParameterValue(type=ParameterType.PARAMETER_NOT_SET) + for name in request.names + if name != NO_VALUE_PARAM + ] + return response + + def _on_describe(self, request, response): + self._bump('describe_parameters') + response.descriptors = [ + ParameterDescriptor(name=name, type=ParameterType.PARAMETER_DOUBLE) + for name in request.names + ] + return response + + def _on_types(self, request, response): + self._bump('get_parameter_types') + response.types = [ + ParameterType.PARAMETER_DOUBLE if name in DECLARED_PARAMS + else ParameterType.PARAMETER_NOT_SET + for name in request.names + ] + return response + + def _on_set(self, request, response): + self._bump('set_parameters') + response.results = [ + SetParametersResult(successful=False, reason='probe is read-only') + for _ in request.parameters + ] + return response + + def _on_set_atomically(self, request, response): + self._bump('set_parameters_atomically') + response.result = SetParametersResult(successful=False, reason='probe is read-only') + return response + + +def generate_test_description(): + detector_params = { + # The detector whose constants this test pins is switched OFF so that every request the + # probe counts is one this test asked for. Its background sweep would otherwise read the + # probe on its own schedule and no count would be attributable to a single call. `off` + # keeps the detector out of the plugin's tick list entirely (see + # graph_watchdog_plugin.cpp), so it never even builds a transport. + 'plugins.graph_watchdog.detectors.param_drift.mode': 'off', + } + gateway_params = { + # A generous per-call timeout keeps a slow first DDS discovery from turning a real + # SUCCESS path into a TIMEOUT. Disabling the negative cache means an early attempt made + # before the probe's services are discoverable cannot lock the node out for a minute + # (the default TTL): it just fails, costs the probe nothing, and test_02 retries while + # first contact is still ahead of it. + 'parameter_service_timeout_sec': 5.0, + 'parameter_service_negative_cache_sec': 0.0, + } + return create_watchdog_test_launch( + detector_params=detector_params, + extra_gateway_params=gateway_params, + demo_nodes=None, + port=PORT, + ) + + +class TestParameterRoundTripCost(unittest.TestCase): + """One transport call, counted at the node that serves it.""" + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._probe = ParameterRequestCounter(PROBE_NODE) + cls._pin_probe = ParameterRequestCounter( + PIN_PROBE_NODE, listed_extras=(NO_VALUE_PARAM, NOT_SET_PARAM)) + cls._executor = SingleThreadedExecutor() + cls._executor.add_node(cls._probe) + cls._executor.add_node(cls._pin_probe) + cls._stop = threading.Event() + cls._spin = threading.Thread(target=cls._spin_until_stopped, daemon=True) + cls._spin.start() + + @classmethod + def _spin_until_stopped(cls): + while not cls._stop.is_set() and rclpy.ok(): + cls._executor.spin_once(timeout_sec=0.1) + + @classmethod + def tearDownClass(cls): + cls._stop.set() + cls._spin.join(timeout=5.0) + cls._executor.remove_node(cls._probe) + cls._executor.remove_node(cls._pin_probe) + cls._executor.shutdown() + cls._probe.destroy_node() + cls._pin_probe.destroy_node() + rclpy.shutdown() + + # -- helpers ---------------------------------------------------------------------------- + + def _measure(self, url, probe=None): + """GET `url` and return the response together with what it cost `probe`. + + The probe's counters are bumped inside the service callback, which completes before + the reply reaches the transport and therefore long before the HTTP response returns. + The settle window after the response is not there to catch a laggard: it is there so + that a request arriving from anywhere ELSE lands inside the delta and breaks the + assertion, instead of silently shifting the next measurement's baseline. + """ + probe = probe if probe is not None else self._probe + before = probe.snapshot() + try: + response = requests.get(url, timeout=HTTP_TIMEOUT) + except requests.exceptions.RequestException as exc: + # A timed-out or dropped request is a legitimate outcome here, not a test error: see + # _RequestFailed. The delta is still taken, because a request that failed at the + # CLIENT may well have cost the probe something at the far end, and the retry loops + # check exactly that before deciding first contact is still ahead of them. + response = _RequestFailed(exc) + time.sleep(1.0) + after = probe.snapshot() + delta = {name: after[name] - before[name] for name in SERVICE_NAMES} + return response, delta + + @staticmethod + def _total(counts): + return sum(counts.values()) + + def _assert_round_trips(self, delta, expected, what): + self.assertEqual( + self._total(delta), expected, + f'{what} cost the node {self._total(delta)} parameter-service round trips, not ' + f'{expected}. Per service: {delta}. Either a round trip was added to or removed ' + f'from Ros2ParameterTransport, or the charge for this case in ' + f'param_drift_detector.cpp no longer matches what the node actually pays.') + + # -- tests ------------------------------------------------------------------------------ + + def test_01_the_probes_are_discovered_and_nothing_reads_them_unasked(self): + """Every later delta is only attributable if the probes are otherwise untouched.""" + wanted = {PROBE_NODE, PIN_PROBE_NODE} + # The retry deadlines in this file are chosen so that ALL of them together, plus the + # single-shot cases between them, expire well inside the ctest timeout. A test written to + # diagnose a slow-discovery path is worthless if ctest kills it before it can print. + deadline = time.monotonic() + 60.0 + seen = set() + while time.monotonic() < deadline: + try: + response = requests.get(f'{API_BASE}/apps', timeout=5) + if response.status_code == 200: + seen = {item.get('id') for item in response.json().get('items', [])} + if wanted <= seen: + break + except requests.exceptions.RequestException: + pass + time.sleep(0.5) + + self.assertTrue( + wanted <= seen, + f'the gateway never discovered {sorted(wanted - seen)} as apps, so no configurations ' + f'request can reach them. Apps seen: {sorted(seen)}') + + before = (self._probe.snapshot(), self._pin_probe.snapshot()) + time.sleep(3.0) + after = (self._probe.snapshot(), self._pin_probe.snapshot()) + self.assertEqual( + before, after, + f'something is reading parameters from a probe on its own: {before} -> {after}. ' + f'Every round-trip count below would then include traffic this test did not ask for.') + for name, counts in ((PROBE_NODE, after[0]), (PIN_PROBE_NODE, after[1])): + self.assertEqual( + self._total(counts), 0, + f'{name} had already served {self._total(counts)} parameter requests before this ' + f'test made any: {counts}. The first-contact measurements below need to be the ' + f'FIRST reads ever made against these nodes.') + + def test_02_a_cold_baseline_visit_costs_four_round_trips(self): + """First contact pays for priming the transport's defaults cache as well as the read.""" + url = f'{API_BASE}/apps/{PROBE_NODE}/configurations' + deadline = time.monotonic() + 45.0 + while True: + response, delta = self._measure(url) + if response.status_code == 200: + break + # A failure BEFORE the probe served anything is a transport that has not discovered + # the services yet: it fails inside wait_for_service, costs the node nothing, and + # the next attempt is still first contact. A failure that DID cost something has + # spent the cold visit, and the measurement below can no longer be made. + self.assertEqual( + self._total(delta), 0, + f'the first configurations read failed with HTTP {response.status_code} after ' + f'costing the probe {self._total(delta)} round trips ({delta}), so first ' + f'contact is gone and the cold cost can no longer be measured') + self.assertLess( + time.monotonic(), deadline, + f'GET {url} never succeeded; last status {response.status_code}: ' + f'{response.text[:400]}') + time.sleep(1.0) + + names = {item.get('name') for item in response.json().get('items', [])} + self.assertEqual( + names, set(DECLARED_PARAMS), + f'the read did not come back with the parameters the probe declares ({names}), so ' + f'the transport did not take its SUCCESS path and the count below is the price of a ' + f'failure, not of a read') + + self._assert_round_trips( + delta, 4, + 'a COLD baseline visit (first list_parameters against the node: the defaults-cache ' + 'priming list+get, then the second list+get the read itself makes)') + # Per service, not just the total. The four is two list+get PAIRS and nothing else; a + # change that swapped one of them for a describe or a get_parameter_types would leave the + # total at four and mean something quite different about what the node pays. + self.assertEqual( + (delta['list_parameters'], delta['get_parameters']), (2, 2), + f'a cold baseline visit is two list+get pairs - the defaults-cache priming one and ' + f'the read itself - and this was not: {delta}') + + def test_03_a_warm_baseline_visit_costs_two_round_trips(self): + """The defaults cache has no TTL, so the cold pair is paid once per node, not per read.""" + url = f'{API_BASE}/apps/{PROBE_NODE}/configurations' + response, delta = self._measure(url) + self.assertEqual(response.status_code, 200, response.text[:400]) + names = {item.get('name') for item in response.json().get('items', [])} + self.assertEqual(names, set(DECLARED_PARAMS), response.text[:400]) + + self._assert_round_trips( + delta, 2, + 'a WARM baseline visit (kBaselineReadRoundTrips: one list, then one batched get of ' + 'every name it returned)') + self.assertEqual( + (delta['list_parameters'], delta['get_parameters']), (1, 1), + f'a warm baseline visit is exactly one list and one batched get, and this was not - a ' + f'total of two says nothing about WHICH two: {delta}') + + def test_04_a_resolved_expect_pin_costs_three_round_trips(self): + """describe_parameters is unconditional on the success path, so the pin pays for it.""" + pin = 'probe_gain' + response, delta = self._measure(f'{API_BASE}/apps/{PROBE_NODE}/configurations/{pin}') + self.assertEqual(response.status_code, 200, response.text[:400]) + self.assertEqual( + response.json().get('data'), DECLARED_PARAMS[pin], + f'the pinned read did not return the value the probe serves, so the transport did ' + f'not take its SUCCESS path: {response.text[:400]}') + + self._assert_round_trips( + delta, 3, + 'a RESOLVED expect pin (kExpectReadRoundTrips: name-scoped list, get, describe)') + self.assertEqual( + delta['describe_parameters'], 1, + f'the descriptor round trip is what makes a resolved pin cost three rather than ' + f'two, and it did not happen: {delta}') + + def test_05_an_unresolved_expect_pin_costs_one_round_trip(self): + """Charged 2 by the detector on purpose; the real cost is 1 and is allowed to be lower. + + The transport answers NOT_FOUND from the name-scoped list alone and never reaches the + get or the describe. ``kExpectNotFoundRoundTrips`` is 2 because a SECOND NOT_FOUND path + - a parameter that is declared but unset - does cost two, and the error code cannot + distinguish them, so the constant takes the higher of the two. Overstating a load bound + is safe; understating it is not. This asserts the real cost, not the charge. + """ + response, delta = self._measure( + f'{API_BASE}/apps/{PROBE_NODE}/configurations/{UNDECLARED_PARAM}') + self.assertEqual( + response.status_code, 404, + f'a parameter the probe never declares must come back 404, not ' + f'{response.status_code}: {response.text[:400]}') + + self._assert_round_trips( + delta, 1, + 'an UNRESOLVED expect pin (the name-scoped list returns no names and the transport ' + 'stops there)') + self.assertEqual( + delta['get_parameters'], 0, + f'the transport read a value for a parameter its own name-scoped list said the node ' + f'does not have: {delta}') + + def test_06_a_pinned_read_has_no_cold_contact_surcharge(self): + """The cold pair belongs to list_parameters, not to a read in general. + + ``get_parameter`` never calls ``cache_default_values``, so an ``expect``-only sweep + (``baseline: false``) never primes that cache and never pays the extra pair - the first + pinned read of a node costs the same 3 as every one after it. The second probe exists + purely to have a node whose defaults cache is still unprimed at this point; the first + one stopped being one the moment test_02 listed it. + """ + pin = 'probe_gain' + before = self._pin_probe.snapshot() + self.assertEqual( + self._total(before), 0, + f'{PIN_PROBE_NODE} has already been read ({before}), so this is no longer a ' + f'first-contact measurement') + + url = f'{API_BASE}/apps/{PIN_PROBE_NODE}/configurations/{pin}' + deadline = time.monotonic() + 45.0 + while True: + response, delta = self._measure(url, probe=self._pin_probe) + if response.status_code == 200: + break + self.assertEqual( + self._total(delta), 0, + f'the first pinned read of {PIN_PROBE_NODE} failed with HTTP ' + f'{response.status_code} after costing it {self._total(delta)} round trips ' + f'({delta}), so first contact is gone and the cost can no longer be measured') + self.assertLess( + time.monotonic(), deadline, + f'GET {url} never succeeded; last status {response.status_code}: ' + f'{response.text[:400]}') + time.sleep(1.0) + + self.assertEqual( + response.json().get('data'), DECLARED_PARAMS[pin], + f'the pinned read did not return the value the probe serves, so the transport did ' + f'not take its SUCCESS path: {response.text[:400]}') + self._assert_round_trips( + delta, 3, + 'the FIRST expect pin read of a node (same as any later one: get_parameter does not ' + 'prime the defaults cache, so there is no cold-contact pair in expect-only mode)') + + def test_07_a_listed_name_with_no_value_costs_two_round_trips(self): + """The rarer NOT_FOUND path, and the reason the charge for NOT_FOUND is 2 and not 1. + + test_05 measures the COMMON NOT_FOUND: the name-scoped list comes back empty and the + transport stops there, for one round trip. This is the other one. The node lists the name, + so the transport goes on to the get, and the get comes back with no value for it - two + round trips, and the same NOT_FOUND error code as the one-round-trip case. + ``kExpectNotFoundRoundTrips`` is 2 because that is the higher of the two and the transport + cannot tell a caller which it took: a load bound may overstate a cost, never understate it. + + Without this case the charge was anchored from one side only. test_05 says the charge may + legitimately exceed the real cost, which on its own is an argument for any number at all; + it takes this measurement to say why the number is exactly two. + """ + response, delta = self._measure( + f'{API_BASE}/apps/{PIN_PROBE_NODE}/configurations/{NO_VALUE_PARAM}', + probe=self._pin_probe) + self.assertEqual( + response.status_code, 404, + f'a parameter the probe lists but returns no value for must come back 404, not ' + f'{response.status_code}: {response.text[:400]}') + + self._assert_round_trips( + delta, 2, + 'a LISTED name the get returns no value for (the name-scoped list finds the name, the ' + 'get finds no value, and the transport returns NOT_FOUND before the describe)') + self.assertEqual( + (delta['list_parameters'], delta['get_parameters'], delta['describe_parameters']), + (1, 1, 0), + f'the two round trips are the list and the get, and the describe must not happen - a ' + f'NOT_FOUND that reached the descriptor read would cost three, which is MORE than ' + f'kExpectNotFoundRoundTrips charges and would make the bound an understatement: ' + f'{delta}') + + def test_08_a_declared_but_unset_pin_is_a_success_costing_three(self): + """A declared-but-unset parameter is NOT the two-round-trip NOT_FOUND path. + + This is worth its own case because it is what that path used to be documented as. A + parameter a node has declared and not set is answered by rclcpp's parameter service with a + ``PARAMETER_NOT_SET`` VALUE, not with an absent one, so the transport gets a parameter + back, goes on to the unconditional describe, and returns SUCCESS carrying null. It costs + the node three round trips, the same as any other resolved pin, and ``expect`` charges it + three - which is right. Had the charge of 2 really been sized for this case it would have + been an UNDERSTATEMENT of what the node pays, and the whole point of that constant is that + it may overstate and must never understate. + """ + response, delta = self._measure( + f'{API_BASE}/apps/{PIN_PROBE_NODE}/configurations/{NOT_SET_PARAM}', + probe=self._pin_probe) + self.assertEqual( + response.status_code, 200, + f'a parameter the node declares but has not set is a value the node HAS (an unset ' + f'one), so the read succeeds: {response.status_code}, {response.text[:400]}') + self.assertIsNone( + response.json().get('data'), + f'the unset parameter came back carrying a value: {response.text[:400]}') + + self._assert_round_trips( + delta, 3, + 'a DECLARED BUT UNSET pin (list, get, describe - the same success path as any other ' + 'resolved pin, which is why expect charges it kExpectReadRoundTrips and not ' + 'kExpectNotFoundRoundTrips)') + self.assertEqual( + delta['describe_parameters'], 1, + f'the descriptor round trip is what makes this a three rather than a two, and it did ' + f'not happen: {delta}') + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + """Verify the gateway/fault_manager stack exits cleanly.""" + + def test_exit_codes(self, proc_info): + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'Process {info.process_name} exited with {info.returncode}', + ) diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_integration.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_integration.cpp new file mode 100644 index 000000000..ea89e1f02 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_integration.cpp @@ -0,0 +1,3693 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Drives the REAL mechanism: a target node exposes parameters over DDS, the detector +// reads them via Ros2ParameterTransport, and raises/clears through a real ReportFault +// service round-trip to a fake fault_manager. NOT a full-gateway e2e; this proves the +// detector end to end within its own scope. +// +// The detector is snapshot-driven: tick() early-returns unless ctx.snapshot is set, iterates +// ctx.snapshot->apps (id + fqn), and raises ONE aggregated fault whose source_id is always +// graph_source_id() - the constant "graph_watchdog", the entity the plugin owns. It never +// depends on what the snapshot contains. So every test seeds the snapshot via set_apps() and +// asserts on the "graph_watchdog" source, never a per-node FQN and never a component id. +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ros2_medkit_gateway/core/providers/introspection_provider.hpp" +#include "ros2_medkit_gateway/core/transports/parameter_transport.hpp" +#include "ros2_medkit_graph_watchdog/aggregated_fault.hpp" +#include "ros2_medkit_graph_watchdog/detector.hpp" +#include "ros2_medkit_graph_watchdog/detector_registry.hpp" +#include "ros2_medkit_graph_watchdog/reliability_gate.hpp" + +using ros2_medkit_gateway::App; +using ros2_medkit_gateway::Component; +using ros2_medkit_gateway::IntrospectionInput; +using ros2_medkit_graph_watchdog::AggregatedFault; +using ros2_medkit_graph_watchdog::DetectorContext; +using ros2_medkit_graph_watchdog::DetectorMode; +using ros2_medkit_graph_watchdog::reliability_allows; +using ros2_medkit_graph_watchdog::ReliabilityGate; +using ReportFault = ros2_medkit_msgs::srv::ReportFault; +using namespace std::chrono_literals; + +namespace { +// The detector class is file-local in param_drift_detector.cpp but self-registers via +// REGISTER_DETECTOR, which runs when that .cpp is linked into this test. Pull an +// instance from the registry (no production factory needed). +std::unique_ptr make_param_drift() { + for (auto & d : ros2_medkit_graph_watchdog::DetectorRegistry::instance().create_all()) { + if (d->id() == "param_drift") { + return std::move(d); + } + } + return nullptr; +} + +/// Runs `fn` when it leaves scope, on EVERY exit path - including the one a failed ASSERT takes. +/// +/// The teardown this file needs is not optional cleanup. A node removed from the fixture's +/// executor only on the success path is still registered with a RUNNING MultiThreadedExecutor when +/// an ASSERT above destroys it, and the process SIGABRTs on a dangling callback group: a readable +/// assertion failure turns into a crash that says nothing about what went wrong. The same applies +/// to a joinable std::thread, where the failure mode is std::terminate. +template +class ScopeExit { + public: + explicit ScopeExit(FnT fn) : fn_(std::move(fn)) { + } + ~ScopeExit() { + // A destructor must not throw: an escape here during stack unwinding from a failed assertion + // is std::terminate, which is strictly worse than the leak this guard exists to prevent. + try { + fn_(); + } catch (const std::exception &) { + // Swallowed on purpose, per the note above. + } + } + ScopeExit(const ScopeExit &) = delete; + ScopeExit & operator=(const ScopeExit &) = delete; + ScopeExit(ScopeExit &&) = delete; + ScopeExit & operator=(ScopeExit &&) = delete; + + private: + FnT fn_; +}; + +template +ScopeExit on_scope_exit(FnT fn) { + return ScopeExit(std::move(fn)); +} + +/// Captures rcutils log output for as long as it is alive and restores the console handler on every +/// exit path. Leaving the process-global handler installed would swallow the log output of every +/// later case in this binary, so the restore cannot be left to the success path. +/// +/// One implementation for the whole file. The rcutils handler CONTRACT hands over a caller-supplied +/// format string and a va_list - it is not a choice the handler makes - so `vsnprintf` here is the +/// one place in this file that cannot use a literal format. Written out per test, that was six +/// copies of the suppression and six guard types; written once, it is one. +/// +/// The handler is a plain C function pointer with no user-data slot, so the live capture is reached +/// through a file-static. It is written by the test thread and read by whichever thread logs (the +/// detector's reader thread does), hence the atomic. +class LogCapture { + public: + LogCapture() { + active().store(this); + rcutils_logging_set_output_handler(&LogCapture::handler); + } + ~LogCapture() { + rcutils_logging_set_output_handler(rcutils_logging_console_output_handler); + active().store(nullptr); + } + LogCapture(const LogCapture &) = delete; + LogCapture & operator=(const LogCapture &) = delete; + LogCapture(LogCapture &&) = delete; + LogCapture & operator=(LogCapture &&) = delete; + + /// How many captured lines contain `needle`. + int count(const std::string & needle) const { + std::lock_guard lk(mutex_); + return static_cast(std::count_if(lines_.begin(), lines_.end(), [&needle](const std::string & line) { + return line.find(needle) != std::string::npos; + })); + } + + private: + static std::atomic & active() { + static std::atomic current{nullptr}; + return current; + } + + static void handler(const rcutils_log_location_t * /*location*/, int /*severity*/, const char * /*name*/, + rcutils_time_point_value_t /*timestamp*/, const char * format, va_list * args) { + char buf[1024]; + va_list copy; + va_copy(copy, *args); + // The format string arrives from the logging call site through the handler signature, so there + // is no literal to write here and no other way to reach the arguments than the va_list. The + // build runs -Werror=format=2; GCC exempts va_list-taking formatters from -Wformat-nonliteral, + // clang does not, so the CI clang-tidy job fails on this line alone. A pragma rather than a + // lint-suppression comment, because the diagnostic is raised as an ERROR and clang-tidy does + // not honour suppression comments for those. Scoped to the single call. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" + vsnprintf(buf, sizeof(buf), format, copy); +#pragma GCC diagnostic pop + va_end(copy); + LogCapture * capture = active().load(); + if (capture == nullptr) { + return; + } + std::lock_guard lk(capture->mutex_); + capture->lines_.emplace_back(buf); + } + + mutable std::mutex mutex_; + std::vector lines_; +}; + +/// Adapts a body that wants the request and the response by reference to the shape rclcpp requires. +/// +/// create_service deduces the callback by comparing its argument tuple against +/// `std::function, std::shared_ptr)>` EXACTLY, so a handler +/// cannot declare those parameters by reference however little it needs a copy of them. Stating +/// that shape once here keeps it out of every fake service below. +template +std::function, std::shared_ptr)> +service_callback(BodyT body) { + return [body = std::move(body)](std::shared_ptr req, + std::shared_ptr resp) { + body(*req, *resp); + }; +} + +/// The four parameter services a hand-rolled probe node needs beyond list/get, purely so the +/// transport's `AsyncParametersClient` reports ready. +/// +/// wait_for_service() on that client waits for ALL SIX; a node missing any of them is answered +/// SERVICE_UNAVAILABLE and negative-cached before a single request reaches its handlers. A test +/// about what a READ costs or how long one may take is then measuring a node that was never read at +/// all - which is exactly how a first attempt at the per-read-bound case below passed for the wrong +/// reason. These four only ever have to exist; nothing in this file drives them. +std::vector add_idle_parameter_services(const rclcpp::Node::SharedPtr & node, + const std::string & fqn) { + return { + node->create_service( + fqn + "/describe_parameters", service_callback( + [](const rcl_interfaces::srv::DescribeParameters::Request & req, + rcl_interfaces::srv::DescribeParameters::Response & resp) { + for (const auto & name : req.names) { + rcl_interfaces::msg::ParameterDescriptor d; + d.name = name; + d.type = rcl_interfaces::msg::ParameterType::PARAMETER_DOUBLE; + resp.descriptors.push_back(d); + } + })), + node->create_service( + fqn + "/get_parameter_types", service_callback( + [](const rcl_interfaces::srv::GetParameterTypes::Request & req, + rcl_interfaces::srv::GetParameterTypes::Response & resp) { + resp.types.assign(req.names.size(), + rcl_interfaces::msg::ParameterType::PARAMETER_DOUBLE); + })), + node->create_service( + fqn + "/set_parameters", service_callback( + [](const rcl_interfaces::srv::SetParameters::Request & req, + rcl_interfaces::srv::SetParameters::Response & resp) { + for (std::size_t i = 0; i < req.parameters.size(); ++i) { + rcl_interfaces::msg::SetParametersResult r; + r.successful = false; + r.reason = "probe is read-only"; + resp.results.push_back(r); + } + })), + node->create_service( + fqn + "/set_parameters_atomically", + service_callback( + [](const rcl_interfaces::srv::SetParametersAtomically::Request & /*req*/, + rcl_interfaces::srv::SetParametersAtomically::Response & resp) { + resp.result.successful = false; + resp.result.reason = "probe is read-only"; + })), + }; +} +} // namespace + +/// A parameter transport under the test's control. The detector's reads are blocking service +/// round-trips against a live node, which makes three behaviours unreachable from a real ROS graph: +/// a read that is still in flight when its app is de-armed, a SUCCESS response that carries no +/// parameters at all, and a transport that cannot be constructed. Each of those guards a real +/// failure mode, so each needs a stand-in to be provable. +class ScriptedTransport : public ros2_medkit_gateway::ParameterTransport { + public: + bool is_self_node(const std::string & /*node*/) const override { + return false; + } + ros2_medkit_gateway::ParameterResult list_parameters(const std::string & node) override { + throw_if_scripted(node); + if (is_unreadable(node)) { + ros2_medkit_gateway::ParameterResult r; + r.success = false; // a node with no working parameter service - no live node can be made to + r.error_code = ros2_medkit_gateway::ParameterErrorCode::SERVICE_UNAVAILABLE; + return r; // answer this way on demand, which is why the stand-in has to + } + enter_read(node); + ros2_medkit_gateway::ParameterResult r; + r.success = true; + { + // Under `m_`, all of it. enter_read() releases the lock when it returns, so reading + // empty_batch_ out here is a plain bool racing set_empty_batch() on the test thread against + // the reader thread - which TSan fails the whole integration test for, and which is a real + // race and not a test artefact: the stand-in stands in for a transport the reader calls + // concurrently with the test reconfiguring it. + std::lock_guard lk(m_); + if (empty_batch_) { + r.data = nlohmann::json::array(); // SUCCESS carrying no view - a legitimate transport reply + return r; + } + const auto it = node_params_.find(node); + if (it != node_params_.end()) { + r.data = nlohmann::json::array(); + for (const auto & [name, value] : it->second) { + r.data.push_back(nlohmann::json{{"name", name}, {"value", value}, {"type", "double"}}); + } + return r; + } + } + r.data = nlohmann::json::array({{{"name", "gain"}, {"value", value_for(node)}, {"type", "double"}}}); + return r; + } + ros2_medkit_gateway::ParameterResult get_parameter(const std::string & node, const std::string & name) override { + throw_if_scripted(node); + if (is_unreadable(node)) { + ros2_medkit_gateway::ParameterResult r; + r.success = false; + r.error_code = ros2_medkit_gateway::ParameterErrorCode::SERVICE_UNAVAILABLE; + return r; + } + enter_read(node); + { + std::lock_guard lk(m_); + if (restrict_declared_) { + const auto per_node = node_declared_.find(node); + const auto & names = per_node != node_declared_.end() ? per_node->second : declared_; + if (names.count(name) == 0) { + ros2_medkit_gateway::ParameterResult r; + r.success = false; + r.error_code = ros2_medkit_gateway::ParameterErrorCode::NOT_FOUND; + return r; + } + } + } + ros2_medkit_gateway::ParameterResult r; + r.success = true; + r.data = nlohmann::json{{"name", name}, {"value", value_for(node)}, {"type", "double"}}; + return r; + } + ros2_medkit_gateway::ParameterResult set_parameter(const std::string & /*node*/, const std::string & /*name*/, + const nlohmann::json & /*value*/) override { + return {}; + } + ros2_medkit_gateway::ParameterResult list_own_parameters() override { + return {}; + } + ros2_medkit_gateway::ParameterResult get_own_parameter(const std::string & /*name*/) override { + return {}; + } + ros2_medkit_gateway::ParameterResult get_default(const std::string & /*node*/, + const std::string & /*name*/) override { + return {}; + } + ros2_medkit_gateway::ParameterResult list_defaults(const std::string & /*node*/) override { + return {}; + } + bool is_node_available(const std::string & /*node*/) const override { + return true; + } + void shutdown() override { + std::lock_guard lk(m_); + blocked_ = false; // never let a teardown wait on a gate the test forgot to open + blocked_nodes_.clear(); + gate_.notify_all(); + } + + void block() { + std::lock_guard lk(m_); + blocked_ = true; + } + void release() { + std::lock_guard lk(m_); + blocked_ = false; + gate_.notify_all(); + } + /// Hold reads of ONE node. The reader thread is single-threaded, so blocking everything stops + /// the whole loop: a test that needs to observe the reader moving past a held read has to leave + /// the other targets running. + void block_node(const std::string & node) { + std::lock_guard lk(m_); + blocked_nodes_.insert(node); + } + void release_node(const std::string & node) { + std::lock_guard lk(m_); + blocked_nodes_.erase(node); + gate_.notify_all(); + } + /// Wait until the reader is actually inside a read, so the test never races the thread. + bool wait_until_reading(int at_least) { + std::unique_lock lk(m_); + return entered_.wait_for(lk, 5s, [&] { + return calls_ >= at_least; + }); + } + /// Wait until a read is actually HELD on the gate. The call counter cannot express this: a read + /// that entered just before block_node() is counted but runs to completion, so a test keying off + /// the count can proceed while nothing is held at all. + bool wait_until_parked() { + std::unique_lock lk(m_); + return parked_cv_.wait_for(lk, 5s, [this] { + return parked_ > 0; + }); + } + /// Default value for every node. + void set_value(double v) { + value_.store(v); + } + /// Override the value of ONE node. Needed whenever a test drifts a single app: the shared value + /// would drift every target at once, and the assertion would then pass on the wrong app. + void set_value(const std::string & node, double v) { + std::lock_guard lk(m_); + node_values_[node] = v; + } + /// Replace the whole parameter set ONE node answers `list_parameters` with. Left unset, the + /// stand-in reports a single `gain`, which is enough to drift a node but nowhere near enough to + /// fill the detector's per-app share of the aggregated description - so a test about which + /// entries survive that cap cannot be written with it at all. Per-node for the same reason + /// set_value and set_declared are: one shared set makes every node answer alike, and the point + /// here is that different nodes contribute different amounts of text. + void set_params(const std::string & node, std::map params) { + std::lock_guard lk(m_); + node_params_[node] = std::move(params); + } + /// Make one node permanently unreadable, whatever else the stand-in does. + void set_unreadable(const std::string & node) { + std::lock_guard lk(m_); + unreadable_.insert(node); + } + /// Undo set_unreadable. A test that proves an unreadable node cannot CLEAR a fault needs the + /// positive control that a readable one still can, or the silence it asserts is satisfied by a + /// detector that stopped emitting anything at all. + void set_readable(const std::string & node) { + std::lock_guard lk(m_); + unreadable_.erase(node); + } + /// Make one node's reads THROW. rclcpp can throw out of a read when the context is invalidated + /// mid-call, or on a malformed service path; a live node cannot be asked to do it on demand. + void set_throwing(const std::string & node) { + std::lock_guard lk(m_); + throwing_.insert(node); + } + bool wait_until_threw(int at_least) { + std::unique_lock lk(m_); + return threw_.wait_for(lk, 5s, [&] { + return throws_ >= at_least; + }); + } + /// How many reads have been REJECTED because their node is unreadable. `calls()` cannot express + /// this: an unreadable node fails before enter_read, so it is never counted there. A test whose + /// subject is a THRESHOLD on failed attempts has to pace itself off the attempts themselves - + /// keying off wall clock instead makes the assertion a function of the pacing budget, which is + /// the very coupling this counter exists to let a test pin. + int failures() { + std::lock_guard lk(m_); + return failures_; + } + bool wait_until_failed(int at_least) { + std::unique_lock lk(m_); + return failed_.wait_for(lk, 30s, [&] { + return failures_ >= at_least; + }); + } + void set_empty_batch(bool on) { + std::lock_guard lk(m_); + empty_batch_ = on; + } + /// Restrict which parameter names the node admits to having. Left unset, the stand-in answers + /// any name, which is precisely the behaviour no real node has: a real one returns NOT_FOUND for + /// anything it never declared, and that reply is deliberately not treated as a drift. + void set_declared(std::set names) { + std::lock_guard lk(m_); + declared_ = std::move(names); + restrict_declared_ = true; + } + /// Restrict which names ONE node admits to having, overriding the shared set for that node. + /// Same reason set_value has a per-node form: with a single shared set every node answers alike, + /// so a test about one node declaring a pin that the rest of the graph does not cannot be + /// written at all - and "the sweep has not reached the node that declares it yet" is exactly the + /// state the unmatched-pin warning must not fire in. + void set_declared(const std::string & node, std::set names) { + std::lock_guard lk(m_); + node_declared_[node] = std::move(names); + restrict_declared_ = true; + } + int calls() { + std::lock_guard lk(m_); + return calls_; + } + + private: + std::mutex m_; + std::condition_variable gate_; + std::condition_variable entered_; + std::condition_variable parked_cv_; + std::condition_variable threw_; + std::condition_variable failed_; + bool blocked_ = false; + bool empty_batch_ = false; + std::set unreadable_; + std::set blocked_nodes_; + std::set throwing_; + std::set declared_; + std::map> node_declared_; + bool restrict_declared_ = false; + std::map node_values_; + std::map> node_params_; + int calls_ = 0; + int parked_ = 0; + int throws_ = 0; + int failures_ = 0; + std::atomic value_{1.0}; + + bool is_unreadable(const std::string & node) { + std::lock_guard lk(m_); + if (unreadable_.count(node) == 0) { + return false; + } + ++failures_; + failed_.notify_all(); + return true; + } + + /// Throw out of the read itself, before anything is counted as a call. The throw has to leave + /// the transport the way a real one would - from inside the blocking call, not around it. + void throw_if_scripted(const std::string & node) { + { + std::lock_guard lk(m_); + if (throwing_.count(node) == 0) { + return; + } + ++throws_; + threw_.notify_all(); + } + throw std::runtime_error("scripted transport failure"); + } + + /// Count the call, then hold it for as long as the test blocks this node. `parked_` is + /// incremented only on the path that actually waits, so wait_until_parked() means a read is + /// held right now rather than merely started. + void enter_read(const std::string & node) { + std::unique_lock lk(m_); + ++calls_; + entered_.notify_all(); + const auto held = [this, &node] { + return blocked_ || blocked_nodes_.count(node) != 0; + }; + if (!held()) { + return; + } + ++parked_; + parked_cv_.notify_all(); + gate_.wait(lk, [&held] { + return !held(); + }); + --parked_; + } + + /// Read AFTER the gate, so a value the test sets while a read is held is the one that read + /// returns - which is how a stale in-flight sample is scripted. + double value_for(const std::string & node) { + std::lock_guard lk(m_); + const auto it = node_values_.find(node); + return it != node_values_.end() ? it->second : value_.load(); + } +}; + +class ParamDriftIntegrationTest : public ::testing::Test { + protected: + // Must run before the FIRST test's fixture is constructed: `exec_` is a fixture data + // member, so its default constructor (which touches the global rclcpp context to + // create an interrupt guard condition) runs before SetUp(). Doing rclcpp::init() only + // in SetUp() is too late for that first construction; SetUpTestSuite() runs earlier. + static void SetUpTestSuite() { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + } + + void SetUp() override { + gateway_ = std::make_shared("pd_it_gateway"); + target_ = std::make_shared("pd_it_target"); + target_->declare_parameter("speed_limit", 1.0); + sink_ = std::make_shared("pd_it_sink"); + srv_ = sink_->create_service( + "/fault_manager/report_fault", + service_callback([this](const ReportFault::Request & req, ReportFault::Response & resp) { + { + std::lock_guard lk(mtx_); + received_.push_back(req); + } + resp.accepted = true; // ReportFault.srv response field is `bool accepted` + })); + client_ = gateway_->create_client("/fault_manager/report_fault"); + exec_.add_node(gateway_); + exec_.add_node(target_); + exec_.add_node(sink_); + spin_ = std::thread([this]() { + exec_.spin(); + }); + ASSERT_TRUE(client_->wait_for_service(5s)); + } + void TearDown() override { + exec_.cancel(); + if (spin_.joinable()) { + spin_.join(); + } + exec_.remove_node(gateway_); + exec_.remove_node(target_); + exec_.remove_node(sink_); + client_.reset(); + srv_.reset(); + sink_.reset(); + target_.reset(); + gateway_.reset(); + } + + DetectorContext make_ctx(DetectorMode mode) { + DetectorContext ctx; + ctx.gateway_node = gateway_.get(); + ctx.node_mutex = &node_mutex_; + ctx.mode = mode; + ctx.gate = nullptr; // ungated by default; the gate-keying test wires a real gate in + ctx.fault_client = client_; + ctx.snapshot = &snapshot_; // tick() early-returns without a snapshot; each test seeds set_apps() + return ctx; + } + + // (Re)populate the owned entity snapshot the detector reads each tick. Uses the + // `App a; a.field = ...;` idiom (App has a RosBinding sub-struct, so designated/partial + // init trips -Wmissing-field-initializers under our strict flags). + // + // The aggregated fault's source_id does NOT depend on this snapshot: it is always + // graph_source_id(), the entity the plugin owns. SourceIdIgnoresTheHostComponent below seeds + // a component precisely to prove that, because a host Component id would match no entity + // scope and the fault would be reachable from no endpoint at all. + void set_apps(const std::vector> & id_fqn) { + snapshot_.apps.clear(); + for (const auto & [id, fqn] : id_fqn) { + App a; + a.id = id; + a.bound_fqn = fqn; + snapshot_.apps.push_back(a); + } + } + + // Wait until 's parameter service is discoverable, so the detector's first + // read succeeds and captures the baseline BEFORE any set_parameter. Without this, an + // early negative-cache miss (service not yet discovered) could make the first + // successful read land after the drift and capture the drifted value as the baseline. + bool wait_param_service(const std::string & node_fqn) { + std::lock_guard lk(node_mutex_); + auto c = gateway_->create_client(node_fqn + "/get_parameters"); + return c->wait_for_service(5s); + } + + // How many matching faults are currently in the log (for absence checks + snapshots). + std::size_t count_faults(const std::string & source_id, uint8_t event_type) { + std::lock_guard lk(mtx_); + std::size_t n = 0; + for (const auto & r : received_) { + if (r.source_id == source_id && r.fault_code == "GRAPH_PARAM_DRIFT" && r.event_type == event_type) { + ++n; + } + } + return n; + } + + // The description of the MOST RECENT FAILED report. That is what the fault store keeps: it + // overwrites the description on every report, so it is the only text an operator can read. A + // helper that scans the whole history can pass on a single transient report and hide the fact + // that the current record no longer names the entry. + std::string latest_failed_desc(const std::string & source_id) { + std::lock_guard lk(mtx_); + for (auto it = received_.rbegin(); it != received_.rend(); ++it) { + if (it->source_id == source_id && it->fault_code == "GRAPH_PARAM_DRIFT" && + it->event_type == ReportFault::Request::EVENT_FAILED) { + return it->description; + } + } + return {}; + } + + // The description of the FIRST failed report that mentions `needle`, or empty if there is none. + // + // Deliberately NOT the latest one. Any ordering rule keyed on an app being newly affected only + // decides the single report in which that app first appears: by the next tick the app has been + // reported, moves into the "already reported" bucket, and every ordering agrees again. Asserting + // on the latest record therefore passes under a wrong bucket order as soon as one more tick has + // gone by, which is what let the pinned-versus-self-captured cross combination go unexercised. + std::string first_failed_desc_containing(const std::string & source_id, const std::string & needle) { + std::lock_guard lk(mtx_); + for (const auto & r : received_) { + if (r.source_id == source_id && r.fault_code == "GRAPH_PARAM_DRIFT" && + r.event_type == ReportFault::Request::EVENT_FAILED && r.description.find(needle) != std::string::npos) { + return r.description; + } + } + return {}; + } + + // True if the CURRENT stored description names EVERY needle - used to prove one aggregated fault + // names all the drifting apps. Deliberately the latest record and not the history: the store + // overwrites the description on every report, so a history scan can pass on one transient report + // while the text an operator actually reads no longer names the entry. + bool latest_failed_desc_contains(const std::string & source_id, const std::vector & needles) { + const std::string desc = latest_failed_desc(source_id); + return !desc.empty() && std::all_of(needles.begin(), needles.end(), [&desc](const std::string & needle) { + return desc.find(needle) != std::string::npos; + }); + } + + // Tick until a NEW matching fault appears beyond `baseline_count` (i.e. arrived AFTER the + // trigger), or timeout. Requiring the fault to be new is what makes this a real proof: a + // detector whose comparison is gutted but still emits a spurious raise+clear during + // warm-up cannot satisfy it, because those land before the snapshot. + // + // The loop bound is 130 iterations (13s at 100ms) so the poll window EXCEEDS the + // transport's kNegativeCacheTtlSec (10s): the detector owns its transport (built from the + // registry) and the fixture cannot inject a shorter TTL, so a single timed-out read under + // ASan/TSan that negative-caches the target for 10s must not outlast the window. The happy + // path returns on first success, so responsive-node tests stay fast. + bool poll_for_new(const std::string & source_id, uint8_t event_type, std::size_t baseline_count, + ros2_medkit_graph_watchdog::Detector & det, DetectorContext & ctx) { + for (int i = 0; i < 130; ++i) { + det.tick(ctx); + std::this_thread::sleep_for(100ms); + if (count_faults(source_id, event_type) > baseline_count) { + return true; + } + } + return false; + } + + // Tick up to `iters` times, returning true as soon as `pred` holds. For non-count + // assertions (e.g. an aggregated description mentioning multiple apps). + template + bool poll_until(Pred pred, ros2_medkit_graph_watchdog::Detector & det, DetectorContext & ctx, int iters = 130) { + for (int i = 0; i < iters; ++i) { + det.tick(ctx); + std::this_thread::sleep_for(100ms); + if (pred()) { + return true; + } + } + return false; + } + + // EVIDENCE that every armed app has actually been read, to be taken before a test perturbs + // anything. Ticking a fixed number of times and hoping is not the same claim. + // + // The detector withholds its clear while any armed app is still unread (see emit_aggregated), so + // a PASSED event is not merely "nothing drifts" - it is the detector stating that it looked + // everywhere and found nothing. wait_param_service() cannot give that: it waits on the GATEWAY + // node's own client and says nothing about the transport's hidden one. Without this gate a first + // read that misses its window negative-caches the node for ten seconds, the first successful read + // lands after the parameter was changed, and the DRIFTED value silently becomes the baseline. + // + // Requires a NEW clear rather than any clear: a gate-denied app arms nothing, so a detector with + // an empty armed set clears on every tick, and "a clear exists" would then be satisfied before + // the app under test had been looked at even once. + bool wait_for_baseline_evidence(ros2_medkit_graph_watchdog::Detector & det, DetectorContext & ctx) { + const auto before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + return poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, before, det, ctx); + } + + rclcpp::Node::SharedPtr gateway_, target_, sink_; + rclcpp::Service::SharedPtr srv_; + rclcpp::Client::SharedPtr client_; + rclcpp::executors::MultiThreadedExecutor exec_; + std::thread spin_; + std::mutex mtx_, node_mutex_; + std::vector received_; + IntrospectionInput snapshot_; // owned entity snapshot the detector reads each tick +}; + +TEST_F(ParamDriftIntegrationTest, RuntimeSetRaisesThenRestoreClears) { + set_apps({{"pd_it_target", "/pd_it_target"}}); + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // defaults: baseline true, no expect + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_target")); // guarantee the baseline read succeeds first + // Evidence, not a fixed sleep: the clear only arrives once the target has actually been read. + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear ever arrived, so the target was never read and the change below would be " + "measured against no baseline at all"; + // Baseline capture alone MUST be silent (kills an "always-raise" detector). + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); + + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + target_->set_parameter(rclcpp::Parameter("speed_limit", 0.2)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + target_->set_parameter(rclcpp::Parameter("speed_limit", 1.0)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)); +} + +// The fault must be raised under the entity the plugin OWNS, never under the host Component, +// even when the snapshot carries one - which in a real gateway it always does. A host Component +// built by HostInfoProvider never sets `external`, and a non-external component does not claim +// its own bare id in a fault scope set, so a fault raised under it is listed by no entity +// endpoint at all: not /apps/graph_watchdog/faults, not /components//faults. It would show +// up only in the flat /faults list, which is also all the e2e polls - so nothing else here can +// catch this. Every other test in this file leaves `components` empty, which is exactly why the +// regression it guards went unnoticed. +TEST_F(ParamDriftIntegrationTest, SourceIdIgnoresTheHostComponent) { + Component host; + host.id = "pd-it-host"; + host.name = "pd-it-host"; + host.fqn = "pd-it-host"; + snapshot_.components.push_back(host); + + set_apps({{"pd_it_target", "/pd_it_target"}}); // speed_limit is declared in SetUp + + auto det = make_param_drift(); + det->configure(nlohmann::json{{"max_reads_per_tick", 4}}); + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_target")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear ever arrived, so the target was never read and no baseline exists to drift from"; + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + ASSERT_EQ(failed_before, 0u) << "baseline capture must be silent for the rest of this test to mean anything"; + target_->set_parameter(rclcpp::Parameter("speed_limit", 0.2)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // The discriminator: nothing was ever reported under the component id. + EXPECT_EQ(count_faults("pd-it-host", ReportFault::Request::EVENT_FAILED), 0u); + EXPECT_EQ(count_faults("pd-it-host", ReportFault::Request::EVENT_PASSED), 0u); +} + +TEST_F(ParamDriftIntegrationTest, ExpectRuleFlagsWrongFromStart) { + // A separate node that comes up already violating the production pin use_sim_time=false. + auto bad = std::make_shared("pd_it_badsim"); + bad->set_parameter(rclcpp::Parameter("use_sim_time", true)); + exec_.add_node(bad); + const auto bad_guard = on_scope_exit([this, bad] { + exec_.remove_node(bad); + }); + set_apps({{"pd_it_badsim", "/pd_it_badsim"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json{{"baseline", false}, {"expect", {{"use_sim_time", false}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_badsim")); // service up before we assert a read-driven raise + // Live use_sim_time (true) != expected (false) -> a NEW raise appears with no param set, + // on first read. A detector that never actually compares would produce no such fault. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); +} + +// A node whose parameter services EXIST (discovery succeeds) but which never answers them - the +// #531 unresponsive-node scenario. What has to hold is not a property of tick(): tick() does no IO +// at all in this design, so the time it takes says nothing about a read timeout, and the same +// assertion passes with the timeout raised to an hour. The bounded read is a property of the READER +// THREAD, and the way to see it is to require the sweep to get PAST the node that will not answer. +// +// The unresponsive app sorts FIRST, so the reader's round-robin reaches it before anything else. +// With the read bounded, it times out and the healthy app behind it is served, over and over. With +// the bound removed the reader parks on the very first target for good: the healthy app is never +// read, its `expect` violation is never seen, and neither assertion below can be satisfied. +// +// Teardown is the second half. The detector is destroyed while the fake service is STILL wedged - +// the test does not release it first - because that is the shape of a real shutdown: its destructor +// has to shut the transport down and JOIN the reader with a read outstanding. +TEST_F(ParamDriftIntegrationTest, AnUnresponsiveNodeDoesNotStallTheReader) { + const std::string kUnresp = "pd_it_aunresp"; // sorts before pd_it_target: first in the rotation + std::atomic stop{false}; + auto unresp = std::make_shared(kUnresp, rclcpp::NodeOptions().start_parameter_services(false)); + // list_parameters is the round-trip both read modes make first; it never replies. It loops on + // `stop` (not rclcpp::ok()) so the fake node can be released deterministically at the very end. + auto list_srv = unresp->create_service( + "/" + kUnresp + "/list_parameters", service_callback( + [&stop](const rcl_interfaces::srv::ListParameters::Request & /*req*/, + rcl_interfaces::srv::ListParameters::Response & /*resp*/) { + while (!stop.load()) { + std::this_thread::sleep_for(10ms); + } + })); + // get_parameters only needs to exist in the graph so discovery (wait_param_service) succeeds. + auto get_srv = unresp->create_service( + "/" + kUnresp + "/get_parameters", service_callback( + [](const rcl_interfaces::srv::GetParameters::Request & /*req*/, + rcl_interfaces::srv::GetParameters::Response & /*resp*/) {})); + rclcpp::executors::SingleThreadedExecutor unresp_exec; + unresp_exec.add_node(unresp); + std::thread unresp_spin([&unresp_exec] { + unresp_exec.spin(); + }); + // Declared BEFORE the detector so it is destroyed AFTER it: the wedged handler owns the fake + // node's only executor thread, so nothing can join that thread until `stop` is set, and the + // detector's own teardown must happen while the node is still refusing to answer. + const auto release_guard = on_scope_exit([&stop, &unresp_exec, &unresp_spin] { + stop.store(true); + unresp_exec.cancel(); + if (unresp_spin.joinable()) { + unresp_spin.join(); + } + }); + + set_apps({{kUnresp, "/" + kUnresp}, {"pd_it_target", "/pd_it_target"}}); + auto det = make_param_drift(); + // `expect` mode: the healthy target declares speed_limit=1.0 (see SetUp) and the pin says 0.0, so + // it violates from its very first read and no baseline warm-up is needed - which matters here, + // because the app that would hold up a warm-up is precisely the one that never answers. + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 100}, {"max_reads_per_tick", 8}, {"expect", {{"speed_limit", 0.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/" + kUnresp)); // services discoverable (but unresponsive) + ASSERT_TRUE(wait_param_service("/pd_it_target")); // the healthy app behind it + + // The sweep got past the node that will not answer. + ASSERT_TRUE(poll_until( + [this]() { + return latest_failed_desc_contains("graph_watchdog", {"pd_it_target", "got=1.0"}); + }, + *det, ctx)) + << "the healthy app behind the unresponsive one was never read, so the reader is parked on a " + "read that never returns: " + << latest_failed_desc("graph_watchdog"); + + // And it KEEPS getting past it. A description that names the new value can only come from a read + // taken after the change, i.e. from a rotation that came round again. + target_->set_parameter(rclcpp::Parameter("speed_limit", 7.25)); + EXPECT_TRUE(poll_until( + [this]() { + return latest_failed_desc_contains("graph_watchdog", {"got=7.25"}); + }, + *det, ctx)) + << "the healthy app was read once and never again, so the sweep stopped rotating: " + << latest_failed_desc("graph_watchdog"); + + // Nothing was invented about the node that never answered: it is skipped, never raised against. + EXPECT_EQ(latest_failed_desc("graph_watchdog").find(kUnresp), std::string::npos) + << "the unresponsive node was reported as drifted on a read that never completed"; + + // Teardown with the fake service still wedged. An unbounded read would leave the reader inside + // rcl when the destructor joins it, and the join would never return. + const auto teardown_started = std::chrono::steady_clock::now(); + det.reset(); + const auto teardown = std::chrono::steady_clock::now() - teardown_started; + EXPECT_LT(teardown, 8s) << "destroying the detector while a node was refusing to answer took " + << std::chrono::duration_cast(teardown).count() + << " ms: the reader could not be joined out of an unbounded read"; +} + +// Namespaced-node gate keying, with a REAL wired gate. The discriminator is LIFECYCLE +// suppression, NOT "FQN denied": the gate has two allow-paths - a KNOWN entity is allowed iff +// armed && lifecycle-ok; an UNKNOWN source_id is allowed once past the global bringup grace. +// So an unknown FQN key past grace returns TRUE. The teeth: a known+armed entity whose +// lifecycle is non-active is suppressed (false) via its app.id key, while the SAME string used +// as an unknown FQN key is allowed (true) because it bypasses per-entity lifecycle. The detector +// keys reliability_allows() by app.id; an FQN-keying detector would slip past the suppression. +// +// The negative window is not a hand-picked constant. Silence only means something if it lasted +// longer than a raise demonstrably needs, so the positive control below is TIMED - from the moment +// the gate opens to the moment the raise lands - and the window the negative phase watched for has +// to be at least that long. A 500 ms window next to a 13 s positive poll is not a discriminator, it +// is an accident of how long the negative loop happened to run. +TEST_F(ParamDriftIntegrationTest, NamespacedNodeGateKeyingLifecycleDiscriminator) { + // node n in namespace /ns: FQN /ns/n != app.id n + auto nsnode = std::make_shared("n", "/ns"); + nsnode->declare_parameter("gain", 1.0); + exec_.add_node(nsnode); + const auto nsnode_guard = on_scope_exit([this, nsnode] { + exec_.remove_node(nsnode); + }); + ASSERT_TRUE(wait_param_service("/ns/n")); + IntrospectionInput snap; + { + App a; + a.id = "n"; + a.bound_fqn = "/ns/n"; + snap.apps.push_back(a); + } + ReliabilityGate gate(3, gateway_.get(), &node_mutex_); + gate.update(snap, 5); + gate.update(snap, 8); // 3 elapsed -> n armed, global grace met + gate.set_lifecycle_state_for_test("n", "inactive"); + ASSERT_FALSE(reliability_allows(&gate, "n")); // app.id key: armed but lifecycle-inactive -> SUPPRESSED + ASSERT_TRUE(reliability_allows(&gate, "/ns/n")); // FQN key: unknown + past grace -> allowed (the bug path) + auto det = make_param_drift(); + det->configure(nlohmann::json{{"tick_interval_ms", 100}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + ctx.gate = &gate; + ctx.snapshot = &snap; + // NEGATIVE phase: correct app.id keying keeps the drifting node suppressed. Timed from the first + // tick, because that is when an FQN-keying detector would start capturing. + const auto negative_started = std::chrono::steady_clock::now(); + for (int i = 0; i < 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + nsnode->set_parameter(rclcpp::Parameter("gain", 0.2)); + for (int i = 0; i < 30; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + const auto negative_window = std::chrono::steady_clock::now() - negative_started; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); + // ^ an FQN-keying detector would see "/ns/n" allowed -> capture + raise; 0 proves app.id keying. + // POSITIVE control: flip lifecycle active -> the same node's drift now raises (proves the read/aggregate + // path works, so the 0 above was genuine app.id suppression, not a dead read). + const auto positive_started = std::chrono::steady_clock::now(); + gate.set_lifecycle_state_for_test("n", "active"); + ASSERT_TRUE(reliability_allows(&gate, "n")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "the node was never read once the gate opened, so the positive control below cannot time " + "anything"; // captures the current value (0.2) silently + const auto before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + nsnode->set_parameter(rclcpp::Parameter("gain", 0.9)); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, before, *det, ctx)); + const auto positive_latency = std::chrono::steady_clock::now() - positive_started; + + EXPECT_GE(negative_window, positive_latency) + << "the suppressed phase was watched for only " + << std::chrono::duration_cast(negative_window).count() + << " ms while an allowed node takes " + << std::chrono::duration_cast(positive_latency).count() + << " ms to arm, be read and raise: the silence above is the window being too short, not the " + "gate keying being right"; +} + +// A drift that is already reported must not be healed by the reliability gate closing on its +// app. The gate exists to hold back a RAISE about an entity that has not settled; it is not +// evidence that the parameter went back to its expected value. The aggregate's raise is gated on +// the aggregate's own source and never on the app, so dropping the app's finding when the gate +// denies it is the only thing that could clear it - straight into clear_fault, which carries no +// gate at all. A nav2 lifecycle_manager pausing a controller would then walk a still-present +// drift to HEALED and re-raise it on resume, flapping on every pause. +TEST_F(ParamDriftIntegrationTest, GateDenialDoesNotHealAPresentDrift) { + auto node = std::make_shared("g", "/gd"); + node->declare_parameter("gain", 1.0); + exec_.add_node(node); + const auto node_guard = on_scope_exit([this, node] { + exec_.remove_node(node); + }); + ASSERT_TRUE(wait_param_service("/gd/g")); + + IntrospectionInput snap; + { + App a; + a.id = "gd_frozen"; + a.bound_fqn = "/gd/g"; + snap.apps.push_back(a); + } + ReliabilityGate gate(3, gateway_.get(), &node_mutex_); + gate.update(snap, 5); + gate.update(snap, 8); // armed, global grace met + gate.set_lifecycle_state_for_test("gd_frozen", "active"); + ASSERT_TRUE(reliability_allows(&gate, "gd_frozen")); + + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + ctx.gate = &gate; + ctx.snapshot = &snap; + + for (int i = 0; i < 6; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + node->set_parameter(rclcpp::Parameter("gain", 0.2)); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // The gate closes while the parameter is still wrong. Nothing about the robot got better. + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + gate.set_lifecycle_state_for_test("gd_frozen", "inactive"); + ASSERT_FALSE(reliability_allows(&gate, "gd_frozen")); + for (int i = 0; i < 8; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "the gate closing healed a drift that is still present"; + EXPECT_NE(latest_failed_desc("graph_watchdog").find("gd_frozen"), std::string::npos) + << "the suppressed app vanished from the CURRENT description, so nothing tells the operator it " + "is still drifting: " + << latest_failed_desc("graph_watchdog"); + + // Positive control: the drift can still be cleared the only legitimate way, by fixing it. + gate.set_lifecycle_state_for_test("gd_frozen", "active"); + const auto passed_before2 = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + node->set_parameter(rclcpp::Parameter("gain", 1.0)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before2, *det, ctx)); +} + +// The other end of the freeze, and the one nothing covered. Freezing a finding is right for a +// PAUSE - the drift is still real and the gate only says "do not raise about an unsettled entity". +// It is wrong forever. A node that leaves `active` for good is never re-evaluated (not armed) and +// never pruned (still present in the graph), so without a release the aggregate keeps re-raising a +// finding nobody can act on: an operator who fixes the parameter and leaves the node deactivated +// watches GRAPH_PARAM_DRIFT stay CONFIRMED for the life of the gateway, naming a value that is no +// longer true. +// +// kFrozenHoldTicks is 60 consecutive unarmed ticks. Both ends of that are pinned here, because a +// release that fires too early is the flapping GateDenialDoesNotHealAPresentDrift forbids and a +// release that never fires is the latch above. Deleting the release leaves the suite green. +// +// The upper end is pinned by a SMALL number of further ticks rather than by a poll loop. The hold +// is a count the detector keeps privately, so the only thing that can hold the literal below to it +// is squeezing the release between "not yet after N" and "by N+3": a poll loop that ticks a +// hundred more times leaves the literal free to drift away from the constant it mirrors, and +// tripling kFrozenHoldTicks then stays green. +TEST_F(ParamDriftIntegrationTest, AFrozenFindingIsReleasedOnceTheHoldElapses) { + constexpr int kFrozenHoldTicks = 60; // mirrors the detector's constant; pinned from both sides below + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}}); + + IntrospectionInput snap; + { + App a; + a.id = "pd_it_hold"; + a.bound_fqn = "/pd_it_hold"; + snap.apps.push_back(a); + } + ReliabilityGate gate(3, gateway_.get(), &node_mutex_); + gate.update(snap, 5); + gate.update(snap, 8); // armed, global grace met + gate.set_lifecycle_state_for_test("pd_it_hold", "active"); + ASSERT_TRUE(reliability_allows(&gate, "pd_it_hold")); + + auto ctx = make_ctx(DetectorMode::Raise); + ctx.gate = &gate; + ctx.snapshot = &snap; + + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) << "the app was never read, so it has no baseline to drift from"; + ASSERT_NE(fake, nullptr); + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(9.0); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "the drift never raised, so there is no frozen finding whose release this test can observe"; + + // The gate closes and stays closed. No sleeps from here on: the hold is counted in TICKS, and + // while the app is unarmed the reader has no targets, so there is nothing to wait for. + gate.set_lifecycle_state_for_test("pd_it_hold", "inactive"); + ASSERT_FALSE(reliability_allows(&gate, "pd_it_hold")); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < kFrozenHoldTicks; ++i) { + det->tick(ctx); + } + // Reports are service round trips, so let the in-flight ones land before asserting that none of + // them was a clear. + std::this_thread::sleep_for(300ms); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "the frozen finding was released before the hold had elapsed, so an ordinary lifecycle " + "pause walks a still-present drift to HEALED"; + + // Three more ticks, no more. The release is due on the very next one. + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_GT(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "the finding stayed frozen past kFrozenHoldTicks, so a node that leaves 'active' for good " + "keeps GRAPH_PARAM_DRIFT raised on a parameter an operator may already have fixed"; + + // Positive control: releasing the finding is not the same as going deaf. The app arms again and + // the drift - which is still present - is re-raised from a fresh read. + gate.set_lifecycle_state_for_test("pd_it_hold", "active"); + const auto failed_after_release = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_after_release, *det, ctx)) + << "the released finding never came back once the app re-armed, even though the parameter is " + "still wrong: the release dropped the drift instead of re-deriving it"; +} + +// Node removal drives prune + clear e2e. Two nodes, one drifts and raises the aggregated +// fault; then the drifted node's App is REMOVED from the snapshot. After > kForgetGrace absent +// sweeps prune_vanished forgets it and drops it from drifted_, so the aggregate must CLEAR. +// Gutting prune_vanished to a no-op leaves the drift latched forever and fails this. +TEST_F(ParamDriftIntegrationTest, NodeRemovalPrunesAndClears) { + auto keep = std::make_shared("pd_it_keep"); + keep->declare_parameter("k", 1.0); + auto gone = std::make_shared("pd_it_gone"); + gone->declare_parameter("g", 1.0); + exec_.add_node(keep); + exec_.add_node(gone); + const auto nodes_guard = on_scope_exit([this, keep, gone] { + exec_.remove_node(keep); + exec_.remove_node(gone); + }); + set_apps({{"pd_it_keep", "/pd_it_keep"}, {"pd_it_gone", "/pd_it_gone"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_keep")); + ASSERT_TRUE(wait_param_service("/pd_it_gone")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear arrived, so at least one of the two apps was never read and has no baseline"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); + + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + gone->set_parameter(rclcpp::Parameter("g", 9.0)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // Remove the drifted node from the snapshot: after > kForgetGrace sweeps it is pruned, + // shedding its drift from the aggregate -> a clear must arrive. + set_apps({{"pd_it_keep", "/pd_it_keep"}}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)); +} + +// Forget raised path. Node drifts (aggregate raised) -> vanishes past grace (aggregate +// clears) -> reappears at a NEW healthy value distinct from its original baseline. Because +// forget() dropped its baseline, the first read after reappearance re-captures silently: NO +// stale/re-raised FAILED for it, and the aggregate stays CLEAR. If the baseline were NOT +// forgotten, the new healthy value would drift against the stale (original) baseline. +TEST_F(ParamDriftIntegrationTest, VanishedThenHealthyReappearanceStaysClear) { + auto keep = std::make_shared("pd_it_tbkeep"); + keep->declare_parameter("k", 1.0); + auto reap = std::make_shared("pd_it_reap"); + reap->declare_parameter("r", 1.0); + exec_.add_node(keep); + exec_.add_node(reap); + const auto nodes_guard = on_scope_exit([this, keep, reap] { + exec_.remove_node(keep); + exec_.remove_node(reap); + }); + set_apps({{"pd_it_tbkeep", "/pd_it_tbkeep"}, {"pd_it_reap", "/pd_it_reap"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_tbkeep")); + ASSERT_TRUE(wait_param_service("/pd_it_reap")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear arrived, so at least one of the two apps was never read and has no baseline"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); + + // Drift reap -> aggregate raised. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + reap->set_parameter(rclcpp::Parameter("r", 9.0)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // Vanish reap past grace -> pruned, aggregate clears. + set_apps({{"pd_it_tbkeep", "/pd_it_tbkeep"}}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)); + + // reap reappears at a NEW healthy value (5.0) - distinct from BOTH the original baseline + // (1.0) and the drifted value (9.0). This is deliberate: if forget() were missing, the + // stale baseline (1.0) would remain and 1.0 != 5.0 would raise a fresh spurious FAILED; with + // forget() in place, the stale baseline is gone and 5.0 is silently re-captured as the new + // baseline. Reusing 1.0 here would make the test pass vacuously even with a missing forget() + // (stale baseline == reappearance value looks "clean" either way). + reap->set_parameter(rclcpp::Parameter("r", 5.0)); // healthy, but distinct from the original baseline + set_apps({{"pd_it_tbkeep", "/pd_it_tbkeep"}, {"pd_it_reap", "/pd_it_reap"}}); + const auto failed_at_reappear = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + for (int i = 0; i < 8; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), failed_at_reappear) + << "stale/re-raised FAILED after healthy reappearance - baseline not forgotten on vanish"; + // The aggregate is currently clear: a fresh PASSED continues to flow. + const auto passed_reappear = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_reappear, *det, ctx)); +} + +// A SUCCESS response carrying an empty parameter array is a legitimate transport reply - the +// gateway transport documents it, for instance when a parameter declaration races the list. It is +// NOT evidence that nothing drifts: evaluate() would compare against no observations at all, find +// nothing, and the tick would clear a fault that is still true. No live node can be made to answer +// that way, which is why this needs the stand-in. +TEST_F(ParamDriftIntegrationTest, AnEmptyBatchDoesNotClearARaisedDrift) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + set_apps({{"pd_it_empty", "/pd_it_empty"}}); + for (int i = 0; i < 6; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for the rest of this test to mean anything"; + + // Drift it, and confirm the fault is genuinely up before testing what clears it. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(9.0); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // Now every read answers SUCCESS with nothing in it. The parameter is still 9.0. + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + fake->set_empty_batch(true); + for (int i = 0; i < 12; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "an empty batch was treated as a complete view and cleared a drift that is still present"; + + // Positive control: a real view showing the value restored does clear it. + fake->set_empty_batch(false); + fake->set_value(1.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)); +} + +// The reader releases the lock for the whole round trip, so a read can still be in flight when the +// tick thread drops its app. Publishing it anyway is not merely stale: prune_vanished has already +// forgotten the baseline and erased the cache, so the late sample lands in an empty cache and +// becomes the baseline the moment the app returns. The node is then compared against a value it +// held only while it was NOT a target - a drift no later read can heal, because the reference +// itself is wrong. This is the failure the still_a_target fence exists to prevent. +// +// Only reachable through the stand-in: a live read returns in milliseconds and nothing in a real +// graph holds one open across the three ticks the prune needs. +TEST_F(ParamDriftIntegrationTest, AReadLandingAfterItsAppWasDroppedIsNotPublished) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + // `keep` stays armed for the whole test. It is what makes the sequencing observable: the reader + // is a single thread, so a read of `keep` completing AFTER the held read of `fence` was released + // proves the fenced publish has already happened. With `fence` alone the loop would park on the + // cv and the test would have to race the very publish it is trying to observe. + const std::string kFence = "/pd_it_fence"; + const std::string kKeep = "/pd_it_keep"; + set_apps({{"pd_it_fence", kFence}, {"pd_it_keep", kKeep}}); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "both apps must be read and clean before the fence can be tested"; + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for the rest of this test to mean anything"; + + // Hold a read of `fence` open, and make it answer with a value the node never had while armed. + // value_for() is read after the gate, so the held call returns 9.0 when it is finally released. + fake->block_node(kFence); + ASSERT_TRUE(fake->wait_until_parked()) << "no read of the fenced app was ever held"; + fake->set_value(kFence, 9.0); + + // Drop `fence` from the graph for long enough to prune it: tick 1 takes it out of `targets`, + // tick 3 (kBaselineForgetGrace + 1) forgets its baseline and erases its cached read. + set_apps({{"pd_it_keep", kKeep}}); + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + + // Let the held read finish, then wait for a read of `keep` to prove the reader moved past it. + const int calls_before_release = fake->calls(); + fake->release_node(kFence); + ASSERT_TRUE(fake->wait_until_reading(calls_before_release + 1)) + << "the reader never got past the released read, so nothing was published either way"; + + // The app returns holding the value it always had. A fresh baseline is 1.0 and nothing drifts. + // Had the late 9.0 been published, it would be the baseline instead, and this read is the drift. + fake->set_value(kFence, 1.0); + set_apps({{"pd_it_fence", kFence}, {"pd_it_keep", kKeep}}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the returned app was never re-read"; + // ~2 s against a 250 ms charge per baseline visit (2 round trips x a 125 ms slot) over two + // targets: several fresh reads of `fence`. + for (int i = 0; i < 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "a read that completed after its app was dropped became the baseline on the app's return"; +} + +// A throw out of a read must not escape the reader thread. reader_loop runs on a std::thread with +// no outer handler, so an escape is std::terminate - the whole gateway process, not just the +// detector. rclcpp can throw from inside a blocking call when the context is invalidated mid-read, +// and the plugin's per-detector guard does not cover this thread. +// +// Deleting the reader's try/catch makes this test abort the test binary rather than fail it, which +// is exactly the point: today nothing in the suite ever makes a read throw. +TEST_F(ParamDriftIntegrationTest, AThrowingReadDoesNotKillTheReaderThread) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + const std::string kThrow = "/pd_it_throw"; + const std::string kKeep = "/pd_it_keep2"; + set_apps({{"pd_it_throw", kThrow}, {"pd_it_keep2", kKeep}}); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "both apps must be read and clean before the throw is introduced"; + ASSERT_NE(fake, nullptr); + + fake->set_throwing(kThrow); + ASSERT_TRUE(fake->wait_until_threw(2)) << "the reader never issued a read that throws"; + + // The surviving reader must still do its job: drift the OTHER app and require the raise. A + // reader that died silently would leave this polling until it times out. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(kKeep, 9.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "the reader stopped working after a read threw"; +} + +// The pre-shutdown callback is what stops the reader while the context is STILL valid. Without it +// the reader can be inside a blocking rcl call when rclcpp invalidates the context, and rcl aborts +// rather than throws - no try/catch reaches that. Nothing in the suite runs the callback, because +// shutting the global context down would take every remaining test with it. A context owned by +// this test drives the same code path in isolation. +TEST_F(ParamDriftIntegrationTest, ContextShutdownStopsTheReaderWhileTheDetectorIsStillAlive) { + auto own_context = std::make_shared(); + own_context->init(0, nullptr); + rclcpp::NodeOptions opts; + opts.context(own_context); + auto own_node = std::make_shared("pd_it_shutdown_gw", opts); + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + ctx.gateway_node = own_node.get(); // the callback is registered on THIS node's context + + set_apps({{"pd_it_shutdown", "/pd_it_shutdown"}}); + det->tick(ctx); + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_reading(1)) << "the reader never started"; + + // Returns only after the callback has waited for the reader to leave its loop, so the call + // count is stable from here on - unless nothing stopped the reader at all. + own_context->shutdown("test"); + + const int at_shutdown = fake->calls(); + std::this_thread::sleep_for(1s); // ~4 baseline visits at the default budget (2 round trips each) + EXPECT_EQ(fake->calls(), at_shutdown) + << "the reader was still issuing reads after the context shut down, so a blocking rcl call " + "could be in flight when the context is invalidated"; +} + +// The budget is spent in service ROUND TRIPS at the watched node - not in node visits, and not in +// transport calls either. One `get_parameter` is three round trips there (a name-scoped list, a +// get, and an unconditional describe), and an `expect` visit makes one per pinned name, so +// charging a slot per visit lets N pins put 3N times the promised load on one node, and charging a +// slot per call still lets it put three times that load, while the knob an operator lowered to +// protect the node reports being honoured either way. +// +// Four pins and a budget of ten round trips per 1000 ms tick period: one slot is 100 ms, so a visit +// costs 4 x 3 = 12 slots and owes 1200 ms. Visits land at ~0.0 s, ~1.2 s and ~2.4 s, and a fourth +// would need ~3.6 s, so a 3 s window holds exactly three visits: twelve calls. Charging per call +// owes 400 ms and fits eight visits (32 calls); charging per visit owes 100 ms and fits about +// thirty (120 calls). +TEST_F(ParamDriftIntegrationTest, TheBudgetIsSpentInRoundTripsNotCalls) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"baseline", false}, + {"tick_interval_ms", 1000}, + {"max_reads_per_tick", 10}, + {"expect", {{"a", 1.0}, {"b", 1.0}, {"c", 1.0}, {"d", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_budget", "/pd_it_budget"}}); + + const auto started = std::chrono::steady_clock::now(); + while (std::chrono::steady_clock::now() - started < 3s) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_NE(fake, nullptr); + const int calls = fake->calls(); + // Both bounds carry weight. Without the upper one, per-call and per-visit charging both pass. + // Without the lower one, so does a reader that paced itself into silence after a single visit - + // the test would then be proving that nothing happened, which any amount of over-pacing also + // satisfies. The gap between them is one visit wide on purpose: a fourth visit inside the window + // is the smallest observable under-charge, and it is excluded. + EXPECT_GE(calls, 12) << "issued " << calls + << " calls in ~3 s: the third visit never arrived, so the budget is being " + "spent slower than the knob promises"; + EXPECT_LE(calls, 15) << "issued " << calls + << " calls in ~3 s against a budget of 10 round trips per 1000 ms: a fourth " + "visit fit in the window, so a visit was charged less than the three " + "round trips each of its four pins really costs the node"; +} + +// The other half of charging what a read really costs: a pin the node does not declare is answered +// from the name-scoped list alone and never reaches the get or the describe, so it costs the node +// less than a pin that resolves. `expect` is documented as checked on every node that HAS the +// parameter, so on a mixed graph most nodes answer NOT_FOUND for most pins - charging those the +// full three round trips would throttle the sweep THREE TIMES as hard as the load it creates - the +// load being one round trip, not two: the charge of two is deliberately the higher of the two +// NOT_FOUND paths, while what the node really pays on the common one is a single name-scoped list. +// The coverage latency an operator computes from the knob would be wrong in the safe-looking +// direction. +// +// Four pins, none of them declared, a budget of eight round trips per 125 ms tick period: one slot +// is 15.625 ms, so a visit costs 4 x 2 = 8 slots and owes 125 ms. The ninth visit therefore starts +// at ~1.0 s. Charging NOT_FOUND the full three owes 187.5 ms and puts it at ~1.5 s; charging it one +// owes 62.5 ms and puts it at ~0.5 s. The band below excludes both. Timing the reads rather than +// counting them in a fixed window is what buys a quarter-second of margin on each side. +TEST_F(ParamDriftIntegrationTest, AnUndeclaredPinIsChargedLessThanOneThatResolves) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + // Declared-set restriction installed in the factory: the reader starts on the first tick and the + // very first visit must already be answered NOT_FOUND, or it is charged as a resolving read. + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_declared(std::set{}); // the node declares none of the pins + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"baseline", false}, + {"tick_interval_ms", 125}, + {"max_reads_per_tick", 8}, + {"expect", {{"a", 1.0}, {"b", 1.0}, {"c", 1.0}, {"d", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_notfound", "/pd_it_notfound"}}); + + // One tick arms the reader; it then round-robins its single target on its own, so the elapsed + // time to the 36th call is pure pacing and not the tick loop's cadence. + const auto started = std::chrono::steady_clock::now(); + det->tick(ctx); + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_reading(36)) << "the reader never issued nine visits, so it is paced " + "far slower than any charge this test can distinguish"; + const auto elapsed = std::chrono::steady_clock::now() - started; + const auto ms = std::chrono::duration_cast(elapsed).count(); + EXPECT_GT(ms, 750) << "nine visits took " << ms + << " ms: an undeclared pin was charged less than the round trip it does cost, " + "so the budget under-states the load the sweep puts on the node"; + EXPECT_LT(ms, 1250) << "nine visits took " << ms + << " ms: an undeclared pin was charged the full price of one that resolves, " + "so the sweep is throttled for a get and a describe it never issued"; +} + +// A clear says "nothing is drifting anywhere", and the only thing that can make that true is having +// LOOKED everywhere. Any horizon expressed in elapsed ticks eventually expires, and the rotation it +// is racing has no upper bound: with a 1 s tick, one read per tick and 120 apps, the derived give-up +// horizon ran out at tick 3612 while sixty apps had still never been contacted, so tick 3613 +// cleared GRAPH_PARAM_DRIFT on evidence that did not exist. Bigger graph, longer silence, MORE +// confidence - exactly backwards, and worst on the graphs where the sweep is slowest. +// +// 160 apps at one round trip per 200 ms tick period: a baseline visit owes two slots, i.e. 400 ms, +// so one rotation takes a minute. The tick loop below carries no sleep and runs far past the old +// horizon inside a fraction of that, so nearly the whole graph is uncontacted throughout. Nothing +// may be cleared, and nothing may be SAID about an app the sweep has not reached either - a warning +// that "no read has succeeded" for a node nobody has tried to read is the same defect in the log. +TEST_F(ParamDriftIntegrationTest, NoClearWhileTheSweepHasNotReachedEveryApp) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 200}, {"max_reads_per_tick", 1}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + for (int i = 0; i < 160; ++i) { + const std::string id = "pd_it_wide" + std::to_string(1000 + i); // uniform width -> sorted == creation order + apps.emplace_back(id, "/" + id); + } + set_apps(apps); + det->tick(ctx); // builds the transport and hands the reader all 160 targets + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_reading(1)) << "the reader never started, so nothing here is under test"; + + // No sleep: this is about tick COUNT, and the point is that no number of ticks can substitute for + // a read. 4000 is past the horizon the old derivation capped out at (3612) for any graph at all. + for (int i = 0; i < 4000; ++i) { + det->tick(ctx); + } + // The reader takes targets round-robin in order, and a baseline visit is exactly one call, so a + // call count below the app count means the tail of the graph was never contacted. This counts + // COVERAGE, not load - the stand-in cannot measure round trips and no bound here depends on it. + ASSERT_LT(fake->calls(), 160) << "the sweep got round all 160 apps inside the window, so " + "'uncontacted' was never the state under test"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), 0u) + << "cleared GRAPH_PARAM_DRIFT while most of the graph had never been contacted - that is " + "health the detector never measured, and it is what any horizon in ticks does once the " + "rotation outlasts it"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "nothing drifted, so nothing may be raised either"; + EXPECT_EQ(log.count("consecutive parameter reads of"), 0) + << "told the operator that reads of an app had failed when no read of it was ever attempted"; + EXPECT_EQ(log.count("giving up on"), 0) + << "gave up on an app the sweep had not reached, on the strength of elapsed ticks alone"; + + // Positive control: reduce the graph to an app that HAS been read and the clear flows at once. + // The zeroes above are the guard doing its job, not a detector that emits nothing. + set_apps({apps.front()}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "no clear even once every remaining app had been read, so the guard never releases"; +} + +// The other half of the same rule. Silence about an app is only ever reported once we have tried +// and failed - repeatedly - to read it, and the count that decides both warnings is a count of +// ATTEMPTS, so it cannot be reached by waiting. Both warnings fire once per run of failures, in +// order, and only about the app that will not answer. +// +// Two apps at eight round trips per 10 ms tick period: one slot is 1.25 ms and both a successful +// baseline visit and a failed one are charged two slots, so the ghost is retried roughly every +// 5 ms and the sixty-first failure lands inside a second. +TEST_F(ParamDriftIntegrationTest, TheUnreadWarningsRequireFailedAttemptsNotElapsedTicks) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_mute"); // before the reader can ever answer for it + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_heard", "/pd_it_heard"}, {"pd_it_mute", "/pd_it_mute"}}); + + for (int i = 0; i < 400 && log.count("giving up on") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_NE(fake, nullptr); + EXPECT_EQ(log.count("giving up on"), 1) << "the app that never answers was never given up on, so it " + "holds back every clear from here on"; + EXPECT_EQ(log.count("consecutive parameter reads of 'pd_it_mute'"), 1) + << "the app that never answers was never named, or was named more than once - the warning is " + "not latched"; + EXPECT_EQ(log.count("consecutive parameter reads of"), 1) + << "the healthy app was named alongside it, even though every read of it succeeded"; + + // Both are one-shot. Keep ticking well past the point they fired. + for (int i = 0; i < 100; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + EXPECT_EQ(log.count("consecutive parameter reads of"), 1) + << "the unread warning repeats every tick once its count is passed"; + EXPECT_EQ(log.count("giving up on"), 1) << "the give-up warning repeats every tick once its count is passed"; +} + +// A baseline visit is one list_parameters(), and that is TWO round trips at the node: the list, +// then the batched get of every name it returned. Charging it one slot - the old per-call rate - +// puts twice the promised load on every watched node in this detector's default mode. +// +// The other half of the rule is what is deliberately NOT charged. The first list_parameters() +// against a node costs two more, because the transport primes its defaults cache before its own +// read, and that cache has no TTL: the extra pair is a one-off per node for the life of the +// gateway. Charging four on every read would throttle the whole run to pay for the first rotation. +// +// One app, a budget of eight round trips per 125 ms tick period: one slot is 15.625 ms, so a visit +// costs 2 slots and owes 31.25 ms, putting the 33rd read at ~1.0 s. One slot per visit puts it at +// ~0.5 s and the cold-start four at ~2.0 s; the band excludes both. +TEST_F(ParamDriftIntegrationTest, ABaselineVisitIsChargedTwoRoundTripsAndNotTheColdStartPair) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 125}, {"max_reads_per_tick", 8}}); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_bl", "/pd_it_bl"}}); + + // One tick arms the reader; it round-robins its single target on its own from there, so the + // elapsed time is pure pacing rather than the tick loop's cadence. + const auto started = std::chrono::steady_clock::now(); + det->tick(ctx); + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_reading(33)) << "the reader never issued 33 baseline visits, so it is " + "paced far slower than any charge this test can distinguish"; + const auto elapsed = std::chrono::steady_clock::now() - started; + const auto ms = std::chrono::duration_cast(elapsed).count(); + EXPECT_GT(ms, 750) << "33 baseline visits took " << ms + << " ms: a visit was charged less than the list-plus-get it really costs the " + "node, so the budget under-states the load by half"; + EXPECT_LT(ms, 1250) << "33 baseline visits took " << ms + << " ms: a visit was charged the cold-contact price of four, which is paid " + "once per node for the life of the gateway, not on every read"; +} + +// The same condition that governs a clear governs the unmatched-`expect`-pin warning, and for the +// same reason: only one node is visited per rotation slot, so a pin that exactly one node declares +// is legitimately unseen until the sweep reaches that node. On any threshold counted in elapsed +// ticks the detector announces that "no app declares" it long before it has looked at the app that +// does - a warning that tells an operator to go and fix a name that is already correct. So the +// warning is withheld until every armed app has been read (or given up on) and at least one of +// them answered. +// +// Twenty-one apps, one pin, one round trip per 10 ms tick period. An undeclared pin is charged two +// round trips, so the twenty nodes that answer NOT_FOUND owe two slots each and the one node that +// declares the pin - it sorts last - is reached after forty slots, i.e. ~400 ms. The 140 ticks +// below span well over a second, so the sweep completes several times over with nothing reported, +// which is only true if the pin was actually MATCHED rather than merely not yet due. +TEST_F(ParamDriftIntegrationTest, APinOnlyTheLastNodeDeclaresIsNotReportedAsUnmatched) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + // Only the last-sorting node has the pin; the rest answer NOT_FOUND for it from the first read. + t->set_declared(std::set{}); + t->set_declared("/pd_it_pinlast", {"a_pin"}); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 10}, {"max_reads_per_tick", 1}, {"expect", {{"a_pin", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + for (int i = 0; i < 20; ++i) { + const std::string id = "pd_it_pin" + std::to_string(100 + i); // uniform width; all sort before "pinlast" + apps.emplace_back(id, "/" + id); + } + apps.emplace_back("pd_it_pinlast", "/pd_it_pinlast"); + set_apps(apps); + + for (int i = 0; i < 140; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + ASSERT_NE(fake, nullptr); + EXPECT_EQ(log.count("no app declares 'a_pin'"), 0) + << "told the operator no app declares a pin that the last node in the rotation declares " + "perfectly well - the warning fired before the sweep had read that node"; +} + +// A throw while building the shared IO must leave io_ null so the next tick retries. Assigning it +// first and filling it in afterwards latches a half-built state: the plugin's per-detector guard +// logs the throw once, every later tick sees a non-null io_ and skips the setup, and the detector +// then runs forever with no reader at all - indistinguishable from a healthy one at every call site. +TEST_F(ParamDriftIntegrationTest, AFailedLazyInitRetriesOnTheNextTick) { + ScriptedTransport * fake = nullptr; + int attempts = 0; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test( + [&fake, &attempts](rclcpp::Node *) -> std::unique_ptr { + if (++attempts == 1) { + throw std::runtime_error("transport unavailable"); + } + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json::object()); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_retry", "/pd_it_retry"}}); + + EXPECT_THROW(det->tick(ctx), std::runtime_error); // the plugin guard swallows this in production + for (int i = 0; i < 8; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(attempts, 2) << "the detector did not retry the transport after the first failure"; + ASSERT_NE(fake, nullptr) << "no transport was ever built, so the detector has no reader"; + EXPECT_GT(fake->calls(), 0) << "the reader never ran after the retry"; +} + +// The gateway must never read its own parameters: the transport spins a hidden node of its own, so +// reading the gateway would have it interrogate itself and can raise drift against the plugin. The +// skip is one line and nothing covered it. +TEST_F(ParamDriftIntegrationTest, TheGatewaysOwnNodeIsNeverRead) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json::object()); + auto ctx = make_ctx(DetectorMode::Raise); + + // Only the gateway itself: no read may happen. + set_apps({{"pd_it_gateway", gateway_->get_fully_qualified_name()}}); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_NE(fake, nullptr); + EXPECT_EQ(fake->calls(), 0) << "the detector read the gateway's own parameters"; + + // Positive control: zero above must mean "skipped the gateway", not "read nothing at all". Any + // mutation that empties the armed set would also give zero, so an ordinary app has to produce + // reads through the very same path. + set_apps({{"pd_it_other", "/pd_it_other"}}); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_GT(fake->calls(), 0) << "no app was ever read, so the zero above proves nothing"; +} + +// A config warning that never reaches the log is no warning at all. Six unit tests verify that the +// warnings vector gets populated and none of them proves it is ever surfaced, so deleting the +// logging call changed nothing. Here the detector is handed a misspelt key and the tick must reach +// the logger - asserted through the rcutils log handler rather than by reading stderr. +TEST_F(ParamDriftIntegrationTest, ConfigWarningsReachTheLogger) { + const LogCapture log; + + auto det = make_param_drift(); + ScriptedTransport * fake = nullptr; + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + const nlohmann::json typo{{"ignore_globs", nlohmann::json::array({"*_stamp"})}}; // misspelt on purpose + det->configure(typo); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_warn", "/pd_it_warn"}}); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + ASSERT_NE(fake, nullptr); + + EXPECT_EQ(log.count("ignore_globs"), 1) + << "expected the typo logged exactly once across ten ticks; got " << log.count("ignore_globs") + << " (0 means it never reached the log, more than 1 means the once-guard is gone and every " + "tick repeats it forever)"; + + // The once-guard has to be RE-ARMED by configure(). The plugin re-delivers configuration to a + // live detector, and without the reset an operator who fixes one key and mis-spells another is + // told nothing at all about the second one - the warning is latched for the rest of the run. + det->configure(typo); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + EXPECT_EQ(log.count("ignore_globs"), 2) + << "the same typo delivered by a second configure() was never logged again, so the " + "once-per-run latch is never released and re-configuration is silent"; +} + +// A pin naming a parameter no node declares is answered NOT_FOUND on every read, and NOT_FOUND is +// deliberately not a drift - a node is not at fault for a parameter it never had. The cost is that +// a misspelt pin is inert forever, and from outside inert is indistinguishable from a pin that is +// checked and passes every time. That is the silent failure this detector exists to rule out, so +// it has to say something. Exactly once, and only about the pin that is actually unmatched. +TEST_F(ParamDriftIntegrationTest, AnExpectPinNoAppDeclaresIsReportedOnce) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // Two pins: one the node has, one misspelt. The good pin is the control - it must stay silent, + // or the warning is just firing on every pin and proves nothing about the typo. + det->configure(nlohmann::json{{"baseline", false}, {"expect", {{"gain", 1.0}, {"gian", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_typo", "/pd_it_typo"}}); + det->tick(ctx); + ASSERT_NE(fake, nullptr); + fake->set_declared({"gain"}); + + for (int i = 0; i < 80 && log.count("no app declares 'gian'") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(log.count("no app declares 'gian'"), 1) << "the misspelt pin was never reported"; + + // Keep ticking: the report is once per pin, not once per tick. + for (int i = 0; i < 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(log.count("no app declares 'gian'"), 1) << "the once-guard is gone and every tick repeats the warning"; + EXPECT_EQ(log.count("no app declares 'gain'"), 0) << "the pin the node actually declares was reported as unmatched"; +} + +// The other side of the same rule, and the one that costs a real alarm if it is wrong. DDS +// discovery drops a node from a single poll routinely, with no restart involved - WarmupTracker +// documents exactly that and absorbs such a gap. If a one-sweep gap re-baselined, the node would +// come back and its CURRENT, still drifted value would become the new reference: the confirmed +// fault heals while the parameter is wrong, and can never fire again because the baseline now IS +// the wrong value. Nothing in the log would say so. +TEST_F(ParamDriftIntegrationTest, ChurnMustNotLaunderARaisedDrift) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + const std::vector> present{{"pd_it_churn", "/pd_it_churn"}}; + set_apps(present); + for (int i = 0; i < 6; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_NE(fake, nullptr); + + // A genuine drift, confirmed up before we disturb anything. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(9.0); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // One dropped poll. The node never restarted and its parameter is still 9.0. + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + set_apps({}); + det->tick(ctx); + std::this_thread::sleep_for(50ms); + set_apps(present); + for (int i = 0; i < 14; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "a single dropped discovery poll re-baselined the node on its drifted value, so the fault " + "healed while the parameter is still wrong and can never fire again"; + + // Positive control: fixing it for real still clears, so the zero above is not a dead detector. + fake->set_value(1.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)); +} + +// The withheld clear must not become a permanent latch. Its purpose is the case where nothing is +// known yet: the round-robin sweep has not reached this app. A node that never answers AT ALL - no +// parameter service, a wedged executor - is a permanent property, and there is no per-node opt-out +// here, so without a bound one such node keeps a fixed drift reported as CONFIRMED for the whole +// lifetime of the gateway. Past the bound the app is declared unmeasurable, logged, and stops +// blocking. +// +// The bound is in FAILED ATTEMPTS, not in elapsed ticks: this app has been read sixty-one times and +// answered none of them, which is evidence, where "sixty ticks went by" is not - the sweep may +// simply not have got here. That is the whole distinction, and it is why the escape hatch survives +// the rule that an app the sweep has never attempted blocks the clear indefinitely. The budget is +// set so the sixty-first attempt lands inside a second rather than at the default pacing. +TEST_F(ParamDriftIntegrationTest, APermanentlyUnreadableAppStopsBlockingTheClear) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + // The ghost is marked unreadable inside the factory, before the transport is ever handed to the + // reader. Marking it after the first tick is a race the reader can win, and if it does the ghost + // has a cached read for the rest of the run, never joins the unread set, and never blocks + // anything - the test would pass while proving nothing. + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_ghost2"); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 20}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + // The real app is served by the stand-in; the ghost has no node at all, so no read for it can + // ever succeed. Both are armed every tick. + set_apps({{"pd_it_real", "/pd_it_real"}, {"pd_it_ghost2", "/pd_it_ghost2"}}); + det->tick(ctx); // builds the transport and hands the reader both targets + ASSERT_NE(fake, nullptr); + // Wait for EVIDENCE that the real app's baseline exists rather than sleeping a fixed time. Only + // /pd_it_real reaches the stand-in's call counter (the ghost fails before it), so a second + // counted read means the first one has already been published to the cache, and the tick after + // it turns that into the baseline. A fixed window here is a race against the sweep's own pacing: + // the ghost's failed visit is charged too, so how long the real app waits for its turn is a + // function of the budget, and losing that race makes the drifted value the baseline instead - + // after which no drift exists at all and the rest of the test is unreachable. + ASSERT_TRUE(fake->wait_until_reading(2)) << "the reader never completed a read of /pd_it_real"; + det->tick(ctx); + + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(9.0); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "the drift never raised, so there is nothing whose clearing this test can observe"; + + // Fix it. The ghost is still silent and always will be. + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + fake->set_value(1.0); + bool cleared = false; + for (int i = 0; i < 120 && !cleared; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + cleared = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED) > passed_before; + } + EXPECT_TRUE(cleared) << "one permanently unreadable app blocked the clear forever, so a drift that " + "was fixed stays CONFIRMED for the lifetime of the gateway"; +} + +// Restarting a node is the documented way to accept a new configuration, so a node that was gone +// long enough to have restarted must come back to a fresh baseline rather than being compared +// against values from before the change. +// +// "Long enough" is deliberately the window WarmupTracker absorbs, not a single sweep: that header +// documents that DDS discovery drops a node from one poll routinely, with no restart involved, and +// re-capturing on such a gap would take the node's current - possibly already drifted - values as +// the new reference, healing a real fault that can then never fire again. +// +// Here the app is absent for three sweeps and returns with a different value. Nothing may be +// reported: that value is the new reference. +TEST_F(ParamDriftIntegrationTest, ARestartLengthAbsenceReBaselinesInsteadOfReportingDrift) { + set_apps({{"pd_it_target", "/pd_it_target"}}); + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_target")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear arrived, so the app was never read and there is no pre-restart baseline whose " + "being forgotten this test could observe"; + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for this test to mean anything"; + + // Absent for longer than the churn window, i.e. long enough to be a restart. + set_apps({}); + for (int i = 0; i < 4; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + + // It comes back carrying the operator's new value. + target_->set_parameter(rclcpp::Parameter("speed_limit", 5.0)); + set_apps({{"pd_it_target", "/pd_it_target"}}); + const auto failed_before_return = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "the returning app was never re-read, so the silence below is the silence of a detector " + "that measured nothing"; + for (int i = 0; i < 12; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), failed_before_return) + << "reported drift against the baseline from before the restart, so the README's recovery " + "path (restart the node to re-baseline) does not work"; + + // POSITIVE CONTROL. Every assertion above this line is an all-zeros one, and a detector that + // reads nothing satisfies them exactly as well as one whose forget() works. Moving off the value + // the node came back with must report - which is only possible if that value really was captured + // as the new reference. + const auto failed_before_drift = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + target_->set_parameter(rclcpp::Parameter("speed_limit", 0.25)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before_drift, *det, ctx)) + << "no drift off the re-captured baseline, so the silence above proves nothing at all"; +} + +// A clear says "nothing is drifting anywhere". The detector may only say that about apps it +// actually read. Here one app is real and healthy while a second app is in the snapshot with an +// FQN that answers nothing, so no read for it can ever succeed. Emitting PASSED in that state +// would report health that was never measured - and because the README tells operators to enable +// healing, those PASSED events are what would walk a genuine, still-present drift to HEALED. +// +// The healthy app is what makes this a real discriminator: without the guard, its clean reads +// leave drifted_ empty and the detector cheerfully clears every single tick. +TEST_F(ParamDriftIntegrationTest, NoClearWhileAnArmedAppWasNeverRead) { + set_apps({{"pd_it_target", "/pd_it_target"}, {"pd_it_ghost", "/pd_it_ghost"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_target")); // the real one is readable + for (int i = 0; i < 12; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), 0u) + << "cleared GRAPH_PARAM_DRIFT while /pd_it_ghost had never been read - that is health " + "the detector never measured"; + // And nothing was invented either: no drift exists, so no raise should have happened. + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); +} + +// An app that has been given up on must STAY given up on. The blocking set is recomputed from +// scratch every tick while the give-up is one-shot, so any bound that moves with the shape of the +// graph silently re-admits an app already logged as "no longer holding back the clear": the graph +// grows, the bound grows past the app's count, and every clear from then on is suppressed with +// nothing in the log to say why. A count of that app's OWN failed attempts cannot do that - it +// depends on nothing but the app, and it only ever grows while the app stays unread. +// +// The armed set is grown and then shrunk around a given-up ghost, and a clear is required to arrive +// on both sides. Forty extra apps at eight round trips per 10 ms tick period is a 105 ms rotation, +// so the grown graph is fully read inside a fraction of the polls below - while a bound scaled by +// the armed set would need 60 x 42 failures, which at one ghost retry per rotation is minutes away. +TEST_F(ParamDriftIntegrationTest, AGivenUpAppDoesNotReBlockWhenTheArmedSetGrowsOrShrinks) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_grow_ghost"); // before the reader can ever answer for it + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + const std::vector> small{{"pd_it_grow_real", "/pd_it_grow_real"}, + {"pd_it_grow_ghost", "/pd_it_grow_ghost"}}; + set_apps(small); + // The first clear IS the give-up: nothing drifts, so the only thing that can be holding the clear + // back is the ghost, and the only thing that can release it is having failed on it often enough. + bool gave_up = false; + for (int i = 0; i < 400 && !gave_up; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + gave_up = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED) > 0; + } + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(gave_up) << "the ghost never stopped blocking, so there is no give-up to re-block"; + + // GROW. Forty apps arrive at once; each of them legitimately blocks the clear until it is read, + // and the ghost must not join them again on the way past. + std::vector> grown = small; + for (int i = 0; i < 40; ++i) { + const std::string id = "pd_it_grow" + std::to_string(100 + i); + grown.emplace_back(id, "/" + id); + } + set_apps(grown); + auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + bool cleared_grown = false; + for (int i = 0; i < 150 && !cleared_grown; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + cleared_grown = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED) > passed_before; + } + EXPECT_TRUE(cleared_grown) << "no clear once the grown graph had been read: an app already given up " + "on was re-admitted to the blocking set because the graph got bigger, " + "and nothing in the log says so"; + + // SHRINK back. Every remaining app is either read or given up on, so the very next ticks clear. + set_apps(small); + passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + bool cleared_small = false; + for (int i = 0; i < 40 && !cleared_small; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + cleared_small = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED) > passed_before; + } + EXPECT_TRUE(cleared_small) << "the ghost started blocking again when the graph shrank back"; +} + +// The stale-read fence has to be keyed on the ARMING EPISODE, not on the app. A read runs with the +// lock released and can take seconds; inside that window the app's lifecycle gate can close and +// re-open, and tick() rewrites the target list every tick, so an identity check on the app id - +// or on the fqn, which is just as stable - lets a value sampled while the node was between +// episodes land in the cache. That value then becomes what the node is measured against: a drift +// that is not real and that no later read can heal, which is the exact failure the fence exists +// for. A token issued once per episode and never reused is what tells the two apart. +// +// Only reachable through the stand-in: a live read returns in milliseconds and nothing in a real +// graph holds one open across a de-arm and a re-arm. +TEST_F(ParamDriftIntegrationTest, AReadFromAnInterruptedArmingEpisodeIsNotPublished) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // A 500 ms tick period against eight round trips is a 62.5 ms slot, so a baseline visit owes + // 125 ms: after the held read is released there is a comfortable margin before the reader could + // come back round to the fenced app. + det->configure(nlohmann::json{{"tick_interval_ms", 500}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + const std::string kEpoch = "/pd_it_epoch"; + const std::string kKeep = "/pd_it_epochkeep"; + set_apps({{"pd_it_epoch", kEpoch}, {"pd_it_epochkeep", kKeep}}); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "both apps must be read and clean before the fence can be tested"; + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for the rest of this test to mean anything"; + + // Hold a read of the fenced app open and arrange for it to answer 9.0 - value_for() is sampled + // after the gate, so that is what the held call returns when it is finally released. Marking the + // app unreadable at the same time only affects LATER reads (the held one is already past that + // check), so nothing can overwrite whatever this one publishes, if it publishes. + fake->block_node(kEpoch); + ASSERT_TRUE(fake->wait_until_parked()) << "no read of the fenced app was ever held"; + fake->set_value(kEpoch, 9.0); + fake->set_unreadable(kEpoch); + + // De-arm and re-arm inside that one read window. One tick out and one tick back in: far short of + // the prune, so the baseline (1.0) is untouched, and the app id and the fqn are both unchanged - + // an identity check on either sees nothing at all happen here. + set_apps({{"pd_it_epochkeep", kKeep}}); + det->tick(ctx); + set_apps({{"pd_it_epoch", kEpoch}, {"pd_it_epochkeep", kKeep}}); + det->tick(ctx); + + const int calls_before_release = fake->calls(); + fake->release_node(kEpoch); + // A read of the OTHER app is the next counted one - an unreadable node fails before the counter - + // so this means the released read has finished and either published or been fenced out. + ASSERT_TRUE(fake->wait_until_reading(calls_before_release + 1)) + << "the reader never got past the released read, so nothing was published either way"; + + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "a value sampled while the app was between arming episodes was published as its reading, " + "so the node is now measured against something it held only while it was not a target"; + + // Positive control: the raise path is alive, so the zero above is the fence and not a dead + // detector. The other app has been armed continuously throughout and drifting it must report. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(kKeep, 9.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "no drift was reported for an app that plainly drifted, so nothing above proves anything"; +} + +// Forgetting a vanished node's baseline must be reachable at EVERY accepted prune_grace. The two +// horizons are independent - the baseline goes after a fixed three missed sweeps, the finding after +// prune_grace of them - and dropping the absence counter is what makes any later cleanup +// unreachable. With prune_grace 0 or 1, both inside the documented range, the counter was dropped +// before it could ever reach the forget, so the forget never ran for any app in any run: the +// baselines map grew without bound under node-name churn and every returning node was measured +// against values from before it left. +// +// Each case runs the same story - baseline, vanish, return holding a DIFFERENT value - and the +// return must be silent, because a forgotten baseline makes that value the new reference. The +// endpoints of the range (0 and 3600), the value below the churn window (1), the detector's own +// fallback (2), what the plugin injects when an operator sets nothing (60), and two values outside +// the range, which must be rejected, must warn, and must leave the fallback behaviour intact. +TEST_F(ParamDriftIntegrationTest, BaselineForgettingIsReachableAtEveryAcceptedPruneGrace) { + const LogCapture log; + + const std::vector> cases{{"0", 0}, {"1", 1}, {"2", 2}, {"60", 60}, + {"3600", 3600}, {"4000", 4000}, {"-1", -1}}; + const int out_of_range = 2; // the last two cases; both must warn and both must keep the fallback + + int seq = 0; + for (const auto & [label, value] : cases) { + SCOPED_TRACE("prune_grace=" + label); + ++seq; + const std::string id = "pd_it_pg" + std::to_string(seq); + const std::string fqn = "/" + id; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}, {"prune_grace", value}}); + auto ctx = make_ctx(DetectorMode::Raise); + + // Baseline captured at the stand-in's default of 1.0. The clear proves the app was read. + set_apps({{id, fqn}}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the app was never read, so it has no baseline to forget"; + ASSERT_NE(fake, nullptr); + + // Absent for longer than either horizon can need: the churn window absorbs two missed sweeps + // and the forget runs on the third, while the shortest configurable grace drops the counter on + // the first. + set_apps({}); + for (int i = 0; i < 6; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + + // It returns holding a value it never held before. With the baseline forgotten that value IS + // the new reference; with it retained, 1.0 against 5.0 is a drift the node can never shake off. + fake->set_value(fqn, 5.0); + set_apps({{id, fqn}}); + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + for (int i = 0; i < 40; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), failed_before) + << "the returning node was reported as drifted against a baseline from before it left, so " + "the forget never ran at this prune_grace"; + + // Positive control: the baseline it came back with is a real one, so moving off it still + // reports. Without this the case above is satisfied by a detector that captured nothing. + fake->set_value(fqn, 9.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "no drift off the re-captured baseline, so the silence above proves nothing"; + } + + EXPECT_EQ(log.count("'prune_grace' must be an integer"), out_of_range) + << "an out-of-range prune_grace was accepted or rejected in silence; an operator has no other " + "way to find out the value they wrote is not the one in force"; +} + +// Every per-app map in this detector is keyed by node name, and node names are not a bounded set: +// discovery has no hidden-node filter, so each `ros2` CLI invocation appears once under a name +// nothing will ever use again. A map no prune path reaches therefore grows for the life of the +// gateway process, and a leak shows up in no fault, no log line and no description - which is why +// this asserts on a count rather than on a behaviour. +// +// The churn here is the real shape: a short-lived node that is present for a single sweep, is gate- +// denied for all of it (it has not been present long enough to arm), and then is gone. That path +// creates the frozen-hold counter and nothing on it ever created anything else, so the frozen-hold +// counter was the one map prune_vanished did not clean. +TEST_F(ParamDriftIntegrationTest, PerAppBookkeepingStaysBoundedUnderNodeNameChurn) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}, {"prune_grace", 2}}); + auto ctx = make_ctx(DetectorMode::Raise); + + IntrospectionInput snap; + ReliabilityGate gate(3, gateway_.get(), &node_mutex_); + ctx.gate = &gate; + ctx.snapshot = &snap; + + const auto seed = [&snap](const std::vector & ids) { + snap.apps.clear(); + for (const auto & id : ids) { + App a; + a.id = id; + a.bound_fqn = "/" + id; + snap.apps.push_back(a); + } + }; + + uint64_t tick_no = 0; + const std::string stable = "pd_it_churn_stable"; + for (int i = 0; i < 200; ++i) { + // One long-lived app plus one that exists for exactly this sweep, under a name never reused. + seed({stable, "ros2_cli_" + std::to_string(i)}); + gate.update(snap, ++tick_no); + det->tick(ctx); + std::this_thread::sleep_for(2ms); + } + ASSERT_NE(fake, nullptr); + + // Let the last few transients age past the prune horizon. + seed({stable}); + for (int i = 0; i < 10; ++i) { + gate.update(snap, ++tick_no); + det->tick(ctx); + std::this_thread::sleep_for(2ms); + } + + const std::size_t tracked = det->tracked_count_for_test(); + EXPECT_GE(tracked, 1u) << "the detector is tracking nothing at all, so the bound below is vacuous"; + EXPECT_LE(tracked, 4u) << "still holding per-app bookkeeping for " << tracked + << " identities after 200 one-sweep nodes came and went: a map keyed by node " + "name is not being pruned, and it will grow for the life of the process"; +} + +// A single app's share of the aggregated description is capped, and that cap is what makes going +// second worth anything at all. Without it, one node drifting on a dozen parameters spends the +// whole 480-character budget by itself and every other affected app is cut - whatever the ordering +// says, because there is no room left to order anything into. +// +// Three apps, each drifting on twelve long-valued parameters. Uncapped, the first alone renders to +// roughly 1900 characters and the other two never appear. Capped at 150 each, all three fit inside +// the budget with room to spare, so naming all three is exactly the per-app cap doing its job. +// +// The stand-in is what makes the sizing exact: a live node's parameter set is not fully under the +// test's control, and this assertion is about character counts. +TEST_F(ParamDriftIntegrationTest, OneAppsShareOfTheDescriptionIsCapped) { + const std::string clean_value(60, 'a'); + const std::string drifted_value(60, 'b'); + const std::vector ids{"pd_it_cap1", "pd_it_cap2", "pd_it_cap3"}; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake, &ids, &clean_value](rclcpp::Node *) { + auto t = std::make_unique(); + // Installed in the FACTORY: a drifted first read would simply become the baseline. + for (const auto & id : ids) { + std::map clean; + for (int p = 0; p < 12; ++p) { + clean.emplace("p" + std::to_string(p), clean_value); + } + t->set_params("/" + id, clean); + } + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 100}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + apps.reserve(ids.size()); + for (const auto & id : ids) { + apps.emplace_back(id, "/" + id); + } + set_apps(apps); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) << "not every app was read, so nothing has a baseline yet"; + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for the cap assertion below to mean anything"; + + for (const auto & id : ids) { + std::map drifted; + for (int p = 0; p < 12; ++p) { + drifted.emplace("p" + std::to_string(p), drifted_value); + } + fake->set_params("/" + id, drifted); + } + + ASSERT_TRUE(poll_until( + [this, &ids]() { + return latest_failed_desc_contains("graph_watchdog", ids); + }, + *det, ctx)) + << "the description names " << latest_failed_desc("graph_watchdog").size() + << " characters' worth of one or two apps and not all three: a single app is spending more " + "than its share of the budget and the others are being cut - " + << latest_failed_desc("graph_watchdog"); + + // Belt and braces on the sizing: the whole point is that three capped apps FIT, so the aggregate + // must be under its own cap here. If it were at the cap the three names above could be a + // coincidence of truncation rather than the per-app budget working. + EXPECT_LE(latest_failed_desc("graph_watchdog").size(), AggregatedFault::kMaxDescriptionChars); +} + +// The description is capped, so ORDER decides what an operator actually reads. Four apps that sort +// early and drift on many parameters fill the cap between them; a node that starts drifting +// afterwards sorts last and would never appear, even though it is the one thing that just changed. +// +// Four noisy apps at the per-app cap of 150 characters overrun the 480-character budget, which is +// what makes this a real trade rather than a hypothetical one - and the assertion below refuses to +// run unless the description really is at its cap. `pd_it_zzz` sorts after all of them, so in plain +// map order it is precisely what falls off the end; ordering newly affected apps first puts it at +// the FRONT of the report in which it first appears, which is where it survives. +TEST_F(ParamDriftIntegrationTest, ANewlyDriftedAppSurvivesTheDescriptionCap) { + const std::string clean_value(60, 'a'); + const std::string drifted_value(60, 'b'); + const std::string kLate = "pd_it_zzz"; // sorts after every pd_it_ordN + std::vector noisy_ids; + for (int n = 1; n <= 4; ++n) { + noisy_ids.push_back("pd_it_ord" + std::to_string(n)); + } + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake, &noisy_ids, &kLate, &clean_value](rclcpp::Node *) { + auto t = std::make_unique(); + for (const auto & id : noisy_ids) { + std::map clean; + for (int p = 0; p < 12; ++p) { + clean.emplace("p" + std::to_string(p), clean_value); + } + t->set_params("/" + id, clean); + } + t->set_params("/" + kLate, {{"gain", 1.0}}); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 100}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + apps.reserve(noisy_ids.size() + 1); + for (const auto & id : noisy_ids) { + apps.emplace_back(id, "/" + id); + } + apps.emplace_back(kLate, "/" + kLate); + set_apps(apps); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) << "not every app was read, so nothing has a baseline yet"; + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent, otherwise the ordering assertions below prove nothing"; + + // The four early-sorting apps drift first and fill the description between them. + for (const auto & id : noisy_ids) { + std::map drifted; + for (int p = 0; p < 12; ++p) { + drifted.emplace("p" + std::to_string(p), drifted_value); + } + fake->set_params("/" + id, drifted); + } + ASSERT_TRUE(poll_until( + [this]() { + return latest_failed_desc("graph_watchdog").size() >= AggregatedFault::kMaxDescriptionChars; + }, + *det, ctx)) + << "the description never filled up, so nothing was competing for room and the ordering could " + "not have mattered either way: " + << latest_failed_desc("graph_watchdog"); + ASSERT_EQ(latest_failed_desc("graph_watchdog").find(kLate), std::string::npos) + << "the app that has not drifted yet is already in the description, so its appearance below " + "would say nothing: " + << latest_failed_desc("graph_watchdog"); + + // Now the late app drifts. The report in which it first appears must lead with it: that is the + // whole of the newly-affected-first rule, and it is the only thing that can keep it in a + // description four other apps have already overrun. Every poll below reads the CURRENT record. + fake->set_params("/" + kLate, {{"gain", 9.0}}); + EXPECT_TRUE(poll_until( + [this, &kLate]() { + return latest_failed_desc("graph_watchdog").rfind(kLate + ":", 0) == 0; + }, + *det, ctx)) + << "the app that just started drifting was cut from the description to make room for apps " + "that merely sort earlier and had already been reported: " + << latest_failed_desc("graph_watchdog"); +} + +// The other half of the ordering, and the one that decides what an operator reads when the +// description is full: a violation of a pin the operator WROTE outranks drift the detector +// captured by itself. The two are not equally interesting. A self-captured entry says a value moved +// and the detector has no idea whether that matters; an `expect` entry says a value an operator +// declared is wrong. Cutting the declared one to make room for four apps' worth of the other is the +// wrong trade, and the cap makes it a real trade rather than a hypothetical one. +// +// Nothing exercised this. Every `expect` configuration in this file also sets `baseline: false`, so +// a pinned violation and self-captured drift never coexisted in a single run and the pinned buckets +// of emit_order() were never reached with anything in them. With `baseline: true` the pins are +// checked against the same list_parameters() the capture uses, so one read produces both kinds - +// which is what makes the combination reachable at all. +// +// The setup is sized so the ordering is the only thing that can decide the outcome. Four noisy apps +// drift on six parameters each, which overruns the detector's per-app share of the description, so +// each contributes exactly that share - and four of them are needed to overflow the cap, not three: +// three capped entries and their separators are 150 + 2 + 150 + 2 + 150 = 454 characters against a +// budget of 480, and it is the fourth that pushes it over. The pinned app sorts LAST of the five, +// so in plain map order it is precisely what falls off the end. And the +// assertion is taken only after every app has been reported at least once, so the separate "newly +// affected first" rule (ANewlyDriftedAppSurvivesTheDescriptionCap) cannot be what keeps it in. +TEST_F(ParamDriftIntegrationTest, AnOperatorPinnedViolationOutranksSelfCapturedDrift) { + const std::string kPin = "pinned_gain"; + const std::string kPinnedApp = "pd_ord_zzz"; // sorts after every pd_ord_noisyN + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake, &kPin](rclcpp::Node *) { + auto t = std::make_unique(); + // Installed in the FACTORY so the very first read of every noisy node is the clean one. A + // drifted first read would simply become that node's baseline and nothing would ever be + // reported, leaving the pinned app alone in a description with room to spare. + for (int n = 1; n <= 4; ++n) { + std::map clean; + for (int p = 0; p < 6; ++p) { + clean.emplace("p" + std::to_string(p), 1.0); + } + t->set_params("/pd_ord_noisy" + std::to_string(n), clean); + } + // The pinned app violates its pin from the first read and can carry nothing else: a pinned + // parameter is excluded from the capture set, so its whole contribution is the one entry an + // operator declared. + t->set_params("/pd_ord_zzz", {{kPin, 9.0}}); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // baseline stays true - that is the whole point - and the budget only keeps the sweep brisk. + det->configure(nlohmann::json{{"tick_interval_ms", 100}, {"max_reads_per_tick", 8}, {"expect", {{kPin, 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + for (int n = 1; n <= 4; ++n) { + const std::string id = "pd_ord_noisy" + std::to_string(n); + apps.emplace_back(id, "/" + id); + } + apps.emplace_back(kPinnedApp, "/" + kPinnedApp); + set_apps(apps); + + det->tick(ctx); // builds the transport and hands the reader all five targets + ASSERT_NE(fake, nullptr); + // The sixth read STARTING means the first five have completed and been published - the reader is + // a single thread round-robinning five targets - so the tick after it captures a clean baseline + // for every noisy app. + ASSERT_TRUE(fake->wait_until_reading(6)) << "the reader never got round all five apps"; + det->tick(ctx); + + // Now every noisy app drifts on all six of its parameters. + for (int n = 1; n <= 4; ++n) { + std::map drifted; + for (int p = 0; p < 6; ++p) { + drifted.emplace("p" + std::to_string(p), 9.0); + } + fake->set_params("/pd_ord_noisy" + std::to_string(n), drifted); + } + ASSERT_TRUE(poll_until( + [this]() { + return latest_failed_desc("graph_watchdog").size() >= AggregatedFault::kMaxDescriptionChars; + }, + *det, ctx)) + << "the description never filled up, so nothing was competing for room and the ordering " + "could not have mattered either way"; + + // Keep ticking well past that point: several more full rotations, so every app has been reported + // at least once and none of them is "newly affected" any more. + for (int i = 0; i < 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + + const std::string latest = latest_failed_desc("graph_watchdog"); + ASSERT_GE(latest.size(), AggregatedFault::kMaxDescriptionChars) + << "the description is no longer at its cap, so the assertions below prove nothing: " << latest; + ASSERT_NE(latest.find("pd_ord_noisy1"), std::string::npos) + << "no self-captured drift is in the description at all, so the pinned entry had the cap to " + "itself and its presence says nothing about priority: " + << latest; + EXPECT_NE(latest.find(kPinnedApp + ":\"" + kPin + "\" expected="), std::string::npos) + << "the violation of a pin an operator WROTE was cut from the description an operator reads, " + "to make room for self-captured drift on apps that merely sort earlier: " + << latest; + EXPECT_EQ(latest.rfind(kPinnedApp + ":", 0), std::size_t{0}) + << "the pinned violation is in the description but not at the front of it, so it survived on " + "where its app happens to sort rather than on being pinned: " + << latest; +} + +// Aggregation: two nodes drift -> ONE GRAPH_PARAM_DRIFT (source "graph_watchdog") whose +// description names BOTH apps; revert one -> the aggregate stays raised (the other still +// drifts, no clear); revert both -> the aggregate clears. +TEST_F(ParamDriftIntegrationTest, MultiNodeAggregationOneFaultNamesBothThenClears) { + auto a1 = std::make_shared("pd_it_agg1"); + a1->declare_parameter("alpha", 1.0); + auto a2 = std::make_shared("pd_it_agg2"); + a2->declare_parameter("beta", 2.0); + exec_.add_node(a1); + exec_.add_node(a2); + const auto nodes_guard = on_scope_exit([this, a1, a2] { + exec_.remove_node(a1); + exec_.remove_node(a2); + }); + set_apps({{"pd_it_agg1", "/pd_it_agg1"}, {"pd_it_agg2", "/pd_it_agg2"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_agg1")); + ASSERT_TRUE(wait_param_service("/pd_it_agg2")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear arrived, so at least one of the two apps was never read and has no baseline"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); + + // Drift both -> a single aggregated FAILED naming both apps. Asserted on the CURRENT record, not + // on the history: what an operator reads is the description the store last overwrote. + a1->set_parameter(rclcpp::Parameter("alpha", 0.1)); + a2->set_parameter(rclcpp::Parameter("beta", 0.2)); + EXPECT_TRUE(poll_until( + [this]() { + return latest_failed_desc_contains("graph_watchdog", {"pd_it_agg1", "pd_it_agg2"}); + }, + *det, ctx)) + << "one aggregated fault naming both apps never appeared: " << latest_failed_desc("graph_watchdog"); + + // Revert ONE: the aggregate must STAY raised, because agg2 still drifts. + // + // The snapshot for that is taken only after the revert has SETTLED - long enough for agg1's + // clean re-read to have landed and dropped it from the finding set. Counting raises from before + // that point is the hole in "some raise exists": every raise emitted while agg1 was still in the + // set satisfies it, so a detector that goes quiet the moment the set shrinks passes. + a1->set_parameter(rclcpp::Parameter("alpha", 1.0)); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + const auto failed_settled = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_settled, *det, ctx)) + << "no raise arrived once the reverted node had dropped out of the finding set, so the " + "aggregate went quiet while pd_it_agg2 is still drifting: " + << latest_failed_desc("graph_watchdog"); + EXPECT_TRUE(latest_failed_desc_contains("graph_watchdog", {"pd_it_agg2"})) + << "the current description no longer names the app that is still drifting: " + << latest_failed_desc("graph_watchdog"); + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "reverting one node cleared the aggregate while another still drifts"; + + // Revert BOTH -> the aggregate clears. + const auto passed_before2 = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + a2->set_parameter(rclcpp::Parameter("beta", 2.0)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before2, *det, ctx)); +} + +// Guards the atomic expect-read fix. Config: baseline:false, 3 expect pins, max_reads_per_tick:6, +// two expect nodes each costing 3 pins x 3 round trips = 9. One node violates its LAST (sorted) pin +// z_bad from the start. Because expect is a std::map (keys sorted a_ok, b_ok, z_bad), a budget that +// truncated a visit at 6 would stop after b_ok, drop the drift from `observed`, and spuriously +// CLEAR a real, unfixed fault. The atomic read fetches the whole expect set for a visited node +// regardless of budget, so the drift stays raised and never flaps. Assert the aggregated FAILED +// stays raised with NO interleaved PASSED clear. +// +// The 250 ms tick is what keeps the assertion meaningful and the test honest at the same time. A +// visit owes 9 slots of read_gap(250, 6) = 41.7 ms, i.e. 375 ms, so a two-node rotation is 750 ms: +// the absence loop below spans more than two full rotations, and a bud1 visit that truncated its +// expect set has several chances to clear inside it. It also keeps the retry cadence short enough +// that a first read lost to discovery still recovers well inside poll_for_new's 13 s bound, which +// was sized to outlast the transport's 10 s negative cache. +TEST_F(ParamDriftIntegrationTest, ExpectBudgetTruncationDoesNotFlap) { + auto bud1 = std::make_shared("pd_it_bud1"); + bud1->declare_parameter("a_ok", 1.0); + bud1->declare_parameter("b_ok", 2.0); + bud1->declare_parameter("z_bad", 9.0); // violates the LAST sorted pin from the start + auto bud2 = std::make_shared("pd_it_bud2"); + bud2->declare_parameter("a_ok", 1.0); + bud2->declare_parameter("b_ok", 2.0); + bud2->declare_parameter("z_bad", 3.0); // all pins satisfied + exec_.add_node(bud1); + exec_.add_node(bud2); + const auto nodes_guard = on_scope_exit([this, bud1, bud2] { + exec_.remove_node(bud1); + exec_.remove_node(bud2); + }); + set_apps({{"pd_it_bud1", "/pd_it_bud1"}, {"pd_it_bud2", "/pd_it_bud2"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json{{"baseline", false}, + {"tick_interval_ms", 250}, + {"max_reads_per_tick", 6}, + {"expect", {{"a_ok", 1.0}, {"b_ok", 2.0}, {"z_bad", 3.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_bud1")); + ASSERT_TRUE(wait_param_service("/pd_it_bud2")); + + // Establish the raised state (tolerates initial discovery misses). Once bud1's z_bad drift is + // observed it stays in drifted_ (a failed read never erases it), so the clear count freezes. + const auto failed0 = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed0, *det, ctx)); + const auto passed_after_raise = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + const auto failed_after_raise = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + + // Tick across several full round-robin rotations. A visit owes 375 ms and there are two nodes, + // so a rotation is 750 ms and this 2 s loop covers more than two of them - bud1 is re-read + // several times inside it. The clear count must NOT advance: z_bad is caught atomically on every + // bud1 visit. + for (int i = 0; i < 40; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_after_raise) + << "spurious clear at rotation tick " << i << " - expect set truncated by a per-call budget"; + } + EXPECT_GT(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), failed_after_raise); +} + +// The other side of the same rule, in the OTHER read mode. A visit in `expect` mode makes one +// get_parameter per pin and any of them can come back with something that is not NOT_FOUND - a +// wedged executor, parameter services torn down mid-sweep. Publishing what was collected up to that +// point would be reporting a partial view as a complete one: evaluate() finds nothing wrong in a +// view that is simply missing the parameter that is wrong, and the tick clears a fault that is +// still true. NOT_FOUND is the one failure that is deliberately carried on with, because a node is +// not at fault for a parameter it never declared. +// +// No live node can be asked to answer a get with SERVICE_UNAVAILABLE on demand, which is why this +// needs the stand-in. +TEST_F(ParamDriftIntegrationTest, AFailedExpectReadDoesNotClearARaisedDrift) { + const std::string kFqn = "/pd_it_expfail"; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 20}, {"max_reads_per_tick", 8}, {"expect", {{"gain", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_expfail", kFqn}}); + + // The stand-in answers 9.0, so the pin is violated from the first read. + det->tick(ctx); + ASSERT_NE(fake, nullptr); + fake->set_value(kFqn, 9.0); + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "the pinned violation never raised, so there is no fault whose clearing this test can watch"; + + // Every read of the pin now fails with SERVICE_UNAVAILABLE. The parameter is still 9.0. + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + fake->set_unreadable(kFqn); + for (int i = 0; i < 40; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "a failed expect read was published as a complete view of the node, and the pinned " + "violation it does not contain was cleared while the parameter is still wrong"; + + // Positive control: a read that succeeds AND satisfies the pin does clear it, so the silence + // above is the guard and not a detector that has stopped emitting. + fake->set_value(kFqn, 1.0); + fake->set_readable(kFqn); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the fault never cleared even once the pin was satisfied on a readable node"; +} + +// tick_interval_ms is injected by the plugin and is half of what paces the reader: read_gap divides +// the TICK PERIOD by the budget, so a detector that took the budget and ignored the period would +// pace every deployment at its own default no matter what an operator configured - and the coverage +// latency the README tells them to compute would be wrong by whatever factor they had changed the +// tick by. Nothing asserted on the key directly. +// +// Two runs differing ONLY in tick_interval_ms. At a budget of eight, one slot is period/8 and a +// baseline visit owes two of them, so sixteen visits after the first owe 4 x period: ~1.6 s at a +// 400 ms period and ~0.4 s at a 100 ms one. Both bands exclude the ignored-key behaviour, which +// paces both runs at the 1000 ms default and takes ~4 s. +TEST_F(ParamDriftIntegrationTest, TheInjectedTickIntervalPacesTheReader) { + const auto time_seventeen_visits = [this](int tick_interval_ms) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + EXPECT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", tick_interval_ms}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_pace", "/pd_it_pace"}}); + + // One tick arms the reader; it round-robins its single target on its own from there, so the + // elapsed time is pure pacing rather than the tick loop's cadence. + const auto started = std::chrono::steady_clock::now(); + det->tick(ctx); + EXPECT_NE(fake, nullptr); + EXPECT_TRUE(fake != nullptr && fake->wait_until_reading(17)) + << "the reader never issued seventeen visits at a " << tick_interval_ms << " ms tick period"; + return std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count(); + }; + + const auto slow_ms = time_seventeen_visits(400); + EXPECT_GT(slow_ms, 1200) << "seventeen visits at a 400 ms tick period took " << slow_ms + << " ms: the reader is pacing faster than the configured period allows"; + EXPECT_LT(slow_ms, 2400) << "seventeen visits at a 400 ms tick period took " << slow_ms + << " ms, which is the default 1000 ms period - the injected value was ignored"; + + const auto fast_ms = time_seventeen_visits(100); + EXPECT_LT(fast_ms, 800) << "seventeen visits at a 100 ms tick period took " << fast_ms + << " ms: shortening the tick period did not shorten the reader's slot, so " + "the injected value is not reaching read_gap"; + EXPECT_GT(fast_ms, 200) << "seventeen visits at a 100 ms tick period took " << fast_ms + << " ms: the reader is not honouring the budget at all"; +} + +// ================================================================================================ +// Evidence for a clear, continued: the two states in which the detector knows NOTHING about an app +// and previously said "nothing is drifting anywhere" anyway. +// ================================================================================================ + +// A gate-denied app never reaches the armed set, so it never reaches the unread set and never joins +// the blocking one either. With nothing armed at all - which is the state of EVERY app for the +// first `warmup_cycles` ticks of every gateway or plugin start - the finding set is empty and the +// blocking set is empty, and that pair is exactly what emit_aggregated reads as "nothing is +// drifting anywhere". clear_fault carries no gate of its own (by design: a fault must always be +// clearable), so the clear goes out. With `healing_enabled`, which this package's README tells +// operators to switch on, those clears walk a persisted CONFIRMED GRAPH_PARAM_DRIFT toward HEALED +// before the detector has issued a single parameter round trip. +// +// The evidence rule does not care WHY an app has no usable read. "The round-robin has not reached +// it" and "the gate has not armed it" are the same state: nothing is known about the app, so +// nothing may be said about it. +TEST_F(ParamDriftIntegrationTest, NoClearWhileTheGateHasNotArmedAnyApp) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}}); + + IntrospectionInput snap; + { + App a; + a.id = "pd_it_warmup"; + a.bound_fqn = "/pd_it_warmup"; + snap.apps.push_back(a); + } + // A REAL gate, fed exactly one snapshot. warmup_cycles is the shipped default of 5 and one tick + // has elapsed, so the app is KNOWN and NOT armed - the state every app is in immediately after a + // restart, reached here without having to drive five seconds of wall clock. + ReliabilityGate gate(5, gateway_.get(), &node_mutex_); + gate.update(snap, 1); + ASSERT_FALSE(reliability_allows(&gate, "pd_it_warmup")); + + auto ctx = make_ctx(DetectorMode::Raise); + ctx.gate = &gate; + ctx.snapshot = &snap; + + for (int i = 0; i < 40; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + std::this_thread::sleep_for(300ms); // reports are service round trips; let the in-flight ones land + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), 0u) + << "cleared GRAPH_PARAM_DRIFT while the only app in the graph was still warming up and had " + "never been read once - health the detector never measured, emitted on every tick of " + "every restart"; + + // Positive control: the guard releases the moment the app arms and is actually read. Without it + // the zero above is satisfied just as well by a detector that emits nothing at all. + gate.update(snap, 20); + ASSERT_TRUE(reliability_allows(&gate, "pd_it_warmup")); + ASSERT_NE(fake, nullptr); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "no clear even once the app had armed and been read, so the guard never releases"; +} + +// The other state: an app that answered ONCE and then stopped for good. Its cached read keeps it in +// the measured set forever - it never joins the unread set, so it never blocks a clear and neither +// unread warning can ever fire about it. The detector then reports "nothing is drifting anywhere" +// off a snapshot of a moment that has passed, indefinitely, and says nothing at all about the fact. +// +// A cached read is evidence about the instant it was taken, not a standing licence. The moment a +// read of an app FAILS, what is cached about it stops being a measurement of the app's current +// state, and the detector is back to knowing nothing. +TEST_F(ParamDriftIntegrationTest, AnAppThatAnsweredOnceAndThenStoppedIsNoLongerCountedAsMeasured) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // The default tick period against the default budget: one slot is 125 ms and a baseline visit, + // successful or failed, is charged two of them - so a failed attempt lands every 250 ms and the + // tenth arrives inside three seconds, well short of the sixty-first that ends the blocking. + det->configure(nlohmann::json{{"tick_interval_ms", 1000}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_onceonly", "/pd_it_onceonly"}}); + + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "the app was never read, so it was never 'measured' and there is nothing here to go stale"; + ASSERT_NE(fake, nullptr); + + // And now it stops answering, permanently. Nothing about its parameters is known from here on. + // + // The baseline for the clear count is taken once a read has actually FAILED and the detector has + // seen it, not the instant the stand-in is reconfigured: a clear emitted while the last completed + // read was still a success is a clear made on evidence that was current, and this test is about + // what happens afterwards. + fake->set_unreadable("/pd_it_onceonly"); + ASSERT_TRUE(fake->wait_until_failed(1)) << "no read of the silenced app ever failed"; + det->tick(ctx); + std::this_thread::sleep_for(200ms); // let a report issued before that tick land + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < 90 && log.count("consecutive parameter reads of 'pd_it_onceonly'") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + EXPECT_EQ(log.count("consecutive parameter reads of 'pd_it_onceonly'"), 1) + << "ten consecutive reads of the app failed and nothing was said about it, because a cached " + "read from before the node went silent keeps it out of the unread set entirely"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "kept clearing GRAPH_PARAM_DRIFT off a cached read taken before the node stopped " + "answering, so a drift introduced since then is reported as health"; + + // Positive control: the app is not written off, it is simply unmeasured. Answering again puts it + // straight back into the measured set and the clear flows. + fake->set_readable("/pd_it_onceonly"); + const auto passed_again = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_again, *det, ctx)) + << "the app answered again and the clear never came back, so the guard latches instead of " + "tracking what is currently known"; +} + +// Counting the give-up in FAILED ATTEMPTS rather than elapsed ticks is right - a tick horizon +// expires on exactly the large graphs where the rotation is slowest - but attempts alone are not a +// duration. The reader paces itself at `max_reads_per_tick` round trips per tick period, so an +// attempt costs read_gap x its charge in wall clock, and the whole horizon scales with the load +// knob. At the accepted ceiling of 100000 the gap floors at one microsecond, a failed baseline +// visit owes two of them, and sixty-one attempts are spent in a couple of milliseconds: the very +// first ticks of a run write off any node whose parameter service DDS has not finished discovering +// yet. The README's own remedy for a slow sweep - raise max_reads_per_tick - is what collapses it. +// +// So the horizon needs a floor that the budget cannot move. Ticks are that floor: they are the one +// quantity the load knob does not scale. Used as a CONJUNCTION with the attempt count it can only +// ever delay the give-up, never bring it forward, so it does not reintroduce the failure mode that +// counting attempts was adopted to fix. +TEST_F(ParamDriftIntegrationTest, GivingUpOnAnAppNeedsElapsedTicksAsWellAsFailedAttempts) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_slowstart"); // before the reader can ever answer for it + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 1000}, {"max_reads_per_tick", 100000}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_slowstart", "/pd_it_slowstart"}}); + + det->tick(ctx); // builds the transport and hands the reader its single target + ASSERT_NE(fake, nullptr); + // Far more failed attempts than the horizon counts, inside a handful of ticks. This is the whole + // point: at this budget the ATTEMPT count is exhausted before the run has properly begun. + ASSERT_TRUE(fake->wait_until_failed(300)) << "the reader never got anywhere near the attempt horizon at " + "the ceiling budget, so this test is not measuring what it claims"; + for (int i = 0; i < 4; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(log.count("giving up on"), 0) + << "wrote the app off as unmeasurable within five ticks of the run starting, purely because " + "the load knob was raised: a node that is merely slow to advertise its parameter services " + "is declared unmeasurable before DDS has finished discovering it"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), 0u) + << "cleared GRAPH_PARAM_DRIFT on the strength of that write-off"; + + // Positive control: the give-up is floored, not removed. Keep ticking and it arrives - otherwise + // one node with no parameter service holds the fault CONFIRMED for the life of the gateway. + for (int i = 0; i < 300 && log.count("giving up on") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(2ms); + } + EXPECT_EQ(log.count("giving up on"), 1) << "the app never stopped blocking the clear, so the floor became a latch"; +} + +// An app id whose BINDING moves to a different node is a different node, whatever the id says. The +// reader already treats it that way - a target's arming token is keyed on (app_id, fqn), so a read +// in flight from the old binding can never publish into the new one - but nothing resets the state +// the old binding left behind: the captured baseline, the cached read, the failure count. The new +// node's first reading is then compared against values a DIFFERENT node happened to hold, which is +// a drift no later read can heal, because the reference itself belongs to somewhere else. +TEST_F(ParamDriftIntegrationTest, AnFqnReBindStartsFromAFreshBaseline) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_value("/pd_it_bind_a", 1.0); + t->set_value("/pd_it_bind_b", 5.0); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 20}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + set_apps({{"pd_it_bind", "/pd_it_bind_a"}}); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "the app was never read under its first binding, so there is no stale baseline to carry over"; + ASSERT_NE(fake, nullptr); + + // The same app id, now bound to a DIFFERENT node holding a different value. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + set_apps({{"pd_it_bind", "/pd_it_bind_b"}}); + for (int i = 0; i < 30; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), failed_before) + << "reported the new node's value as drift against a baseline captured from a DIFFERENT node: " + << latest_failed_desc("graph_watchdog"); + + // Positive control: the new node's value really was captured as the reference, so moving off it + // still reports. Without this the silence above is satisfied by a detector that captured nothing. + const auto failed_before_drift = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value("/pd_it_bind_b", 9.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before_drift, *det, ctx)) + << "no drift off the re-captured baseline, so the silence above proves nothing at all"; +} + +// The unmatched-pin warning waits for the sweep to cover the graph, which raises the question of +// whether it can still fire on a graph big enough to take many ticks to cover, and on one where +// short-lived nodes keep arriving. Both are answered here: sixty apps, and a run of one-sweep +// transients on top of them. The warning has to arrive in both. +// +// Sixty apps at eight round trips per 10 ms tick period is a 1.25 ms slot; an undeclared pin is +// charged two, so a rotation is 150 ms and the loop below spans many of them. The transients are +// the shape a real graph churns in - one `ros2` CLI invocation, present for a sweep under a name +// nothing will use again - and each of them legitimately holds the warning back while it is there. +// What must not happen is that they hold it back for good. +TEST_F(ParamDriftIntegrationTest, AnUnmatchedPinIsStillReportedOnALargeAndChurningGraph) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_declared(std::set{"gain"}); // no node declares the misspelt pin + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 10}, {"max_reads_per_tick", 8}, {"expect", {{"gian", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + for (int i = 0; i < 60; ++i) { + const std::string id = "pd_it_wide2_" + std::to_string(100 + i); // uniform width -> sorted == creation order + apps.emplace_back(id, "/" + id); + } + + // Churn phase: every tick one short-lived app arrives under a name never reused, on top of the + // sixty that stay. Nothing may be reported while such an app has not been looked at. + for (int i = 0; i < 60; ++i) { + auto churned = apps; + churned.emplace_back("pd_it_cli_" + std::to_string(i), "/pd_it_cli_" + std::to_string(i)); + set_apps(churned); + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + ASSERT_NE(fake, nullptr); + + // The churn stops, the sixty stay. The sweep can now finish, and the misspelt pin must be named. + set_apps(apps); + for (int i = 0; i < 300 && log.count("no app declares 'gian'") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + EXPECT_EQ(log.count("no app declares 'gian'"), 1) + << "the misspelt pin was never named on a sixty-app graph that had just been churning, so the " + "warning is dead on exactly the graphs a silent pin is easiest to miss on"; + EXPECT_EQ(log.count("no app declares 'gain'"), 0) << "the pin every node declares was reported as unmatched"; +} + +// ================================================================================================ +// Pins for behaviour that a mutation of the detector's own constants used to leave green. +// ================================================================================================ + +// The per-read timeout is a BOUND, and a bound is only real if a read that overruns it is +// abandoned. Nothing here could see that: tick() does no IO, so its duration says nothing, and the +// unresponsive-node test only requires the sweep to get PAST a node that never answers at all - +// which it does whether the bound is half a second or half an hour. +// +// The discriminator is a node that answers LATE rather than never. Its parameter service takes +// 1.5 s to reply, which is past the 0.5 s per-read bound and inside anything meaningfully larger. +// At the real bound the read is abandoned and the node is never read, so it can never appear in a +// description; raise the bound and the same reply lands and it does. The counter on the service +// handler is what keeps that absence honest: it proves the read was actually issued and the node +// actually served it late, rather than the transport never having found the service at all. +TEST_F(ParamDriftIntegrationTest, AReadThatOverrunsThePerReadBoundIsAbandoned) { + const std::string kSlow = "pd_it_to_slow"; // sorts after pd_it_to_ok: read second in the rotation + const std::string kOk = "pd_it_to_ok"; + std::atomic slow_calls{0}; + + auto ok = std::make_shared(kOk); + ok->declare_parameter("speed_limit", 3.5); // violates the pin below from the first read + exec_.add_node(ok); + const auto ok_guard = on_scope_exit([this, ok] { + exec_.remove_node(ok); + }); + + // The late node. list_parameters is the round trip an `expect` read makes first, so holding it is + // what makes the whole read overrun; get_parameters exists so discovery finds a complete node. + auto slow = std::make_shared(kSlow, rclcpp::NodeOptions().start_parameter_services(false)); + auto slow_list = slow->create_service( + "/" + kSlow + "/list_parameters", service_callback( + [&slow_calls](const rcl_interfaces::srv::ListParameters::Request & /*req*/, + rcl_interfaces::srv::ListParameters::Response & resp) { + ++slow_calls; + std::this_thread::sleep_for(1500ms); + resp.result.names = {"speed_limit"}; + })); + auto slow_get = slow->create_service( + "/" + kSlow + "/get_parameters", + service_callback([](const rcl_interfaces::srv::GetParameters::Request & req, + rcl_interfaces::srv::GetParameters::Response & resp) { + for (std::size_t i = 0; i < req.names.size(); ++i) { + rcl_interfaces::msg::ParameterValue v; + v.type = rcl_interfaces::msg::ParameterType::PARAMETER_DOUBLE; + v.double_value = 7.5; // would violate the pin, if it were ever read + resp.values.push_back(v); + } + })); + const auto slow_idle = add_idle_parameter_services(slow, "/" + kSlow); + // Its own executor: the 1.5 s sleep owns whatever thread serves it, and blocking the fixture's + // executor would stall the healthy node and the fake fault service with it. + rclcpp::executors::SingleThreadedExecutor slow_exec; + slow_exec.add_node(slow); + std::thread slow_spin([&slow_exec] { + slow_exec.spin(); + }); + const auto slow_guard = on_scope_exit([&slow_exec, &slow_spin] { + slow_exec.cancel(); + if (slow_spin.joinable()) { + slow_spin.join(); + } + }); + + set_apps({{kOk, "/" + kOk}, {kSlow, "/" + kSlow}}); + auto det = make_param_drift(); + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 100}, {"max_reads_per_tick", 8}, {"expect", {{"speed_limit", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/" + kOk)); + ASSERT_TRUE(wait_param_service("/" + kSlow)); + + // Positive control FIRST: the healthy node's pinned violation is reported, which proves the + // transport, the reader and the aggregate are all alive on this graph. + ASSERT_TRUE(poll_until( + [this, &kOk]() { + return latest_failed_desc_contains("graph_watchdog", {kOk, "got=3.5"}); + }, + *det, ctx)) + << "the healthy node's pinned violation never reported, so the absence below proves nothing: " + << latest_failed_desc("graph_watchdog"); + + // And the late node WAS reached: it served the request, it just served it too late. + for (int i = 0; i < 200 && slow_calls.load() == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + ASSERT_GT(slow_calls.load(), 0) << "the late node's parameter service was never called at all, so nothing " + "here exercised the per-read bound"; + + // Give a read that was NOT abandoned every chance to land: the reply is 1.5 s behind the request. + for (int i = 0; i < 40; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + EXPECT_EQ(latest_failed_desc("graph_watchdog").find(kSlow), std::string::npos) + << "a node whose parameter service took 1.5 s to answer was read anyway, so the per-read " + "bound is no longer half a second and one slow node can hold the sweep for as long as it " + "likes: " + << latest_failed_desc("graph_watchdog"); +} + +// The unread warning is worth something only if it means what it says. Firing it at the first +// failed read makes it a bringup notice: one dropped reply on a busy graph is not "this node does +// not answer", and an operator who is told it every time stops reading the log. Both sides of the +// threshold are pinned here, off the ATTEMPT counter rather than off wall clock, because the +// attempt count is the thing under test and pacing it by time would make the assertion a function +// of the budget instead. +TEST_F(ParamDriftIntegrationTest, TheUnreadWarningWaitsForTenFailedAttemptsNotOne) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_tenth"); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // One slot is 125 ms and a failed baseline visit is charged two, so attempts arrive every 250 ms: + // slow enough that the assertion below cannot race the attempt it is asserting about. + det->configure(nlohmann::json{{"tick_interval_ms", 1000}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_tenth", "/pd_it_tenth"}}); + + det->tick(ctx); + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_failed(5)) << "the reader never made five attempts against the silent app"; + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count("consecutive parameter reads of"), 0) + << "named the app after a handful of failed reads: a single dropped reply is not evidence " + "that a node has stopped answering, and a warning that fires on one is noise"; + + for (int i = 0; i < 200 && log.count("consecutive parameter reads of 'pd_it_tenth'") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_EQ(log.count("consecutive parameter reads of 'pd_it_tenth'"), 1) + << "the app that never answers was never named, so there is no threshold here to pin"; + EXPECT_GE(fake->failures(), 10) << "the warning fired after only " << fake->failures() + << " failed attempts, short of the ten it documents"; +} + +// A per-app detail that overruns its share is cut from the MIDDLE, keeping both ends. The head +// names the app and its first drifted parameter; the tail is what stops the entry ending mid-token +// and is the only place the last of a long run of drifted parameters can survive. Head-only +// truncation renders the whole tail unreachable and leaves every existing assertion green, because +// they all look at the front of the entry. +TEST_F(ParamDriftIntegrationTest, AnOverlongPerAppDetailKeepsItsTailAsWellAsItsHead) { + const std::string kApp = "pd_it_tail"; + // Six drifted parameters render to about 230 characters, comfortably past the 150 an app may + // spend, and the last of them carries a name nothing else in the text can produce. + const std::vector names{"p1_aaaa", "p2_bbbb", "p3_cccc", "p4_dddd", "p5_eeee", "zz_lastone"}; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake, &kApp, &names](rclcpp::Node *) { + auto t = std::make_unique(); + std::map clean; + for (const auto & n : names) { + clean.emplace(n, 1.0); + } + t->set_params("/" + kApp, clean); // installed in the factory: the first read must be the clean one + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 100}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{kApp, "/" + kApp}}); + + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) << "the app was never read, so it has no baseline to drift from"; + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for the assertions below to mean anything"; + + std::map drifted; + for (const auto & n : names) { + drifted.emplace(n, 9.0); + } + fake->set_params("/" + kApp, drifted); + ASSERT_TRUE(poll_until( + [this]() { + return latest_failed_desc("graph_watchdog").find("...") != std::string::npos; + }, + *det, ctx)) + << "the entry never overran its share, so there was nothing to trim and this test would pass " + "under any trimming rule at all: " + << latest_failed_desc("graph_watchdog"); + + const std::string desc = latest_failed_desc("graph_watchdog"); + EXPECT_EQ(desc.rfind(kApp + ":\"p1_aaaa\"", 0), std::size_t{0}) + << "the head no longer names the app and its first drifted parameter: " << desc; + EXPECT_NE(desc.find("zz_lastone"), std::string::npos) + << "the trimmed entry keeps only its head, so the last of a long run of drifted parameters is " + "unreachable however many of them there are: " + << desc; + EXPECT_EQ(desc.find("p3_cccc"), std::string::npos) + << "nothing was actually cut, so 'both ends' is not what is being tested here: " << desc; +} + +// A graph on which every armed app has been GIVEN UP on is swept in name only. Nothing answered, so +// nothing is known about which parameters exist anywhere, and a pin cannot be called unmatched on +// the strength of it - that is a warning telling an operator to go and fix a name which may well be +// correct, produced by a detector that read nothing. +TEST_F(ParamDriftIntegrationTest, AnUnmatchedPinIsNotReportedWhenEveryAppWasGivenUpOn) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_allmute"); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 10}, {"max_reads_per_tick", 8}, {"expect", {{"gian", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_allmute", "/pd_it_allmute"}}); + + // Tick well past the point the only app is given up on: from there the sweep is "complete" in the + // sense the clear uses, and nothing but the answered-something guard stands between the pin and a + // warning about it. + for (int i = 0; i < 400 && log.count("giving up on") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_NE(fake, nullptr); + ASSERT_EQ(log.count("giving up on"), 1) << "the app was never given up on, so the swept-in-name-only state this " + "test is about was never reached"; + for (int i = 0; i < 60; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + EXPECT_EQ(log.count("no app declares 'gian'"), 0) + << "called the pin unmatched on a graph where not one app ever answered a read, which says " + "nothing whatever about which parameters the graph declares"; + + // Positive control: once an app does answer and still does not declare the pin, the warning is + // due and must arrive. Otherwise the zero above is a detector that never reports anything. + fake->set_declared(std::set{"gain"}); + fake->set_readable("/pd_it_allmute"); + for (int i = 0; i < 300 && log.count("no app declares 'gian'") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + EXPECT_EQ(log.count("no app declares 'gian'"), 1) + << "the pin was never reported even once an app had answered and did not declare it"; +} + +// The unread warnings are one-shot per RUN OF FAILURES, not once per process. An app that goes +// quiet, comes back, and goes quiet again has failed twice, and the second time is worth saying: +// the operator who fixed it needs to know it has broken again. What makes that possible is erasing +// the latch when the app leaves the unread set - and leaving the latch behind is invisible, because +// silence is what a working latch looks like too. +TEST_F(ParamDriftIntegrationTest, TheUnreadWarningFiresAgainAfterTheAppRecoversAndFailsOnceMore) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_relapse"); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 20}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_relapse", "/pd_it_relapse"}}); + + const std::string needle = "consecutive parameter reads of 'pd_it_relapse'"; + for (int i = 0; i < 400 && log.count(needle) == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_NE(fake, nullptr); + ASSERT_EQ(log.count(needle), 1) << "the app that never answered was never named the first time"; + + // It recovers. The reader reads it, the failure count is dropped, and so must the latch be. + fake->set_readable("/pd_it_relapse"); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the recovered app was never read again, so nothing could have reset its latch either way"; + + // And it breaks again. + fake->set_unreadable("/pd_it_relapse"); + for (int i = 0; i < 400 && log.count(needle) < 2; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + EXPECT_EQ(log.count(needle), 2) + << "the app broke a second time and nothing was said, because the one-shot latch from the " + "first run of failures was never erased - so from the first failure onward this app is " + "silent for the life of the gateway"; +} + +// The description's ordering has four buckets and the cross combination is what decides a real +// report: a pinned violation that has already been reported against self-captured drift that is +// brand new. Priority runs pinned-before-self-captured FIRST and newly-affected-before-reported +// only WITHIN a bucket, so the pinned entry leads even though the other one is the thing that just +// changed - an operator who declared a value cares that it is still wrong more than about a value +// the detector itself decided to watch. +// +// The two rules were only ever exercised one at a time, so swapping the middle two buckets left the +// whole suite green. The assertion is taken on the FIRST report the new app appears in, because +// that is the only report in which the two orderings differ at all. +TEST_F(ParamDriftIntegrationTest, AReportedPinnedViolationStillOutranksBrandNewSelfCapturedDrift) { + const std::string kPin = "pinned_gain"; + const std::string kPinned = "pd_ord_x_pinned"; // sorts AFTER the self-captured app + const std::string kSelf = "pd_ord_a_self"; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake, &kPin, &kPinned, &kSelf](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_params("/" + kPinned, {{kPin, 9.0}}); // violates its pin from the very first read + t->set_params("/" + kSelf, {{"gain", 1.0}}); // clean, so its first read is a silent capture + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 50}, {"max_reads_per_tick", 8}, {"expect", {{kPin, 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{kSelf, "/" + kSelf}, {kPinned, "/" + kPinned}}); + + // The pinned violation is reported on its own for a while: by the time the other app drifts it is + // firmly in the already-reported bucket, which is the whole point of the combination. + ASSERT_TRUE(poll_until( + [this, &kPinned]() { + return latest_failed_desc_contains("graph_watchdog", {kPinned}); + }, + *det, ctx)) + << "the pinned violation never reported, so there is no reported-pinned entry to rank"; + ASSERT_NE(fake, nullptr); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_EQ(latest_failed_desc("graph_watchdog").find(kSelf), std::string::npos) + << "the self-capturing app is in the description already, so its arrival below says nothing: " + << latest_failed_desc("graph_watchdog"); + + // Now the self-capturing app drifts. It is brand new to the report and it sorts first, so plain + // map order and newly-affected-first both put it ahead; only pinned-first keeps the pin in front. + fake->set_params("/" + kSelf, {{"gain", 9.0}}); + ASSERT_TRUE(poll_until( + [this, &kSelf]() { + return !first_failed_desc_containing("graph_watchdog", kSelf).empty(); + }, + *det, ctx)) + << "the self-captured drift was never reported at all"; + const std::string first = first_failed_desc_containing("graph_watchdog", kSelf); + EXPECT_EQ(first.rfind(kPinned + ":", 0), std::size_t{0}) + << "brand-new self-captured drift was put ahead of a violation of a pin an operator wrote, so " + "on a full description the declared value is the one that gets cut: " + << first; +} + +// The captured baseline is dropped on the THIRD consecutive missed sweep, and both ends of that +// matter. Dropping it sooner re-captures a node on its already-drifted value after an ordinary +// dropped discovery poll - a confirmed fault heals while the parameter is still wrong, and can +// never fire again because the new baseline IS the wrong value. Dropping it later means a restart, +// which is the documented way to accept a new configuration, keeps reporting the operator's own +// change as drift. +// +// Existing coverage only pinned the short side. `prune_grace` is set well above the horizon here so +// the FINDING prune cannot reach it: the baseline forget is the only thing that can be under test. +TEST_F(ParamDriftIntegrationTest, TheBaselineForgetHorizonIsExactlyThreeMissedSweeps) { + struct Case { + int absences; + bool expect_drift; + const char * why; + }; + const std::vector cases{ + {2, true, + "two missed sweeps re-baselined the node, so one dropped discovery poll away from that " + "launders a real drift into the new reference"}, + {3, false, + "three missed sweeps did NOT re-baseline the node, so a restart - the documented way to " + "apply a new configuration - reports the operator's own change as drift"}, + }; + + int seq = 0; + for (const auto & [absences, expect_drift, why] : cases) { + SCOPED_TRACE("absences=" + std::to_string(absences)); + ++seq; + const std::string id = "pd_it_forget" + std::to_string(seq); + const std::string fqn = "/" + id; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // prune_grace far above the absence counts below, so `dropping` is never true and the finding + // prune cannot stand in for the baseline forget. + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}, {"prune_grace", 60}}); + auto ctx = make_ctx(DetectorMode::Raise); + + set_apps({{id, fqn}}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the app was never read, so it has no baseline whose fate this case could observe"; + ASSERT_NE(fake, nullptr); + + // Exactly `absences` sweeps without it. prune_vanished counts one missed sweep per tick. + set_apps({}); + for (int i = 0; i < absences; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + + // It comes back holding a value it never held while it was present. + fake->set_value(fqn, 5.0); + set_apps({{id, fqn}}); + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + bool drifted = false; + for (int i = 0; i < 60 && !drifted; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + drifted = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED) > failed_before; + } + EXPECT_EQ(drifted, expect_drift) << why << ": " << latest_failed_desc("graph_watchdog"); + } +} + +// Withholding a clear is correct and, from outside the process, completely invisible. No fault is +// raised, no fault is cleared, and a GRAPH_PARAM_DRIFT already in the store simply never heals - +// which is exactly what a detector that is working and finding nothing looks like. Worse, the apps +// doing the holding are usually ones the per-app unread warning may NOT name: nothing has been +// attempted against them, because they are waiting their turn or the gate has not armed them, and +// naming those would be a bringup warning about healthy nodes. So the aggregate has to speak for +// itself: once per episode, how long it has been withheld and how many apps are behind it. +// +// 160 apps at one round trip per 200 ms tick period is a rotation of a minute, so the tail of the +// graph stays uncontacted for the whole run. No sleeps: the horizon is counted in TICKS. +TEST_F(ParamDriftIntegrationTest, AWithheldClearSaysSoInTheLog) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 200}, {"max_reads_per_tick", 1}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + for (int i = 0; i < 160; ++i) { + const std::string id = "pd_it_held" + std::to_string(1000 + i); + apps.emplace_back(id, "/" + id); + } + set_apps(apps); + det->tick(ctx); + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_reading(1)) << "the reader never started, so nothing here is under test"; + + const std::string needle = "is being withheld from clearing"; + for (int i = 0; i < 30; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 0) << "announced a withheld clear after thirty ticks: an ordinary rotation on a " + "large graph takes longer than that, so this fires during normal bringup"; + + for (int i = 0; i < 200; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 1) + << "the clear has been withheld for over two hundred ticks and nothing in the log says so, " + "so an operator watching GRAPH_PARAM_DRIFT refuse to heal has no way to tell a graph the " + "sweep cannot cover from a detector that is working normally (got " + << log.count(needle) << " lines)"; + + // The episode ends when the graph is actually covered, and the warning must re-arm with it: a + // one-shot-per-process latch would say nothing at all the next time coverage breaks down. + set_apps({apps.front()}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "no clear even once the graph was one already-read app, so the episode never ended"; + + set_apps(apps); + for (int i = 0; i < 300; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 2) + << "coverage broke down a second time and nothing was said, so the warning is one-shot per " + "process rather than per episode"; +} + +// An `expect` pin on a parameter the node DECLARED and never SET. +// +// Not a corner case and not the NOT_FOUND carve-out. rclcpp lets a node declare a dynamically typed +// parameter with no value, and its parameter service answers that name with a PARAMETER_NOT_SET +// VALUE rather than an absent one - so the transport gets a parameter back and returns SUCCESS +// carrying null. The pin therefore DOES fire, which is right: a pin says the value must be 1.0, and +// a node holding no value at all is not holding 1.0. Nothing asserted that in either direction. +// +// What the fault SAYS has to survive the round trip too. A NaN double dumps as `null` as well, so +// one description said the same thing about "the node never set this parameter" and "the value is +// NaN" - two states with different remedies, told apart by nothing. Both are pinned here, on the +// same node and in the same description, so the text has to distinguish them or this fails. +// +// Deliberately a LIVE node and the real transport, not the scripted stand-in. The premise is a +// reply shape rclcpp produces; a stand-in scripted to produce it would restate the premise instead +// of testing it. +TEST_F(ParamDriftIntegrationTest, AnExpectPinOnADeclaredButUnsetParameterRaisesAndReadsAsUnset) { + auto node = std::make_shared("pd_it_unset"); + rcl_interfaces::msg::ParameterDescriptor dynamic_typing; + dynamic_typing.dynamic_typing = true; // a statically typed declaration demands a value or an override + node->declare_parameter("threshold", rclcpp::ParameterValue{}, dynamic_typing); // declared, never set + node->declare_parameter("headroom", std::numeric_limits::quiet_NaN()); // a value, and it is NaN + exec_.add_node(node); + const auto node_guard = on_scope_exit([this, node] { + exec_.remove_node(node); + }); + set_apps({{"pd_it_unset", "/pd_it_unset"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json{{"baseline", false}, {"expect", {{"threshold", 1.0}, {"headroom", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + ASSERT_TRUE(wait_param_service("/pd_it_unset")); // service up before a read-driven raise is asserted + + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "an expect pin on a parameter the node declared and never set raised nothing at all, so the " + "operator is told the pin is satisfied by a node holding no value for it"; + + const std::string desc = latest_failed_desc("graph_watchdog"); + ASSERT_NE(desc.find("threshold"), std::string::npos) << "the unset parameter is not named: " << desc; + ASSERT_NE(desc.find("headroom"), std::string::npos) << "the NaN parameter is not named: " << desc; + + // Each entry's own `got=` field, cut at the entry separator so the two cannot be confused. + const auto got_of = [&desc](const std::string & param) { + const auto name_at = desc.find('"' + param + '"'); + if (name_at == std::string::npos) { + return std::string{}; + } + const auto got_at = desc.find(" got=", name_at); + if (got_at == std::string::npos) { + return std::string{}; + } + const auto end = desc.find(';', got_at); + return desc.substr(got_at + 5, end == std::string::npos ? std::string::npos : end - got_at - 5); + }; + const std::string unset_got = got_of("threshold"); + const std::string nan_got = got_of("headroom"); + ASSERT_FALSE(unset_got.empty()) << "no got= field for the unset parameter in: " << desc; + ASSERT_FALSE(nan_got.empty()) << "no got= field for the NaN parameter in: " << desc; + + EXPECT_EQ(unset_got, "") << "a parameter the node declared and never set reads as '" << unset_got + << "' in the fault an operator sees: " << desc; + EXPECT_NE(unset_got, nan_got) << "the unset parameter and the NaN one both read as '" << unset_got + << "', so the description cannot tell an operator which fault this is: " << desc; +} diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_policy.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_policy.cpp new file mode 100644 index 000000000..0d2af2d43 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_policy.cpp @@ -0,0 +1,648 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ros2_medkit_graph_watchdog/param_drift_policy.hpp" + +using nlohmann::json; +using ros2_medkit_graph_watchdog::glob_match; +using ros2_medkit_graph_watchdog::is_unset_param; +using ros2_medkit_graph_watchdog::kUnsetParameterRendering; +using ros2_medkit_graph_watchdog::param_entry_value; +using ros2_medkit_graph_watchdog::param_values_equal; +using ros2_medkit_graph_watchdog::ParamDriftConfig; +using ros2_medkit_graph_watchdog::ParamDriftPolicy; +using ros2_medkit_graph_watchdog::read_gap; +using ros2_medkit_graph_watchdog::unset_param_value; +using Params = std::map; + +namespace { +/// The `got=` field of one drifted descriptor - the text an operator reads for the value the node +/// actually holds. Empty when the descriptor has no such field, which fails the assertion rather +/// than quietly comparing two empty strings. +std::string rendered_got(const std::string & descriptor) { + const auto at = descriptor.find(" got="); + return at == std::string::npos ? std::string{} : descriptor.substr(at + 5); +} +} // namespace + +TEST(GlobMatch, StarWildcard) { + EXPECT_TRUE(glob_match("*_stamp", "header_stamp")); + EXPECT_TRUE(glob_match("qos_overrides.*", "qos_overrides./scan.reliability")); + EXPECT_TRUE(glob_match("*", "anything")); + EXPECT_TRUE(glob_match("exact", "exact")); + EXPECT_FALSE(glob_match("*_stamp", "stamp_header")); + EXPECT_FALSE(glob_match("exact", "exacter")); +} + +TEST(ParamValuesEqual, NumericNormalization) { + EXPECT_TRUE(param_values_equal(json(20), json(20.0))); // int vs double, same value + EXPECT_TRUE(param_values_equal(json(1.5), json(1.5))); + EXPECT_FALSE(param_values_equal(json(20.0), json(5.0))); + EXPECT_TRUE(param_values_equal(json(false), json(false))); + EXPECT_FALSE(param_values_equal(json(false), json(true))); + EXPECT_TRUE(param_values_equal(json("a"), json("a"))); + EXPECT_FALSE(param_values_equal(json("a"), json("b"))); +} + +// A NaN-valued param (common "unset/disabled" sentinel) must NOT read as drifted against +// itself - IEEE NaN != NaN would make it a permanent false positive that never clears. +TEST(ParamValuesEqual, NaNParamIsStable) { + const double nan = std::numeric_limits::quiet_NaN(); + EXPECT_TRUE(param_values_equal(json(nan), json(nan))); + EXPECT_FALSE(param_values_equal(json(nan), json(1.0))); + // Signed zero is not a drift. Written as an INTEGER parameter against a pin carrying a decimal + // point, because that is the pairing our own comparison has to decide: two doubles of the same + // kind never reach int_equals_float_exactly at all and only exercise nlohmann's operator==. + // -0.0 is the value that makes the integrality check subtle - it is neither greater nor less + // than zero, so a naive `frac != 0.0` test would have to get the sign of zero right. + EXPECT_TRUE(param_values_equal(json(0), json(-0.0))); + EXPECT_TRUE(param_values_equal(json(-0.0), json(0))); // and the mirrored branch +} + +// int64 values differing beyond 2^53 must be caught: a `double` coercion collapses them into a +// MISSED drift on 64-bit IDs, counters and epoch-ns params. Inside an ARRAY, because that is a +// path of our own - the recursion in param_values_equal - and because a 1-D array of large +// integers against a pin written with decimal points is exactly how the two get mixed. Two plain +// integers of the same kind would fall straight through to nlohmann's operator== and test nothing +// this package owns. +TEST(ParamValuesEqual, LargeInt64ExactCompareInsideAnArray) { + EXPECT_TRUE(param_values_equal(json::array({9007199254740992LL}), json::array({9007199254740992.0}))); + EXPECT_FALSE(param_values_equal(json::array({9007199254740993LL}), json::array({9007199254740992.0}))); + // A late element is the one that gets lost when the recursion stops early. + EXPECT_FALSE( + param_values_equal(json::array({1LL, 2LL, 9007199254740993LL}), json::array({1.0, 2.0, 9007199254740992.0}))); +} + +TEST(ParamValuesEqual, NaNInsideArray) { + const double nan = std::numeric_limits::quiet_NaN(); + EXPECT_TRUE(param_values_equal(json::array({nan, 1.0}), json::array({nan, 1.0}))); // never-changed NaN[] not drift + EXPECT_FALSE(param_values_equal(json::array({nan, 1.0}), json::array({nan, 2.0}))); + EXPECT_FALSE(param_values_equal(json::array({1.0}), json::array({1.0, 2.0}))); // length differs +} + +TEST(ParamDriftConfig, FlattensNestedExpectToDottedParamNames) { + // The gateway un-nests dotted config keys, so a pin on a dotted param name + // (expect.qos_overrides./scan.reliability) arrives NESTED. from_json must flatten it back + // to the full dotted param name - the key the observed param map actually uses. `baseline`/ + // `ignore` are single-level keys and survive extract_detector_config's nested output as-is. + json detector_cfg = { + {"baseline", false}, + {"expect", {{"use_sim_time", false}, {"qos_overrides", {{"/scan", {{"reliability", "best_effort"}}}}}}}, + {"ignore", json::array({"*_stamp"})}}; + const auto cfg = ParamDriftConfig::from_json(detector_cfg); + EXPECT_FALSE(cfg.baseline_capture); + ASSERT_EQ(cfg.expect.count("use_sim_time"), 1u); + EXPECT_EQ(cfg.expect.at("use_sim_time"), json(false)); + ASSERT_EQ(cfg.expect.count("qos_overrides./scan.reliability"), 1u); // recovered from nesting + EXPECT_EQ(cfg.expect.at("qos_overrides./scan.reliability"), json("best_effort")); + ASSERT_EQ(cfg.ignore_globs.size(), 1u); + EXPECT_EQ(cfg.ignore_globs[0], "*_stamp"); +} + +TEST(ParamDriftConfig, MalformedExpectWarnsAndKeepsEmpty) { + // A present-but-non-object expect (operator error) must warn, like the other fields. + const auto cfg = ParamDriftConfig::from_json({{"expect", "use_sim_time"}}); // scalar, not object + EXPECT_TRUE(cfg.expect.empty()); + EXPECT_FALSE(cfg.warnings.empty()); +} + +TEST(ParamDriftConfig, MisspeltKeyWarnsInsteadOfBeingDropped) { + // A wrong TYPE on a known key already warned; a wrong NAME did not, though it fails + // the same way: `ignore_globs` (the struct's field name) leaves the globs empty and + // param_drift keeps flagging exactly the params the operator meant to suppress. + const auto cfg = ParamDriftConfig::from_json({{"ignore_globs", json::array({"*_stamp"})}}); + EXPECT_TRUE(cfg.ignore_globs.empty()); + ASSERT_EQ(cfg.warnings.size(), 1u); + EXPECT_NE(cfg.warnings[0].find("ignore_globs"), std::string::npos); +} + +TEST(ParamDriftConfig, EveryKnownKeyIsAcceptedWithoutWarning) { + // Guards the known-key set against drifting away from the fields from_json reads: + // a key dropped from the set here would start warning on a correct config. + // baseline stays ENABLED here: `ignore` filters only the self-captured set, so pairing it with + // `baseline: false` is a dead combination that now warns on purpose (see the test below). + const auto cfg = ParamDriftConfig::from_json({{"baseline", true}, + {"ignore", json::array({"*_stamp"})}, + {"max_reads_per_tick", 4}, + {"expect", {{"use_sim_time", false}}}}); + EXPECT_TRUE(cfg.warnings.empty()) << (cfg.warnings.empty() ? "" : cfg.warnings.front()); +} + +TEST(ParamDriftConfig, DefaultsWhenEmpty) { + const auto cfg = ParamDriftConfig::from_json(json::object()); + EXPECT_TRUE(cfg.baseline_capture); // default true + EXPECT_TRUE(cfg.expect.empty()); + EXPECT_TRUE(cfg.ignore_globs.empty()); + EXPECT_TRUE(cfg.warnings.empty()); +} + +// Malformed config must not be dropped silently - it must warn (a mistyped `ignore` +// would otherwise re-enable drift on suppressed params with no trail). +TEST(ParamDriftConfig, MalformedIgnoreWarnsAndKeepsEmpty) { + const auto scalar = ParamDriftConfig::from_json({{"ignore", "*_stamp"}}); // scalar, not array + EXPECT_TRUE(scalar.ignore_globs.empty()); + EXPECT_FALSE(scalar.warnings.empty()); + + const auto wrong_elem = ParamDriftConfig::from_json({{"ignore", json::array({1, 2})}}); // int array + EXPECT_TRUE(wrong_elem.ignore_globs.empty()); + EXPECT_FALSE(wrong_elem.warnings.empty()); +} + +TEST(ParamDriftConfig, MalformedBaselineWarnsAndKeepsDefault) { + const auto cfg = ParamDriftConfig::from_json({{"baseline", "false"}}); // quoted string, not bool + EXPECT_TRUE(cfg.baseline_capture); // explicit override lost -> kept default true + EXPECT_FALSE(cfg.warnings.empty()); +} + +TEST(ParamDriftConfig, MaxReadsPerTick) { + EXPECT_EQ(ParamDriftConfig::from_json(json::object()).max_reads_per_tick, 8); // default + EXPECT_EQ(ParamDriftConfig::from_json({{"max_reads_per_tick", 20}}).max_reads_per_tick, 20); + const auto bad = ParamDriftConfig::from_json({{"max_reads_per_tick", 0}}); // non-positive + EXPECT_EQ(bad.max_reads_per_tick, 8); // kept default + EXPECT_FALSE(bad.warnings.empty()); +} + +TEST(ParamDriftPolicy, SelfCaptureFirstSeenNoDrift) { + ParamDriftPolicy p{ParamDriftConfig{}}; // baseline true, no expect + const auto out = + p.evaluate("/n", Params{{"speed", json(1.0)}, {"use_sim_time", json(false)}}, /*capture_allowed=*/true); + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, RuntimeChangeDriftsThenClears) { + ParamDriftPolicy p{ParamDriftConfig{}}; + p.evaluate("/n", Params{{"speed", json(1.0)}}, true); // capture + auto out = p.evaluate("/n", Params{{"speed", json(0.2)}}, true); // changed + ASSERT_EQ(out.size(), 1u); + EXPECT_NE(out[0].find("speed"), std::string::npos); + out = p.evaluate("/n", Params{{"speed", json(1.0)}}, true); // restored + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, NewParamAfterCaptureIsSilent) { + ParamDriftPolicy p{ParamDriftConfig{}}; + p.evaluate("/n", Params{{"a", json(1)}}, true); + const auto out = p.evaluate("/n", Params{{"a", json(1)}, {"b", json(2)}}, true); // b is new + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, IgnoredParamNeverDrifts) { + ParamDriftConfig cfg; + cfg.ignore_globs = {"*_stamp"}; + ParamDriftPolicy p{cfg}; + p.evaluate("/n", Params{{"header_stamp", json(1)}}, true); + const auto out = p.evaluate("/n", Params{{"header_stamp", json(999)}}, true); + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, ExpectRuleFiresRegardlessOfBaseline) { + ParamDriftConfig cfg; + cfg.baseline_capture = false; // no self-capture + cfg.expect = {{"use_sim_time", json(false)}}; + ParamDriftPolicy p{cfg}; + // First observation already violates - no prior "capture" needed. + auto out = p.evaluate("/n", Params{{"use_sim_time", json(true)}}, true); + ASSERT_EQ(out.size(), 1u); + EXPECT_NE(out[0].find("use_sim_time"), std::string::npos); + out = p.evaluate("/n", Params{{"use_sim_time", json(false)}}, true); + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, ExpectAbsentParamIsNotDrift) { + ParamDriftConfig cfg; + cfg.baseline_capture = false; + cfg.expect = {{"use_sim_time", json(false)}}; + ParamDriftPolicy p{cfg}; + const auto out = p.evaluate("/n", Params{{"other", json(1)}}, true); // node lacks use_sim_time + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, BaselineFalseIgnoresUnlistedParams) { + ParamDriftConfig cfg; + cfg.baseline_capture = false; + cfg.expect = {{"use_sim_time", json(false)}}; + ParamDriftPolicy p{cfg}; + p.evaluate("/n", Params{{"use_sim_time", json(false)}, {"speed", json(1.0)}}, true); + const auto out = p.evaluate("/n", Params{{"use_sim_time", json(false)}, {"speed", json(9.0)}}, true); + EXPECT_TRUE(out.empty()); // speed changed but unlisted + no baseline +} + +TEST(ParamDriftPolicy, ExpectPlusBaselineBothProduceDescriptors) { + ParamDriftConfig cfg; + cfg.expect = {{"use_sim_time", json(false)}}; // baseline_capture stays true + ParamDriftPolicy p{cfg}; + p.evaluate("/n", Params{{"use_sim_time", json(false)}, {"speed", json(1.0)}}, true); // capture + const auto out = p.evaluate("/n", Params{{"use_sim_time", json(true)}, {"speed", json(0.2)}}, true); + ASSERT_EQ(out.size(), 2u); // one expect drift + one baseline drift + const std::string joined = out[0] + out[1]; + EXPECT_NE(joined.find("use_sim_time"), std::string::npos); + EXPECT_NE(joined.find("speed"), std::string::npos); +} + +TEST(ParamDriftPolicy, CaptureGatedByCaptureAllowed) { + ParamDriftPolicy p{ParamDriftConfig{}}; + EXPECT_TRUE( + p.evaluate("/n", {{"x", json(1.0)}}, /*capture_allowed=*/false).empty()); // not armed: no capture, no drift + EXPECT_TRUE(p.evaluate("/n", {{"x", json(9.0)}}, /*capture_allowed=*/true) + .empty()); // first armed sweep: captures 9.0, no drift + const auto d = p.evaluate("/n", {{"x", json(1.0)}}, true); // now 1.0 != 9.0 -> drift + ASSERT_EQ(d.size(), 1u); + EXPECT_NE(d[0].find("/n"), std::string::npos); + EXPECT_NE(d[0].find('x'), std::string::npos); +} + +TEST(ParamDriftPolicy, ExpectFiresEvenWhenCaptureNotAllowed) { + ParamDriftConfig cfg; + cfg.baseline_capture = false; + cfg.expect = {{"use_sim_time", json(false)}}; + ParamDriftPolicy p{cfg}; + const auto d = p.evaluate("/n", {{"use_sim_time", json(true)}}, /*capture_allowed=*/false); + ASSERT_EQ(d.size(), 1u); // expect is absolute; independent of capture gating +} + +TEST(ParamDriftPolicy, ForgetDropsBaseline) { + ParamDriftPolicy p{ParamDriftConfig{}}; + p.evaluate("/n", {{"x", json(1.0)}}, true); + p.forget("/n"); + EXPECT_TRUE(p.evaluate("/n", {{"x", json(0.2)}}, true).empty()); // re-captures 0.2, not a drift +} + +// read_gap is the only thing enforcing max_reads_per_tick: the reader takes one target per +// pass and waits out the rest of its slot, so a zero gap is no budget at all. +TEST(ParamDriftPolicy, ReadGapSpreadsTheBudgetOverTheTickPeriod) { + EXPECT_EQ(read_gap(1000, 8), std::chrono::microseconds(125000)); + EXPECT_EQ(read_gap(200, 4), std::chrono::microseconds(50000)); +} + +// A budget larger than the tick period in milliseconds truncates to zero under integer +// millisecond division, which removes the bound instead of tightening it. +TEST(ParamDriftPolicy, ReadGapStaysNonZeroWhenTheBudgetExceedsTheTickInMilliseconds) { + EXPECT_GT(read_gap(1000, 2000).count(), 0); + EXPECT_EQ(read_gap(1000, 2000), std::chrono::microseconds(500)); +} + +// A non-positive budget must not divide by zero; it falls back to one read per period. +TEST(ParamDriftPolicy, ReadGapFallsBackToOneReadPerPeriod) { + EXPECT_EQ(read_gap(1000, 0), std::chrono::microseconds(1000000)); + EXPECT_EQ(read_gap(1000, -5), std::chrono::microseconds(1000000)); +} + +// A fault description is cut at a byte offset, so a rendered value must never contain a +// multi-byte sequence a cut could split, and must never make the renderer throw. +TEST(ParamDriftPolicy, RenderedValuesAreAsciiOnly) { + ParamDriftPolicy p(ParamDriftConfig{}); + p.evaluate("/n", Params{{"path", json("/maps/cafe/map.yaml")}}, true); + const auto out = p.evaluate("/n", Params{{"path", json("/maps/caf\xc3\xa9/map.yaml")}}, true); + ASSERT_EQ(out.size(), 1u); + for (const char c : out[0]) { + EXPECT_LT(static_cast(c), 0x80u) + << "non-ASCII byte in a description that is later cut at a byte offset"; + } +} + +// A parameter whose bytes are not valid UTF-8 must not throw out of evaluate(): the plugin's +// per-detector guard would swallow it and the detector would then neither raise nor clear for as +// long as that value exists, so a confirmed fault could never heal. +TEST(ParamDriftPolicy, InvalidUtf8InAValueDoesNotThrow) { + ParamDriftPolicy p(ParamDriftConfig{}); + // No explicit length: the literal is 10 bytes and hand-counting it got that wrong once, which + // read past the end of the literal and only showed up under AddressSanitizer. + const std::string bad("bad\xff\xfevalue"); + p.evaluate("/n", Params{{"s", json("ok")}}, true); + EXPECT_NO_THROW({ + const auto out = p.evaluate("/n", Params{{"s", json(bad)}}, true); + EXPECT_EQ(out.size(), 1u); + }); +} + +// One array-valued parameter must not be able to consume the whole description budget. +// +// The 400-element array is the `got=` field, so that is what has to be measured. Measuring the text +// between `baseline=` and ` got=` measures the four-element BASELINE, which is a dozen characters +// long whatever elide() does - disabling elision entirely left that form of this test green. +// +// The budget is asserted against a LITERAL and not against the constant itself. Asserting +// `<= kMaxRenderedValueChars` says only that elide() honours whatever number it is handed, so +// widening the budget widens the assertion with it and the description cap this feeds silently +// grows: at 480 a single value fills the whole aggregated description on its own and every other +// drifted entry is cut. That is the same class of defect this test was repaired to catch. +TEST(ParamDriftPolicy, LongValuesAreElided) { + constexpr std::size_t kBudget = 64; // pinned; see TheRenderedValueBudgetIsSixtyFourCharacters + const json big(std::vector(400, 7)); + ParamDriftPolicy p(ParamDriftConfig{}); + p.evaluate("/n", Params{{"arr", json(std::vector(4, 1))}}, true); + const auto out = p.evaluate("/n", Params{{"arr", big}}, true); + ASSERT_EQ(out.size(), 1u); + const auto got_at = out[0].find(" got="); + ASSERT_NE(got_at, std::string::npos); + const std::string rendered_got = out[0].substr(got_at + 5); + EXPECT_LE(rendered_got.size(), kBudget) + << "the changed value rendered to " << rendered_got.size() << " chars, past the per-value budget"; + EXPECT_LT(rendered_got.size(), big.dump().size()) + << "the 400-element array was rendered in full, so elide() is a no-op on the field that " + "actually carries a long value"; +} + +// The budget itself, pinned once with a literal so every other assertion may name the constant. +// +// Seven of these fit inside AggregatedFault's 480-character description and two fit inside the +// 150 characters one app may spend, which is what makes a multi-app, multi-parameter report +// readable at all. Raising it is not a free tuning choice: the description cap does not grow with +// it, so a wider value budget buys one entry room at the direct expense of every entry behind it. +TEST(ParamDriftPolicy, TheRenderedValueBudgetIsSixtyFourCharacters) { + EXPECT_EQ(ParamDriftPolicy::kMaxRenderedValueChars, 64u) + << "the per-value rendering budget moved. That is a change to what an operator can read in a " + "capped description, not an implementation detail: update this pin deliberately, and check " + "how many entries still fit under AggregatedFault::kMaxDescriptionChars when you do"; +} + +// The gap has a floor as well as microsecond resolution: a budget above a million reads per +// second truncates to zero again, and a zero gap is a reader that never waits. +TEST(ParamDriftPolicy, ReadGapNeverReachesZeroEvenForAnAbsurdBudget) { + EXPECT_EQ(read_gap(1000, 2000000), std::chrono::microseconds(1)); + EXPECT_GT(read_gap(1, 100000).count(), 0); +} + +// The reader spends one gap per round trip, so what the budget actually buys the watched node is +// `period / gap` round trips per tick period - and that is NOT `reads` unless the division comes +// out even. Flooring makes the gap shorter than asked and the rate correspondingly higher: +// `tick_interval_ms: 1` with `max_reads_per_tick: 600`, both accepted configurations, floors to a +// 1 us gap and 1000 round trips against a documented bound of 600. +// +// The whole accepted range is swept rather than one pair, because the overshoot only appears when +// the division is uneven and any single hand-picked pair is overwhelmingly likely to divide evenly. +// Both directions are pinned: the rate must never exceed the budget, and the gap must not be +// padded beyond the one microsecond the rounding can add, or the knob would silently throttle. +TEST(ParamDriftPolicy, ReadGapNeverLetsTheAchievableRateExceedTheBudget) { + const std::vector ticks{1, 2, 3, 5, 7, 10, 100, 125, 250, 333, 1000, 5000}; + const std::vector reads{1, 2, 3, 6, 7, 8, 9, 11, 13, 64, 600, 999, 1024, 7777, 100000}; + for (const int tick_ms : ticks) { + for (const int budget : reads) { + const auto gap = read_gap(tick_ms, budget).count(); + const std::int64_t period_us = static_cast(tick_ms) * 1000; + ASSERT_GT(gap, 0) << "tick_interval_ms=" << tick_ms << " max_reads_per_tick=" << budget; + EXPECT_LE(period_us / gap, budget) + << "tick_interval_ms=" << tick_ms << " with max_reads_per_tick=" << budget << " yields a " << gap + << " us gap, i.e. " << period_us / gap + << " round trips per tick period: the knob an operator lowered to protect a node is being " + "overshot"; + // Tightness, so "hold the bound" cannot be met by simply pacing far slower than asked. One + // slot per read plus at most the single microsecond rounding up can add. + EXPECT_LT(gap * budget, period_us + budget) + << "tick_interval_ms=" << tick_ms << " with max_reads_per_tick=" << budget << " spends " << gap + << " us per round trip, well beyond the slot the budget asks for"; + } + } +} + +// The exact configuration the bound used to break on, spelled out so the regression is named. +TEST(ParamDriftPolicy, ReadGapHoldsTheBoundOnASubMillisecondSlot) { + EXPECT_EQ(read_gap(1, 600), std::chrono::microseconds(2)) + << "a 1000 us tick period over 600 reads is 1.67 us per read; rounding down to 1 us buys the " + "sweep 1000 round trips per tick period against a bound of 600"; +} + +// is_number_integer() spans the whole 64-bit range and get() truncates, so the range check +// has to happen on the wide value. Otherwise 4294967297 arrives as 1, and -4294967295 arrives as +// 1 as well - a negative request passing a positive-only guard, both with no warning. +TEST(ParamDriftPolicy, OutOfRangeBudgetIsRejectedNotTruncated) { + for (const auto & bad : {json(4294967297LL), json(-4294967295LL), json(-1), json(0)}) { + const auto cfg = ParamDriftConfig::from_json(json{{"max_reads_per_tick", bad}}); + EXPECT_EQ(cfg.max_reads_per_tick, 8) << "accepted " << bad.dump(); + EXPECT_FALSE(cfg.warnings.empty()) << "accepted " << bad.dump() << " with no warning"; + } + const auto ok = ParamDriftConfig::from_json(json{{"max_reads_per_tick", 32}}); + EXPECT_EQ(ok.max_reads_per_tick, 32); + EXPECT_TRUE(ok.warnings.empty()); + + // The ceiling itself. An off-by-one here either rejects a legal budget or lets the value above + // it through, and neither shows up anywhere else - every other case in this test is orders of + // magnitude away from the bound. + const auto at_ceiling = + ParamDriftConfig::from_json(json{{"max_reads_per_tick", ParamDriftConfig::kMaxReadsPerTickCeiling}}); + EXPECT_EQ(at_ceiling.max_reads_per_tick, ParamDriftConfig::kMaxReadsPerTickCeiling); + EXPECT_TRUE(at_ceiling.warnings.empty()) << "the ceiling value itself must be accepted"; + + const auto past_ceiling = + ParamDriftConfig::from_json(json{{"max_reads_per_tick", ParamDriftConfig::kMaxReadsPerTickCeiling + 1}}); + EXPECT_EQ(past_ceiling.max_reads_per_tick, 8) << "a budget one past the ceiling was accepted"; + EXPECT_FALSE(past_ceiling.warnings.empty()); +} + +// nlohmann compares an integer against a float by casting the integer to double, which collapses +// values differing beyond 2^53 - exactly the missed drift the equality helper claims to avoid. +// Reachable from an expect pin written with a decimal point against an integer parameter. +TEST(ParamDriftPolicy, IntegerVersusFloatIsComparedExactly) { + EXPECT_FALSE(param_values_equal(json(9007199254740993LL), json(9007199254740992.0))); + EXPECT_TRUE(param_values_equal(json(9007199254740992LL), json(9007199254740992.0))); + EXPECT_TRUE(param_values_equal(json(20), json(20.0))); // the ordinary case must stay equal + EXPECT_FALSE(param_values_equal(json(1), json(1.5))); + EXPECT_FALSE(param_values_equal(json(1), json(std::numeric_limits::quiet_NaN()))); + EXPECT_FALSE(param_values_equal(json(1), json(std::numeric_limits::infinity()))); + + // The MIRRORED branch: a float on the left against an integer on the right swaps the operands + // before calling int_equals_float_exactly, and only the int-left form was covered - so the swap + // could have been the wrong way round in one direction and nothing here would have said so. A + // float-left NaN is the ordinary shape of it: a NaN-valued parameter read from a node, compared + // against an integer pin. + EXPECT_FALSE(param_values_equal(json(std::numeric_limits::quiet_NaN()), json(1))); + EXPECT_FALSE(param_values_equal(json(std::numeric_limits::infinity()), json(1))); + EXPECT_FALSE(param_values_equal(json(1.5), json(1))); + EXPECT_TRUE(param_values_equal(json(20.0), json(20))); + EXPECT_FALSE(param_values_equal(json(9007199254740992.0), json(9007199254740993LL))); + + // A negative pin against an unsigned parameter, which is the ordinary shape: the transport + // stores every positive integer unsigned. Converting the double to uint64_t before checking its + // sign is undefined behaviour, and on x86-64 -1.0 lands on UINT64_MAX - so the one parameter + // value that would then compare EQUAL to it is the largest one representable. + EXPECT_FALSE(param_values_equal(json(std::uint64_t{18446744073709551615ULL}), json(-1.0))); + EXPECT_FALSE(param_values_equal(json(std::uint64_t{5}), json(-1.0))); + EXPECT_TRUE(param_values_equal(json(std::uint64_t{5}), json(5.0))); // the branch still works + + // The 2^63 bounds. A double at or past +2^63 has no int64_t to compare against, so it is + // reported as different even where the two are numerically the same - erring towards raising a + // drift rather than missing one. -2^63 IS representable and must still compare equal. + EXPECT_FALSE(param_values_equal(json(std::uint64_t{9223372036854775808ULL}), json(9223372036854775808.0))); + EXPECT_FALSE(param_values_equal(json(1), json(-18446744073709551616.0))); // -2^64 + EXPECT_TRUE(param_values_equal(json(std::numeric_limits::min()), json(-9223372036854775808.0))); +} + +// Configuration that parses cleanly and can never do anything must say so. Silence is the worst +// outcome here: the detector looks configured and reports nothing, forever. +TEST(ParamDriftConfig, InertCombinationsAreReported) { + const auto nothing_to_read = ParamDriftConfig::from_json(json{{"baseline", false}}); + EXPECT_FALSE(nothing_to_read.warnings.empty()) << "baseline off with no expect reads nothing at all"; + + const auto dead_ignore = ParamDriftConfig::from_json( + json{{"baseline", false}, {"expect", {{"use_sim_time", false}}}, {"ignore", {"*_x"}}}); + EXPECT_FALSE(dead_ignore.warnings.empty()) << "ignore cannot filter anything when baseline is off"; + + // And the ordinary configurations stay quiet. + EXPECT_TRUE(ParamDriftConfig::from_json(json::object()).warnings.empty()); + EXPECT_TRUE(ParamDriftConfig::from_json(json{{"baseline", false}, {"expect", {{"a", 1}}}}).warnings.empty()); +} + +// An ignore pattern that matches every parameter name suppresses the whole self-captured half. +// The detector then looks configured and reports nothing, which at runtime is indistinguishable +// from it working and finding nothing - the exact silent failure it is built to rule out. +TEST(ParamDriftConfig, ACatchAllIgnoreIsReported) { + EXPECT_FALSE(ParamDriftConfig::from_json(json{{"ignore", {"*"}}}).warnings.empty()); + EXPECT_FALSE(ParamDriftConfig::from_json(json{{"ignore", {"**"}}}).warnings.empty()) + << "a glob of nothing but stars matches everything too"; + EXPECT_FALSE(ParamDriftConfig::from_json(json{{"ignore", {"debug_*", "*"}}}).warnings.empty()) + << "one catch-all among several patterns still swallows everything"; + + // Pins still report, so this is a different message - but still not silence. + EXPECT_FALSE(ParamDriftConfig::from_json(json{{"ignore", {"*"}}, {"expect", {{"a", 1}}}}).warnings.empty()); + + // An ordinary pattern stays quiet. + EXPECT_TRUE(ParamDriftConfig::from_json(json{{"ignore", {"debug_*"}}}).warnings.empty()); +} + +// A pin written two ways that flatten to the same parameter name loses one of them, and a pin +// with no value at all disappears. Both used to happen with no diagnostic. +TEST(ParamDriftConfig, LostExpectPinsAreReported) { + const auto collide = ParamDriftConfig::from_json(json::parse(R"({"expect":{"a":{"b":1},"a.b":2}})")); + EXPECT_FALSE(collide.warnings.empty()); + EXPECT_EQ(collide.expect.size(), 1u); + + const auto empty_pin = ParamDriftConfig::from_json(json::parse(R"({"expect":{"qos_overrides":{}}})")); + EXPECT_FALSE(empty_pin.warnings.empty()); + EXPECT_TRUE(empty_pin.expect.empty()); +} + +// Parameter NAMES come verbatim from the watched node and are never validated, so they carry the +// same two hazards values do. An invalid UTF-8 byte in the name, or a valid multi-byte name split +// by the aggregated description's byte-offset cut, makes the gateway's JSON writer throw - which +// answers 500 on GET /faults and takes every OTHER fault down with it. +TEST(ParamDriftPolicy, ParameterNamesAreSanitisedToo) { + ParamDriftPolicy p(ParamDriftConfig{}); + const std::string bad_name("na\xff\xfeme", 6); + p.evaluate("/n", Params{{bad_name, json(1.0)}}, true); + const auto out = p.evaluate("/n", Params{{bad_name, json(2.0)}}, true); + ASSERT_EQ(out.size(), 1u); + for (const char c : out[0]) { + EXPECT_LT(static_cast(c), 0x80u) << "non-ASCII byte from a parameter NAME"; + } + EXPECT_NO_THROW(json(out[0]).dump()); +} + +// A parameter a node DECLARED and never SET must not read as `null` in a fault description. +// +// The transport answers such a parameter with a JSON null, and a NaN double DUMPS as `null` too, so +// one description said the same thing for "the node never set this parameter" and "the value is +// NaN" - two different faults with two different remedies, and nothing in the text to tell them +// apart. Both are reachable from an `expect` pin, so both really do end up in front of an operator. +// +// The equality half is asserted here as well, and not assumed: the marker is carried through the +// same map a real value is, so whatever it is has to keep the pin's verdict intact - unequal to the +// pinned value (or the raise this rendering describes would silently disappear) and equal to +// itself (or a parameter that is unset every sweep would drift against its own baseline for ever). +TEST(ParamDriftPolicy, AnUnsetParameterIsRenderedDistinctlyFromANaNValue) { + ParamDriftConfig cfg; + cfg.baseline_capture = false; + cfg.expect = {{"threshold", json(1.0)}}; + ParamDriftPolicy p{cfg}; + + const auto unset = p.evaluate("/unset", Params{{"threshold", unset_param_value()}}, /*capture_allowed=*/false); + ASSERT_EQ(unset.size(), 1u) << "an expect pin on a parameter the node never set must still raise: a node with no " + "value at all does not satisfy a pin that says the value must be 1.0"; + const auto nan_drift = p.evaluate("/nan", Params{{"threshold", json(std::numeric_limits::quiet_NaN())}}, + /*capture_allowed=*/false); + ASSERT_EQ(nan_drift.size(), 1u) << "a NaN value against a pin of 1.0 is a drift too"; + + const std::string unset_got = rendered_got(unset[0]); + const std::string nan_got = rendered_got(nan_drift[0]); + ASSERT_FALSE(unset_got.empty()) << "no got= field in '" << unset[0] << "'"; + ASSERT_FALSE(nan_got.empty()) << "no got= field in '" << nan_drift[0] << "'"; + EXPECT_NE(unset_got, nan_got) << "an unset parameter and a NaN value both rendered as '" << unset_got + << "', so an operator cannot tell which fault they are looking at"; + EXPECT_EQ(unset_got, std::string(kUnsetParameterRendering)) << "an unset parameter rendered as '" << unset_got << "'"; + EXPECT_NE(unset_got, "null") << "the rendering of an unset parameter is exactly the text it must stop being"; + + // The description is cut at a BYTE offset by the aggregated fault, so the marker's rendering + // carries the same ASCII-only obligation every other rendered value does. + for (const char c : unset[0]) { + EXPECT_LT(static_cast(c), 0x80u) << "non-ASCII byte in the unset rendering"; + } + + // The verdict the rendering describes, pinned directly. + EXPECT_FALSE(param_values_equal(unset_param_value(), json(1.0))) << "an unset parameter would satisfy a pin"; + EXPECT_FALSE(param_values_equal(json(1.0), unset_param_value())) << "and the mirrored operand order"; + EXPECT_TRUE(param_values_equal(unset_param_value(), unset_param_value())) + << "a parameter that is unset on every sweep would drift against its own captured baseline for ever"; + + // A JSON OBJECT is the marker's whole safety argument: it is a shape no ROS parameter value and + // no `expect` pin can take, so it cannot collide with something a node or an operator wrote. A + // string sentinel could - a node is free to hold the string below. + EXPECT_TRUE(unset_param_value().is_object()); + EXPECT_FALSE(is_unset_param(json(kUnsetParameterRendering))) + << "a node holding this exact string would be reported as unset"; + EXPECT_FALSE(is_unset_param(json(nullptr))); + EXPECT_FALSE(is_unset_param(json(std::numeric_limits::quiet_NaN()))); +} + +// Which transport replies become the marker. Both signals are load-bearing and neither subsumes the +// other: the TYPE is what a live rclcpp node sends (`not_set` alongside a null value), and the null +// VALUE is the backstop for a transport that omits the type, since null is not a legal encoding of +// any ROS parameter value and so must never reach a description as one. A NaN double is caught by +// neither - it is a float that merely DUMPS as `null` - which is the distinction being kept. +TEST(ParamDriftPolicy, OnlyADeclaredButUnsetTransportEntryBecomesTheMarker) { + EXPECT_TRUE(is_unset_param(param_entry_value(json{{"name", "a"}, {"type", "not_set"}, {"value", nullptr}}))) + << "the shape a live rclcpp node produces for a parameter it declared and never set"; + EXPECT_TRUE(is_unset_param(param_entry_value(json{{"name", "a"}, {"value", nullptr}}))) + << "a null value with no type is still not a legal ROS parameter value"; + + EXPECT_FALSE(is_unset_param(param_entry_value(json{{"name", "a"}, {"type", "double"}, {"value", 1.0}}))); + EXPECT_EQ(param_entry_value(json{{"name", "a"}, {"type", "double"}, {"value", 1.0}}), json(1.0)); + EXPECT_EQ(param_entry_value(json{{"name", "a"}, {"type", "string"}, {"value", "not_set"}}), json("not_set")) + << "a STRING parameter whose value is the word 'not_set' is an ordinary value, not an unset parameter"; + EXPECT_FALSE(is_unset_param( + param_entry_value(json{{"name", "a"}, {"type", "double"}, {"value", std::numeric_limits::quiet_NaN()}}))) + << "a NaN double is a value the node HAS; folding it into the marker would recreate the ambiguity"; + EXPECT_FALSE(is_unset_param(param_entry_value(json{{"name", "a"}, {"type", "bool"}, {"value", false}}))) + << "false is a value, not an absence"; +} + +// Head-only truncation renders two different values identically whenever they share a long prefix - +// the common case for robot_description and for nav2 footprint arrays where only a late element +// differs. The fault would fire correctly and then prove nothing about what changed. +TEST(ParamDriftPolicy, ElisionKeepsBothEndsSoValuesStayDistinguishable) { + std::vector before(40, 0.5); + std::vector after = before; + after.back() = 9.5; // only the LAST element differs + ParamDriftPolicy p(ParamDriftConfig{}); + p.evaluate("/costmap", Params{{"footprint", json(before)}}, true); + const auto out = p.evaluate("/costmap", Params{{"footprint", json(after)}}, true); + ASSERT_EQ(out.size(), 1u); + const auto baseline_at = out[0].find("baseline="); + const auto got_at = out[0].find(" got="); + ASSERT_NE(baseline_at, std::string::npos); + ASSERT_NE(got_at, std::string::npos); + const std::string rendered_baseline = out[0].substr(baseline_at + 9, got_at - baseline_at - 9); + const std::string rendered_got = out[0].substr(got_at + 5); + EXPECT_NE(rendered_baseline, rendered_got) + << "both values rendered as '" << rendered_baseline << "', so the fault shows no difference"; +}