diff --git a/.github/workflows/pixi.yml b/.github/workflows/pixi.yml index 7c16cad0..b7e16686 100644 --- a/.github/workflows/pixi.yml +++ b/.github/workflows/pixi.yml @@ -16,7 +16,9 @@ jobs: runs-on: ubuntu-latest # Experimental - don't block development while RoboStack support matures continue-on-error: true - timeout-minutes: 45 + # The build step alone is ~32 min of this, and the job has been landing at 37-40 min for + # weeks with about 3 min of run-to-run variance, so 45 left no room for a package growing. + timeout-minutes: 60 defaults: run: shell: bash diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 7266c9ff..df49ab2b 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -309,9 +309,24 @@ jobs: - name: Extend test timeouts for ASan overhead run: | - find build/ -name "CTestTestfile.cmake" \ - -exec sed -i 's/TIMEOUT "60"/TIMEOUT "180"/g' {} + \ - -exec sed -i 's/TIMEOUT "120"/TIMEOUT "360"/g' {} + + # Multiply EVERY declared CTest timeout by three, whatever it is. The + # x3 factor is the one the two literal rewrites this replaced encoded + # (60 -> 180, 120 -> 360), so no test's effective budget changes here; + # what changes is that a test declared at any OTHER value is no longer + # silently left on its native budget under a sanitizer that makes it + # several times slower. A fixed pair of literals goes stale the moment + # a budget is raised for good reasons, and the failure it produces is + # the worst kind: ctest kills the process and the test's own output + # dies with it, so the run reports a timeout and no test name. + find build/ -name "CTestTestfile.cmake" -exec \ + perl -pi -e 's/\bTIMEOUT "(\d+)"/sprintf(q{TIMEOUT "%d"}, $1 * 3)/ge' {} + + # Print what the tests will actually run with, so a sanitizer job that + # is killed on time can be read against its real budgets. `cat` first + # so grep sees one stream: its exit status then means "the rewrite had + # nothing to act on", which under pipefail correctly fails the step, + # rather than "one -exec batch happened to hold no timeouts". + find build/ -name "CTestTestfile.cmake" -exec cat {} + \ + | grep -oE 'TIMEOUT "[0-9]+"' | sort -t'"' -k2 -n | uniq -c - name: Run unit + integration tests with ASan + UBSan timeout-minutes: 30 @@ -414,9 +429,12 @@ jobs: - name: Extend test timeouts for TSan overhead run: | - find build/ -name "CTestTestfile.cmake" \ - -exec sed -i 's/TIMEOUT "60"/TIMEOUT "180"/g' {} + \ - -exec sed -i 's/TIMEOUT "120"/TIMEOUT "360"/g' {} + + # Same generic x3 rewrite as the ASan job - see the comment there for + # why it is not a list of literal values. + find build/ -name "CTestTestfile.cmake" -exec \ + perl -pi -e 's/\bTIMEOUT "(\d+)"/sprintf(q{TIMEOUT "%d"}, $1 * 3)/ge' {} + + find build/ -name "CTestTestfile.cmake" -exec cat {} + \ + | grep -oE 'TIMEOUT "[0-9]+"' | sort -t'"' -k2 -n | uniq -c - name: Run unit + integration tests with TSan timeout-minutes: 30 diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt index 51779c26..8c0fbdaa 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt @@ -28,6 +28,9 @@ find_package(ament_cmake REQUIRED) find_package(ros2_medkit_gateway REQUIRED) find_package(ros2_medkit_msgs REQUIRED) find_package(rclcpp REQUIRED) +# The reused gateway parameter transport includes rcl_interfaces message headers directly; +# it built only through rclcpp's transitive exports. +find_package(rcl_interfaces REQUIRED) find_package(nlohmann_json REQUIRED) find_package(lifecycle_msgs REQUIRED) find_package(tf2_msgs REQUIRED) @@ -82,7 +85,7 @@ add_library(ros2_medkit_graph_watchdog MODULE ${DETECTOR_SRCS}) target_include_directories(ros2_medkit_graph_watchdog PRIVATE include ${GATEWAY_SRC_INCLUDE_DIR}) medkit_target_dependencies(ros2_medkit_graph_watchdog - ros2_medkit_gateway rclcpp ros2_medkit_msgs lifecycle_msgs tf2_msgs geometry_msgs) + ros2_medkit_gateway rclcpp rcl_interfaces ros2_medkit_msgs lifecycle_msgs tf2_msgs geometry_msgs) target_link_libraries(ros2_medkit_graph_watchdog nlohmann_json::nlohmann_json) install(TARGETS ros2_medkit_graph_watchdog LIBRARY DESTINATION lib/${PROJECT_NAME}) @@ -121,6 +124,8 @@ if(BUILD_TESTING) find_package(ament_cmake_gtest REQUIRED) find_package(rosgraph_msgs REQUIRED) + # test_param_drift_integration.cpp includes rcutils/logging.h directly. + find_package(rcutils REQUIRED) ament_add_gtest(test_warmup_tracker test/test_warmup_tracker.cpp) target_include_directories(test_warmup_tracker PRIVATE include ${GATEWAY_SRC_INCLUDE_DIR}) @@ -187,12 +192,14 @@ if(BUILD_TESTING) target_link_libraries(test_reliability_gate nlohmann_json::nlohmann_json) medkit_set_test_domain(test_reliability_gate) - # LifecycleShutdownSuppressor: needs a REAL ReliabilityGate (departed_lifecycle_state_of() - # reads live from ctx.gate) - same real-Node/ROS_DOMAIN_ID rationale as test_reliability_gate - # above, unlike test_allowlist_suppressor (pure interface, no gate needed). + # Pure matching logic over hand-built inputs: no rclcpp::Node, so no domain is needed. ament_add_gtest(test_orphan_policy test/test_orphan_policy.cpp) target_include_directories(test_orphan_policy PRIVATE include) + ament_add_gtest(test_param_drift_policy test/test_param_drift_policy.cpp) + target_include_directories(test_param_drift_policy PRIVATE include) + target_link_libraries(test_param_drift_policy nlohmann_json::nlohmann_json) + ament_add_gtest(test_qos_policy test/test_qos_policy.cpp) target_include_directories(test_qos_policy PRIVATE include) medkit_target_dependencies(test_qos_policy rmw) # rmw is transitively found via rclcpp; no find_package needed @@ -202,9 +209,10 @@ if(BUILD_TESTING) medkit_target_dependencies(test_aggregated_fault ros2_medkit_gateway rclcpp ros2_medkit_msgs) target_link_libraries(test_aggregated_fault nlohmann_json::nlohmann_json) - # Pure suppressor-chain interface: no raise_fault/clear_fault calls (no ReliabilityGate - # sources needed), no live rclcpp::Node, so no ROS_DOMAIN_ID is required - same as - # test_qos_policy above. + # Real DDS endpoints read through the live graph API, so each of these takes a domain of its + # own via medkit_set_test_domain. The two that ALSO register a fake /fault_manager/report_fault + # service additionally share a RESOURCE_LOCK, because they spin real nodes for many seconds and + # running them concurrently on this box starves the discovery each one waits on. ament_add_gtest(test_qos_mismatch_integration test/test_qos_mismatch_integration.cpp src/detectors/qos_mismatch_detector.cpp @@ -232,6 +240,27 @@ if(BUILD_TESTING) medkit_set_test_domain(test_orphan_integration) set_tests_properties(test_orphan_integration PROPERTIES TIMEOUT 120 RESOURCE_LOCK graph_watchdog_integration_domain) + ament_add_gtest(test_param_drift_integration + test/test_param_drift_integration.cpp + src/detectors/param_drift_detector.cpp + src/detector_registry.cpp + src/reliability_gate.cpp + src/lifecycle_watcher.cpp + ${LIFECYCLE_STATE_READER_SOURCES} + ${PARAMETER_TRANSPORT_SOURCES}) + target_include_directories(test_param_drift_integration PRIVATE include ${GATEWAY_SRC_INCLUDE_DIR}) + medkit_target_dependencies(test_param_drift_integration + ros2_medkit_gateway rclcpp rcl_interfaces rcutils ros2_medkit_msgs lifecycle_msgs) + target_link_libraries(test_param_drift_integration nlohmann_json::nlohmann_json) + medkit_set_test_domain(test_param_drift_integration) + # 300 s, not 120: several cases here are paced off the detector's own attempt and tick horizons + # (sixty failed reads, sixty unarmed ticks, a read that answers past the per-read bound) and each + # of those costs real seconds that no budget knob can compress away. A timeout the suite can + # outgrow kills the process before gtest prints which case failed, which is the one thing a + # timeout must never do to a diagnostic suite. + set_tests_properties(test_param_drift_integration PROPERTIES + TIMEOUT 300 RESOURCE_LOCK graph_watchdog_integration_domain) + # === QoS-mismatch e2e (launch_testing) === # # Acceptance gate: proves GRAPH_QOS_MISMATCH raises and clears through the @@ -257,6 +286,77 @@ if(BUILD_TESTING) ENVIRONMENT "GATEWAY_TEST_PORT=19130;ROS_DOMAIN_ID=89;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" RESOURCE_LOCK graph_watchdog_e2e_domain) + # param_drift e2e: the detector's DEFAULT mode. The config-plumbing scenarios below all set + # baseline: false and none of them asserts a clear, so without this the self-capturing behaviour + # operators actually get, and the whole healing story the README documents, had no e2e at all. + # 300 s, not 240: the file's own deadlines are now the arming gate (60), the silent-capture + # window (15), the raise poll (60), the still-raised check (30) and the heal poll (60), and 240 + # no longer covers their sum. A budget shorter than the internal deadlines is killed by ctest + # before any of the messages those deadlines exist to print reaches the log, which turns every + # one of them into "timed out". + add_launch_test(test/e2e/test_param_drift_e2e.test.py TIMEOUT 300) + set_tests_properties(test_e2e_test_param_drift_e2e.test.py PROPERTIES + LABELS "integration;e2e" + ENVIRONMENT "GATEWAY_TEST_PORT=19170;ROS_DOMAIN_ID=89;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" + RESOURCE_LOCK graph_watchdog_e2e_domain) + + # Round-trip cost e2e: what ONE parameter read really costs the node that serves it, counted + # at a node that answers the parameter services by hand. param_drift's read budget is charged + # in round trips (2 / 3 / 2), every other test in this package proves the arithmetic GIVEN + # those numbers, and the stand-in transport they use counts CALLS - so nothing else here can + # notice a round trip being added to or removed from Ros2ParameterTransport. Deliberately runs + # with param_drift OFF: the reads are driven over the configurations endpoint so each count + # belongs to exactly one transport call. + # 300 s, not 180: this test's own retry deadlines (discovery, first contact against each of the + # two probes) plus the single-shot measurements between them can legitimately add up to more + # than 180 on a slow-discovery run - which is the run it exists to diagnose. A timeout shorter + # than the internal budgets kills the process before any of those diagnostics print, so the + # failure reads as "test timed out" and says nothing about round trips. + add_launch_test(test/e2e/test_param_roundtrip_e2e.test.py TIMEOUT 300) + set_tests_properties(test_e2e_test_param_roundtrip_e2e.test.py PROPERTIES + LABELS "integration;e2e" + ENVIRONMENT "GATEWAY_TEST_PORT=19180;ROS_DOMAIN_ID=89;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" + RESOURCE_LOCK graph_watchdog_e2e_domain) + + # === Config-plumbing e2e (launch_testing) === + # + # Proves nested per-detector config reaches a live detector through the REAL gateway + # (extract_plugin_config -> detector_config.hpp -> ParamDriftConfig::from_json). No C++ test + # above can give that proof: they all call configure() directly with hand-built JSON and + # never touch the delivery path. + # + # One source file, three CTest targets. The plugin reads its config once, at set_context() + # time, so proving the nested `expect` reader and the nested `mode` reader needs separate + # gateway launches rather than separate test methods in one. WATCHDOG_E2E_SCENARIO selects + # which launch and which assertions run in each process. + add_launch_test(test/e2e/test_config_plumbing_e2e.test.py + TARGET test_config_plumbing_e2e_positive + TIMEOUT 180) + set_tests_properties(test_config_plumbing_e2e_positive PROPERTIES + LABELS "integration;e2e" + ENVIRONMENT "GATEWAY_TEST_PORT=19140;ROS_DOMAIN_ID=89;WATCHDOG_E2E_SCENARIO=positive;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" + RESOURCE_LOCK graph_watchdog_e2e_domain) + + add_launch_test(test/e2e/test_config_plumbing_e2e.test.py + TARGET test_config_plumbing_e2e_mode_off + TIMEOUT 180) + set_tests_properties(test_config_plumbing_e2e_mode_off PROPERTIES + LABELS "integration;e2e" + ENVIRONMENT "GATEWAY_TEST_PORT=19150;ROS_DOMAIN_ID=89;WATCHDOG_E2E_SCENARIO=mode_off;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" + RESOURCE_LOCK graph_watchdog_e2e_domain) + + # Third launch: `mode: off` written BARE, which the ROS parser types as a YAML 1.1 boolean + # rather than a string. That is the form operators write, and a string-only reader silently + # keeps the detector raising. + add_launch_test(test/e2e/test_config_plumbing_e2e.test.py + TARGET test_config_plumbing_e2e_mode_off_yaml_bool + TIMEOUT 180) + set_tests_properties(test_config_plumbing_e2e_mode_off_yaml_bool PROPERTIES + LABELS "integration;e2e" + ENVIRONMENT + "GATEWAY_TEST_PORT=19160;ROS_DOMAIN_ID=89;WATCHDOG_E2E_SCENARIO=mode_off_yaml_bool;GRAPH_WATCHDOG_PLUGIN_PATH=${_WATCHDOG_E2E_SO}" + RESOURCE_LOCK graph_watchdog_e2e_domain) + ros2_medkit_relax_vendor_warnings() endif() diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md index 2225c130..638af37f 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md @@ -7,7 +7,8 @@ Detectors read the graph and raise faults through a `ReportFault` service client gateway node; the faults surface via FaultManager on the gateway `/faults` API. This package carries the plugin skeleton, the central reliability gate that holds raises -until the graph has quiesced, and two detectors, `qos_mismatch` and `orphan`. The remaining +until the graph has quiesced, and three detectors, `qos_mismatch`, `orphan` and +`param_drift`. The remaining silent-fault classes land in follow-up changes, each against its own issue; their fault codes are already reserved in the frozen `GRAPH_*` namespace (see "Fault codes"). @@ -72,6 +73,17 @@ differs only in its numeric field and the rule above already spares it. `/robot/scan`, is six edits apart, so no budget that is still specific will reach it. Do not rely on this detector for that class. +### `param_drift` keys + +| Key | Type | Default | Meaning | +|-----|------|---------|---------| +| `mode` | string \| bool | `raise` | As above. A bare `off` arrives as a YAML boolean; both forms disable the detector. | +| `baseline` | bool | `true` | Self-capture the value of every parameter not listed in `expect`, then flag later changes to it. `false` checks only what `expect` pins, and skips reading the full parameter list. | +| `expect.` | any | - | Absolute expected value for ``, checked on every node that has it. This is the only rule that catches a value that was wrong from the first tick, which self-capture cannot see. A node that does not declare `` answers NOT_FOUND, which is not a drift - so a misspelt name checks nothing at all. Once the sweep has covered every armed app - each one read, or given up on after repeated failed reads - and at least one of them answered, a pin that no app declares is warned about once, naming it. | +| `ignore` | string[] | `[]` | Parameter-name globs never flagged. Applies to the self-captured set only, so it cannot silence an `expect` pin. A pattern of nothing but `*` matches every name and suppresses self-capture wholesale, which is indistinguishable at runtime from finding nothing; that combination warns at startup. | +| `max_reads_per_tick` | int | `8` | Parameter-service ROUND TRIPS this detector may spend per tick, round-robin over nodes. Round trips, not calls: one call is several requests at the watched node, and the node pays for each of them. A `baseline` visit is charged 2 (a list, then one batched get) whatever the node's parameter count; each `expect` pin is charged 3 (a list, a get and a descriptor read), or 2 on a node that does not declare it - see the table under "Coverage latency" for what each of those really costs the node. So a self-capture visit spends 2 of the budget and pinning N parameters spends up to 3N. It is an AMORTISED rate, not a per-tick cap: there is no pacing INSIDE a visit, so a node's whole read set is issued back to back and the reader waits out the debt afterwards. Four pins against a budget of 8 fire 12 round trips inside one tick period and then idle for the rest of the next one; what the knob bounds is the sustained rate, not the burst. Accepted range 1..100000; anything else keeps the default and warns. | +| `prune_grace` | int | `60` | Consecutive sweeps an app may be missing before its finding is dropped: the finding survives `prune_grace` absences and goes on the next one. Injected for every detector at plugin scope and overridable per detector; a value outside 0..3600 is rejected with a warning and this detector falls back to 2, not to the plugin default. The captured BASELINE has a shorter horizon of its own, and that one is NOT configurable - it is the compile-time `kBaselineForgetGrace`: the baseline is dropped on the third consecutive missed sweep, the two the reliability gate absorbs plus one, so an ordinary dropped discovery poll cannot re-capture a node on its already-drifted value. Setting `prune_grace` below 2 pulls that forward onto the sweep the finding itself is dropped on, so the two then go together. | + ### `qos_mismatch` keys | Key | Type | Default | Meaning | @@ -81,6 +93,39 @@ rely on this detector for that class. | `allowlist` | string[] | `[]` | Subscriber FQNs never reported. Exact match only. | +## Closing the loop: healing config is required + +Every detector here reports level-triggered: it re-raises FAILED while the condition holds +and emits PASSED on every clean sweep. That is what lets the fault_manager's debounce +counter walk from CONFIRMED back to HEALED. The fault_manager only counts PASSED events +toward healing when `healing_enabled` is true for that fault, and it defaults to `false`. +Without it, a `GRAPH_*` fault that has genuinely cleared stays CONFIRMED forever. Enable +healing globally on the fault_manager node: + +```bash +ros2 run ros2_medkit_fault_manager fault_manager_node --ros-args \ + -p healing_enabled:=true \ + -p healing_threshold:=1 # 2 consecutive clean sweeps to heal (confirmation_threshold defaults to -1) +``` + +or scope it to this plugin only, by longest-prefix match on `source_id`, which is always +`graph_watchdog`: + +```yaml +# entity_thresholds.yaml, loaded via the fault_manager's entity_thresholds.config_file param +graph_watchdog: + healing_enabled: true + healing_threshold: 1 # 2 consecutive clean sweeps to heal (confirmation_threshold defaults to -1) +``` + +Healing time does not depend on how long the fault was active. The debounce counter is +clamped to `[confirmation_threshold, healing_threshold]` on every event, so a FAILED event +can push it down to `confirmation_threshold` and no further; a long-running fault does not +dig a hole that later clean sweeps have to climb out of. After the fix, each clean sweep +steps the counter up by one, so healing takes at most +`healing_threshold - confirmation_threshold` consecutive clean sweeps - exactly that many once +the counter has walked down to the floor, fewer if the fault was raised only briefly. + ## Where GRAPH_* faults hang Every `GRAPH_*` fault is raised with `source_id: graph_watchdog`. That is an App the @@ -339,6 +384,204 @@ local graph-cache queries, so every topic is checked every tick. `/rosout` and `configure()`; both C++ tiers hand `configure()` a JSON object directly. +#### `param_drift` (GRAPH_PARAM_DRIFT) + +Watches ROS 2 node parameters and raises `GRAPH_PARAM_DRIFT` when a live value diverges +from its reference. It needs no configuration to be useful: it captures each parameter's +value once the node arms and flags later runtime changes. Pinning a parameter to an +absolute value with `expect` additionally catches the case self-capture cannot see, a value +that was already wrong when the node started. + +**A changed parameter is not the same as a broken robot.** The detector reports that a +value moved and does not grade the consequence. Whether the new value matters is a question +about the machine, not about the graph, so it raises at WARN and leaves the judgement to +whoever reads the fault. + +**Aggregated fault, not per-node.** All drift in the graph is one graph-level +`GRAPH_PARAM_DRIFT`, not a fault per owning node, for the same reason as the other +detectors: the fault_manager identifies a fault by `fault_code` alone, so per-node faults +would collide into a single record. The description enumerates every currently-drifted +`(node, parameter)` pair, up to a cap of 480 characters - past that the text is truncated. Each +app's own contribution is trimmed to 150 characters before that, so one node drifting on many +parameters cannot fill the 480 by itself, and the entries are ordered `expect` violations first, +then self-captured drift, with newly detected apps ahead of the ones already reported inside each +group, so the cap cannot hide the two things worth reading. It clears +(`EVENT_PASSED`) on every sweep where nothing drifts, so one node reverting is not enough while +another still drifts, subject to the same `healing_enabled` requirement described in "Closing the +loop" above. A clear is withheld while any app still has no usable read - one the sweep has not +reached, one whose reads keep failing, or one the reliability gate has not armed - because health +that was never measured is not reported as health. That withholding is bounded in both directions: +an app whose reads keep failing stops holding the clear back once it has been given up on (see +below), and a gate-denied app stops once the frozen hold has elapsed. An app whose reads keep +failing is named in the log once ten consecutive attempts have failed; one the sweep has not yet +attempted is not named, because nothing is known about it yet - instead, a clear that has been +withheld for a whole minute of ticks is logged once, saying how many apps are behind it and naming +one, so an operator watching a fault refuse to heal can tell a graph the sweep cannot cover from a +detector that is simply working. + +```yaml +plugins: + graph_watchdog: + detectors: + param_drift: + expect: + use_sim_time: false # flag any node running on sim time in production + ignore: ["*_stamp"] # only filters self-captured params, so keep baseline enabled +``` + +`baseline` and `expect` combine: an empty `expect` with `baseline: true` is pure +self-capture; a sparse `expect` with `baseline: true` is pins plus self-capture; +`baseline: false` checks only the pinned parameters and moves far less data, because it never +reads the full parameter list. It is not cheaper against the budget, though: a self-capture visit +is charged 2 round trips no matter how many parameters the node has, while a single pin the node +declares is charged 3. + +**Coverage latency.** The sweep is budgeted by service ROUND TRIPS at the watched nodes - not by +node count, and not by transport calls either. `max_reads_per_tick` bounds how many round trips +happen per tick period, and the reader paces itself to that rate rather than reading back to back. +A node's whole read set is read in one visit, and the visit is then charged in round trips rather +than in calls. What it is charged is not always what the node paid: + +| Visit | Round trips charged | What the node actually pays | +|-------|---------------------|-----------------------------| +| `baseline: true` | 2 | 2 - one list, one batched get, whatever the parameter count. 4 on the very first contact; see below. | +| one `expect` pin the node declares | 3 | 3 - a name-scoped list, a get, and a descriptor read the transport makes unconditionally on the success path | +| one `expect` pin the node does not declare | 2 | 1 - the name-scoped list comes back empty and the transport stops there. The charge is the higher of the two NOT_FOUND paths: a name the list DOES find and the get then returns no value for costs 2, and the error code cannot tell them apart, so the bound overstates rather than understates. | +| one `expect` pin the node declares but has not set | 3 | 3 - not a NOT_FOUND at all. rclcpp answers an unset parameter with a `PARAMETER_NOT_SET` value rather than an absent one, so the read succeeds, pays for the descriptor like any other resolved pin, and the pin fires: a node holding no value at all does not satisfy a pin that names one. It is reported as `got=`, not as `got=null`, because a NaN-valued parameter reads as `null` too and the two are different faults. | + +The right-hand column is measured rather than asserted: `test/e2e/test_param_roundtrip_e2e.test.py` +drives the real transport against a node that answers the parameter services by hand and counts +every request it is asked to serve. + +Visiting every node once therefore takes at least about + +- `ceil(2 * node_count / max_reads_per_tick)` tick periods with `baseline: true`, and +- with `baseline: false`, `ceil(3 * pin_count * node_count / max_reads_per_tick)` where every node + declares every pin, down to `ceil(2 * pin_count * node_count / max_reads_per_tick)` where none of + them does, + +and a drift on a node not yet visited is invisible until then. At the defaults (budget 8, 1 s +tick) a 20-node graph is about 5 ticks in self-capture mode; the same graph with four pins every +node declares is about 30, and about 20 if no node declares any of them. Treat that as a floor +rather than a bound: a read that overruns its slot is not compensated, so a few slow nodes stretch +the rotation further. + +In `baseline: true` mode the very FIRST read of a given node costs two round trips more than the +table charges, because `list_parameters` primes the transport's defaults cache before doing its own +read. That excess is deliberately not charged: the cache has no expiry, so the pair is paid once +per node for the life of the gateway, and charging it on every read would throttle the whole run to +cover one rotation. The practical effect is that the first rotation after startup puts up to twice +the budgeted load on the nodes in it, and every rotation after that stays within budget. +`baseline: false` has no cold surcharge at all: `get_parameter` never primes that cache, so the +first pinned read of a node costs the same three round trips as every later one. + +Coverage latency governs repairs as well as detections: a node that reverted but has not been read +again yet keeps its stale entry, so the aggregated fault can stay raised for up to one rotation +after the underlying fix. Raise `max_reads_per_tick`, or lower the tick interval, to shorten that +window on a big graph. + +There is one stale entry those two knobs do NOT shorten. A node that is present but has left the +reliability gate's `active` state is not read at all, so its finding is frozen rather than +re-derived - which is right for a pause, and wrong for good. `kFrozenHoldTicks`, a compile-time 60 +consecutive gate-denied ticks (a minute at the default tick period), bounds it: past that the +finding is released and the app also stops holding back a clear, and both come back from a fresh +read if it ever arms again. Because that horizon is counted in TICKS and not in rotations, raising +`max_reads_per_tick` does nothing to it and shortening `tick_interval_ms` shortens it only +incidentally, by making a tick shorter. + +What is NOT derived from the rotation is anything the detector says about an app that will not +answer. Both the log line and the point at which it stops holding back a clear rest primarily on +completed, FAILED read attempts and not on elapsed ticks, so an app that is merely waiting its turn +is never announced as silent: the log names an app once ten consecutive reads of it have failed, +and the detector stops letting it hold back a clear only once more than sixty have. Each of those +carries an additional FLOOR in ticks, of the same size, which the app must also have spent +unread - an attempt is not a duration, and without the floor `max_reads_per_tick: 100000` spends +sixty-one attempts in a couple of milliseconds and writes off a node that has merely not finished +being discovered. The floor is a conjunction, so it can only ever delay the two, never bring them +forward. An armed app the sweep has never ATTEMPTED blocks the clear for as long as that stays +true, however long the rotation takes; a cached read is not a substitute, so an app whose reads +start failing blocks the clear again from the first failure. + +The reads run on a thread the plugin owns, never on the gateway's ROS executor, so a node +that stops answering stalls only this detector's sweep and not the gateway's request +handling. + +**Getting back to clean after a deliberate change.** The detector cannot tell an +intentional runtime change from a misconfiguration; it only knows the value moved away from +what it last saw. Reverting the parameter always works: the aggregate clears on the first sweep +where nothing drifts and every app has either been read, been given up on after repeated failed +reads, or exhausted the frozen hold with the gate still shut. + +Restarting the node only works if the restart is slow enough to be SEEN. A captured baseline +is dropped on the third consecutive sweep its app is absent from (sooner only if +`prune_grace` is set below 2, see its row above), and only then does the node's return start from +a fresh capture. That window is deliberate. DDS discovery drops a +node from a single poll routinely, with no restart involved, so a one-sweep window would +re-baseline that node on its already-drifted value and heal a true positive that could then +never fire again, because the new baseline IS the wrong value. The cost runs the other way: a +node that comes back inside the window keeps its pre-restart baseline and goes on being +reported as drifted until someone reverts the change. `respawn` in a launch file defaults to +no delay, so the usual edit-the-YAML-and-restart flow is frequently never observed as an +absence at all: a respawn that completes inside one tick leaves the node present on every +sweep, and no window, however short, catches that. To make a restart re-baseline, keep the +node out of the graph long enough to miss three sweeps: a `respawn_delay`, or a stop, a pause, +then a start. Accepting a new value as the baseline without a restart does not exist today. + +**Test tiers:** + +1. **Unit** (`test/test_param_drift_policy.cpp`): the pure comparison, rendering and + configuration rules against hand-built values - the glob matching, the exact + integer-versus-float comparison and the NaN carve-out, the `baseline` and `expect` + combinations, the config range checks, the ASCII-only elided rendering the capped description + depends on, and the `read_gap` spacing the pacing budget is built on. +2. **Integration** (`test/test_param_drift_integration.cpp`): real `rclcpp` nodes with real + parameters, read over the real parameter service against a fake `ReportFault` service, plus a + scripted stand-in transport for the replies no live node can be made to produce on demand - a + read still in flight when its app is de-armed, a SUCCESS carrying no parameters at all, a read + that throws, a node that never answers. It covers the grace-based prune when a node vanishes, + the re-baseline after an absence long enough to be a restart together with the single dropped + poll that must NOT re-baseline, that the baseline forget is reachable at every accepted + `prune_grace`, that the baseline horizon is exactly three missed sweeps and not two or four, + the clear path, that a closing reliability gate does not heal a present drift, and every state + in which a clear must be withheld because nothing is known about an app: one the sweep has not + reached, one the gate has not armed, and one that answered once and then stopped. It also + covers that both unread warnings need FAILED attempts AND elapsed ticks rather than either + alone, that an app given up on stops blocking the clear, that a withheld clear says so in the + log once, that a read overrunning the per-read bound is abandoned, that an app re-bound to a + different node starts from a fresh baseline, the per-app share of the description and that + trimming it keeps both ends, and the ordering that decides what survives the capped + description: a newly drifted app ahead of the ones already reported, and an `expect` violation + ahead of self-captured drift - including the cross case, where a pinned violation already + reported still outranks brand-new self-captured drift. The pacing budget is covered in both + tiers - `read_gap`'s arithmetic in the unit tests above, and the rate the reader actually + achieves against a configured budget here. +3. **E2e** (`test/e2e/test_param_drift_e2e.test.py`, + `test/e2e/test_param_roundtrip_e2e.test.py`, + `test/e2e/test_config_plumbing_e2e.test.py`): five real gateway launches. + The first is the acceptance gate: it drives the detector's DEFAULT self-capturing mode end to + end - a real node whose parameter the test changes over the real parameter service, a real + fault_manager, and both the raise and the heal read back from the operator-visible + `GET /api/v1/faults`. + The second is what makes the round-trip table above a measurement rather than an assertion. It + runs with `param_drift` switched OFF and drives the same two transport functions the detector + uses over `GET /{entity}/configurations`, against a node that answers the parameter services by + hand and counts every request it is asked to serve: a cold and a warm `baseline` visit, a + resolved pin, both NOT_FOUND paths (the unlisted name that costs 1 and the listed one with no + value that costs 2, which is what the charge of 2 is sized for), a declared-but-unset pin that + is a success costing 3, and a first pinned read of an untouched node. Nothing else in the + package can notice a round trip being added to or removed from the TRANSPORT - every other test + proves the arithmetic GIVEN those numbers, and the stand-in transport they use counts CALLS. + It does not read the detector's charge constants either, though: those are held to the + measurement by the timing cases in the integration suite, which pace the reader off them. + The last three are the only proof that a nested SUB-OBJECT under a detector key survives the + delivery path: `expect.` arrives from the gateway as nested JSON and has to be flattened + back to the dotted parameter name. (Plain per-detector scalars and arrays are proven by the + orphan e2e.) Every C++ test above calls `configure()` directly with hand-built JSON and never + touches the delivery path. One launch proves the nested + `expect` reader, one proves `mode: "off"`, and one proves the bare `mode: off` that the + ROS parser types as a YAML 1.1 boolean rather than a string, which is the form operators + actually write. + ## Reliability (bringup-quiesce) Silent-fault detectors are prone to bringup noise: a node joining the graph, a @@ -413,7 +656,7 @@ itself. The two things a detector opts into explicitly are `/tf_static`, whose publishers latch so a late subscriber must match the durability to receive them). -## Adding a detector +## Adding a detector 1. Create `src/detectors/.cpp`. 2. Subclass `Detector`; implement `id()` and `tick(DetectorContext&)`. Read the @@ -434,4 +677,3 @@ Frozen in `include/ros2_medkit_graph_watchdog/graph_fault_codes.hpp`: `GRAPH_QOS_MISMATCH`, `GRAPH_ORPHAN`, `GRAPH_NODE_DISAPPEARED`, `GRAPH_TF_STALE`, `GRAPH_PARAM_DRIFT`, `GRAPH_LATENCY_BUDGET`, plus one extension of the frozen namespace for a new capability beyond the original six: `GRAPH_NODE_INACTIVE`. - diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst index 27dd162b..2aed0cf0 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst @@ -89,8 +89,9 @@ Reliability core Detectors --------- -``qos_mismatch`` and ``orphan`` are the detectors this package ships so far. The -remaining silent-fault classes land in follow-up changes, each against its own issue. +``qos_mismatch``, ``orphan`` and ``param_drift`` are the detectors this package ships so +far. The remaining silent-fault classes land in follow-up changes, each against its own +issue. ``qos_mismatch`` raises ``GRAPH_QOS_MISMATCH``. It watches every topic's publisher/subscriber QoS pairs rather than parameter values: each @@ -129,11 +130,10 @@ are not checked. The checks match only concrete incompatible enum pairs, so ``SYSTEM_DEFAULT``/``UNKNOWN`` (never reported by a live endpoint, which always carries the resolved profile) never raise. -**Aggregation via the shared helper.** ``qos_mismatch`` is the first detector to use -the new ``AggregatedFault`` helper (``aggregated_fault.hpp``), factored out of -a shared helper so later detectors do not each reimplement the same -level-triggered raise/clear pattern. The rationale is -(see "Aggregation rationale" above): the fault_manager identifies a fault by +**Aggregation via the shared helper.** ``qos_mismatch`` was the first detector to use the +``AggregatedFault`` helper (``aggregated_fault.hpp``); ``orphan`` and ``param_drift`` go +through the same one, so no detector reimplements the level-triggered raise/clear +pattern. The rationale: the fault_manager identifies a fault by ``fault_code`` alone, so one ``GRAPH_QOS_MISMATCH`` per mismatched topic would collide into a single record under the shared code. The detector keeps one ``AggregatedFault`` instance for the whole graph and, each tick, hands it every currently-mismatched @@ -141,12 +141,17 @@ topic's description (keyed by topic name so a repeat mismatch on the same topic overwrites rather than duplicates); an empty map on a clean tick clears (``EVENT_PASSED``), level-triggered semantics (see "Closing the loop" in the README for the ``healing_enabled`` requirement to actually -reach HEALED). ``graph_source_id()``, also in ``aggregated_fault.hpp``, is the same -host-Component-id-or-literal-fallback logic - the detectors' -aggregated faults land under the same ``source_id``. +reach HEALED). ``graph_source_id()``, also in ``aggregated_fault.hpp``, returns the +constant ``kGraphWatchdogEntityId`` (``"graph_watchdog"``) and nothing else, so every +detector's aggregated faults land under the same ``source_id``. It deliberately does NOT +fall back to the host Component id: a runtime host Component built by ``HostInfoProvider`` +never sets ``external``, ``collect_component_app_fqns`` only puts a Component's bare id in +the fault scope set when it does, and a fault raised under it is therefore listed by no +entity endpoint at all. The ``ctx`` argument is unused and kept only because every call +site already has it in hand. -**Coverage is exhaustive, not budgeted.** Unlike a call-budgeted -round-robin sweep (bounded by a parameter-service transport call budget), +**Coverage is exhaustive, not budgeted.** Unlike a budgeted +round-robin sweep (bounded by a parameter-service round-trip budget), ``qos_mismatch`` reads no external service - ``get_topic_names_and_types()`` and the two ``get_*_info_by_topic()`` calls are local graph-cache queries, so every topic is checked every tick with no coverage-latency trade-off to configure or budget knob to @@ -221,9 +226,99 @@ so it also proves a string array and an integer survive the parameter path into ``configure()``, which no C++ tier can show. +``param_drift`` raises ``GRAPH_PARAM_DRIFT``. It reads node parameters over the +parameter service and reports a value that no longer matches its reference. The +reference is either self-captured, the value the parameter had when the node armed, or +pinned in config through ``expect``. A node absent for longer than the window the +reliability gate absorbs loses its captured baseline, so a restart - the documented way to apply +a new configuration - re-captures instead of reporting the change as drift. A shorter gap keeps +the baseline on purpose: DDS discovery drops a node from a single poll routinely, and re-capturing +on that would take the node's current, possibly already drifted, value as the new reference and +heal a real fault that could then never fire again. Only the pinned form can catch a value that was +already wrong at startup; self-capture by construction treats whatever it first sees as +correct. + +**A budgeted sweep, unlike every other detector here.** ``qos_mismatch`` and ``orphan`` +read the local graph cache and are therefore exhaustive every tick. This one makes real +service calls to other nodes, so it is bounded by ``max_reads_per_tick`` and sweeps the +graph round-robin. The budget is spent in service ROUND TRIPS at the watched node rather +than in transport calls, because one call is several requests there: a self-capture read +is a list plus a batched get, and an ``expect`` pin the node declares is a list, a get and a +descriptor read. A pin the node does NOT declare stops at the list, and is charged two rather +than the one it costs, because a second and rarer NOT_FOUND path does cost two and the error +code cannot tell them apart - a load bound may overstate, never understate. That buys a bound +on the load the sweep puts on watched nodes and pays with +coverage latency: a drift is invisible until its node's turn comes, and so is a repair. +The reads run on a thread the plugin owns rather than the gateway executor, so a node +that stops answering stalls this sweep only. + +**Three horizons for a stale entry.** A node missing from one sweep is not forgotten +immediately. Its captured BASELINE is dropped on the third consecutive sweep without it - the +two the warmup tracker's forget grace absorbs, plus one - so that a one-tick discovery gap does +not silently reset the baseline and hide a real drift behind a fresh capture. Its FINDING +survives ``prune_grace`` consecutive absences and is dropped on the next one, so that the same +gap does not clear the aggregated fault either. Only the finding horizon is configurable: +``prune_grace`` is a per-detector config key, while the baseline horizon is the compile-time +``kBaselineForgetGrace``. A ``prune_grace`` below 2 makes the finding the shorter of the two and +the baseline then goes on the same sweep, because the absence counter it rides on is dropped +with the finding. + +The third horizon is not about absence at all. A node that stays PRESENT but leaves the +reliability gate's ``active`` state is never re-evaluated (it is not armed) and never pruned (it +is still there), so its finding would otherwise be re-raised for ever - an operator who fixes +the parameter and leaves the node deactivated would watch ``GRAPH_PARAM_DRIFT`` stay CONFIRMED +naming a value that is no longer true. ``kFrozenHoldTicks`` (60 consecutive gate-denied ticks, +one minute at the default tick period) bounds that: past it the finding is released and the app +also stops holding back a clear, and both re-derive from a fresh read if it ever arms again. +Because it is a compile-time constant and counted in ticks rather than in rotations, neither +``max_reads_per_tick`` nor ``tick_interval_ms`` shortens it. + +**Severity is WARN, deliberately.** The detector knows a value moved; it does not know +whether that breaks the robot. Grading the consequence needs knowledge of the machine +that the graph does not carry. + +**Nothing is cleared that was not measured.** A clear asserts that nothing is drifting anywhere, +which is only true if every app was actually looked at, so the aggregate emits nothing at all while +any app still has no usable read. That covers three states which are all "nothing is known about +this app": the round-robin has not reached it, its reads keep failing, or the reliability gate has +not armed it - and the last of those is the ordinary state of every app for the first +``warmup_cycles`` ticks of a run, so without it a restart clears before issuing a single round trip. +A cached read counts as evidence only until a read of that app FAILS, after which the app is +unmeasured again. Each hold is bounded from the other side: an app is given up on after both sixty +failed attempts and sixty ticks unread, and a gate-denied one after the frozen hold. The bound on +attempts alone would not do, because an attempt is not a duration - the reader paces itself off +``max_reads_per_tick``, so at the accepted ceiling sixty-one attempts are spent in milliseconds and +a node still being discovered would be written off. Finally, a clear withheld for a minute of ticks +is logged once with the count of apps behind it, because from outside the process a correctly +withheld clear and a detector that has nothing to report look exactly the same. + +**Test tiers.** ``test_param_drift_policy.cpp`` pins the pure comparison, rendering and +config rules. ``test_param_drift_integration.cpp`` drives real nodes with real parameters over +the real parameter service against a fake ``ReportFault`` service, including a node that +never answers, which must not hang the tick, and a node that answers past the per-read bound, +which must not be read at all. +``test/e2e/test_param_drift_e2e.test.py`` is the acceptance gate: one gateway launch carrying +the whole raise-and-clear story for the detector's default self-capturing mode, with a real +fault_manager and both ends read back from ``GET /api/v1/faults``. +``test/e2e/test_param_roundtrip_e2e.test.py`` measures what one parameter read really costs the +node that serves it, against a node that answers the parameter services by hand and counts every +request - so a round trip added to or removed from the TRANSPORT is caught, instead of quietly +invalidating the round-trip figures the budget is built on. Nothing else in the package can +notice that: every other test proves the arithmetic GIVEN those figures, and the stand-in +transport they use counts calls. It does not, however, read the detector's own charge constants +at all - it runs with ``param_drift`` switched off - so what holds those to the measurement is +the timing suite in ``test_param_drift_integration.cpp``, which times the reader against a +configured budget and therefore moves when a charge does. Cost and charge are pinned separately +and on purpose. +``test/e2e/test_config_plumbing_e2e.test.py`` drives three further gateway launches covering the +nested ``expect`` sub-object reader, ``mode: "off"``, and the bare ``mode: off`` that the +ROS parser types as a YAML 1.1 boolean; those three prove configuration delivery and +suppression only, and assert no clear. + + Status --------------- The plugin loads, ticks the graph, and shuts down cleanly. The reliability core is real -and already ticking. Two silent-fault detector classes raise through it today, -``qos_mismatch`` and ``orphan``. The remaining classes land in follow-up changes, each -against its own issue. +and already ticking. Three silent-fault detector classes raise through it today, +``qos_mismatch``, ``orphan`` and ``param_drift``. The remaining classes land in follow-up +changes, each against its own issue. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp index 3e2e088f..7fda64c0 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp @@ -63,9 +63,10 @@ class AggregatedFault { public: /// Cap on the generated description. One aggregated fault names every affected entity, so /// on a large graph the raw text grows without bound - an oversized ReportFault description - /// then travels into every /faults payload and the fault_manager's store. param_drift's - /// hand-rolled aggregation applies the same cap; this keeps every detector that uses the - /// helper consistent with it. + /// then travels into every /faults payload and the fault_manager's store. Every detector + /// aggregates through this helper, so the cap is applied in one place rather than copied per + /// detector. A detector may additionally trim what a SINGLE entity contributes before handing + /// it over (param_drift does), so that one entity cannot spend the whole budget by itself. static constexpr std::size_t kMaxDescriptionChars = 480; AggregatedFault(const char * code, std::uint8_t severity) : code_(code), severity_(severity) { diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp index c56062d6..2db5b83d 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -28,7 +29,12 @@ namespace ros2_medkit_gateway { struct IntrospectionInput; -} +// Only named in the test-only transport factory signature below, so a declaration is enough. +// Including the gateway header here would pull it into every consumer of this one, and the test +// targets that do not add GATEWAY_SRC_INCLUDE_DIR would then resolve it from install/ while the +// plugin resolves it from the source tree - the split this package's own CMakeLists warns about. +class ParameterTransport; +} // namespace ros2_medkit_gateway namespace ros2_medkit_graph_watchdog { @@ -128,6 +134,21 @@ class Detector { virtual std::size_t tracked_count_for_test() const { return 0; } + + /// Test-only injection hook for detectors that read parameters over a transport. Returns false + /// when the detector does not use one, so a test can assert it reached the right detector rather + /// than silently doing nothing. + /// + /// It lives on the base because detectors are file-local and self-registering: a test only ever + /// holds a `Detector*` from the registry, so there is no derived type to cast to. The alternative + /// is leaving the behaviours that only a stand-in transport can produce - a read still in flight + /// when its app is de-armed, a SUCCESS response carrying no parameters, a transport that fails to + /// construct - untested, and none of them is reachable from a live ROS graph. + using ParameterTransportFactory = + std::function(rclcpp::Node *)>; + virtual bool set_parameter_transport_factory_for_test(const ParameterTransportFactory & /*factory*/) { + return false; + } }; } // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/param_drift_policy.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/param_drift_policy.hpp new file mode 100644 index 00000000..f026e3c2 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/param_drift_policy.hpp @@ -0,0 +1,475 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ros2_medkit_graph_watchdog/detector_config_keys.hpp" // collect_unknown_detector_keys + +namespace ros2_medkit_graph_watchdog { + +/// Minimal glob matcher supporting `*` (matches any run, including empty). No `?`, +/// no character classes - ROS param ignore patterns only need `*`. +inline bool glob_match(const std::string & pattern, const std::string & text) { + std::size_t p = 0; + std::size_t t = 0; + std::size_t star = std::string::npos; + std::size_t mark = 0; + while (t < text.size()) { + if (p < pattern.size() && pattern[p] == '*') { + star = p++; + mark = t; + } else if (p < pattern.size() && pattern[p] == text[t]) { + ++p; + ++t; + } else if (star != std::string::npos) { + p = star + 1; + t = ++mark; + } else { + return false; + } + } + while (p < pattern.size() && pattern[p] == '*') { + ++p; + } + return p == pattern.size(); +} + +/// The transport's own name for the type of a parameter a node DECLARED and never SET. +/// Ros2ParameterTransport::parameter_type_to_string maps rclcpp's PARAMETER_NOT_SET to this. +inline constexpr const char * kNotSetParameterType = "not_set"; + +/// Key of the out-of-band marker that stands in for such a parameter inside an observed param map. +inline constexpr const char * kUnsetParameterKey = "x-medkit-unset"; + +/// How that marker reads in a fault description. Unquoted on purpose: a STRING parameter whose +/// value happens to be `` renders WITH quotes, so no node-supplied value can produce this +/// exact text and an operator reading it is never looking at something a node wrote. +inline constexpr const char * kUnsetParameterRendering = ""; + +/// The stand-in carried through an observed param map for a parameter the node declared and +/// never set. +/// +/// A JSON OBJECT, and that is the whole of the safety argument. ROS parameter values are bool, +/// int, double, string and the 1-D array forms of those, so an object is a shape no parameter +/// value can take; and ParamDriftConfig::flatten_expect recurses THROUGH objects and only ever +/// stores the scalar or array it bottoms out on, so an operator's `expect` pin cannot hold one +/// either. Neither side of a comparison can therefore be mistaken for this. A string sentinel +/// would have neither property - a node is free to hold the string `` - and the pin an +/// operator wrote on that node would silently stop reporting. +inline nlohmann::json unset_param_value() { + return nlohmann::json{{kUnsetParameterKey, true}}; +} + +/// True for the marker above, and for nothing a node or an operator can write. +inline bool is_unset_param(const nlohmann::json & v) { + return v.is_object() && v.contains(kUnsetParameterKey); +} + +/// The value one transport parameter entry (`{"name","value","type"}`) contributes to an observed +/// param map, with a declared-but-never-set parameter mapped to the marker instead of a bare null. +/// +/// Two signals, because either alone leaves a way for `null` to reach a fault description. The +/// TYPE is the transport's explicit statement and the one a live node produces: rclcpp answers a +/// declared-but-unset name with a PARAMETER_NOT_SET value rather than an absent one, so the read +/// SUCCEEDS carrying null and `type` is `not_set`. The VALUE is the backstop: null is not a legal +/// encoding of any ROS parameter value, so a transport that omits the type still cannot put a bare +/// `null` in front of an operator. A NaN double is caught by NEITHER - it arrives as a float that +/// merely DUMPS as `null` - and that is exactly the distinction this exists to keep: after this, +/// `got=null` in a description means the value is NaN (or infinite) and nothing else. +inline nlohmann::json param_entry_value(const nlohmann::json & entry) { + const auto type = entry.find("type"); + if (type != entry.end() && type->is_string() && type->get() == kNotSetParameterType) { + return unset_param_value(); + } + const auto value = entry.find("value"); + if (value == entry.end() || value->is_null()) { + return unset_param_value(); + } + return *value; +} + +/// True when an integer and a float denote the SAME number, without routing the integer through +/// a double. nlohmann's own cross-type compare does exactly that cast, so beyond 2^53 two +/// different values compare equal and the drift is missed. Reachable from an `expect` pin written +/// with a decimal point against an integer parameter, or a parameter whose type flips between +/// INTEGER and DOUBLE. +inline bool int_equals_float_exactly(const nlohmann::json & i, const nlohmann::json & f) { + const double d = f.get(); + if (!std::isfinite(d)) { + return false; // NaN or infinity is never equal to an integer + } + // Integrality without an equality test on doubles: -Werror=float-equal forbids == and != here, + // and it is right to - the same flag caught a real sentinel bug in this package before. + double whole = 0.0; + const double frac = std::modf(d, &whole); + if (frac > 0.0 || frac < 0.0) { + return false; // a fractional value can never equal an integer + } + // 2^63 is not representable as int64_t; compare against the double bound instead. + if (d < -9223372036854775808.0 || d >= 9223372036854775808.0) { + return false; + } + if (i.is_number_unsigned()) { + // The sign has to be checked BEFORE the cast. Converting a negative double to uint64_t is + // undefined behaviour, and it does not merely produce a harmless mismatch: on x86-64 -1.0 + // lands on UINT64_MAX, so a parameter holding that value compares EQUAL to a pin of -1.0 and + // the drift is missed. Every positive integer arrives from the transport stored unsigned, so + // this branch is the ordinary path, not a corner of it. + return d >= 0.0 && i.get() == static_cast(d); + } + return i.get() == static_cast(d); +} + +/// Equality for drift detection. Three carve-outs sit in front of nlohmann's `operator==`: +/// +/// - A mixed integer/float pair goes through int_equals_float_exactly rather than `operator==`, +/// whose cross-type compare casts the integer to a double and so collapses int64s differing +/// beyond 2^53 into a MISSED drift. +/// - Two NaN doubles are equal, because IEEE `NaN != NaN` would otherwise make any NaN-valued +/// param (a common "unset/disabled" sentinel) a permanent false positive that never clears. +/// - Arrays recurse element-wise, so both carve-outs apply inside an array param as well (ROS +/// params are at most 1-D, so no deeper recursion is needed). +/// +/// Everything else defers to `operator==`, and the unset marker (see unset_param_value) is +/// deliberately one of them rather than a fourth carve-out: as a JSON object it compares unequal to +/// every value a node or an `expect` pin can hold, so a pin against a parameter the node never set +/// still reports - and equal to itself, so a parameter that is unset on every sweep does not drift +/// against its own captured baseline for ever. Both directions are pinned in the unit tests. +inline bool param_values_equal(const nlohmann::json & a, const nlohmann::json & b) { + if (a.is_number_float() && b.is_number_float() && std::isnan(a.get()) && std::isnan(b.get())) { + return true; + } + if (a.is_number_integer() && b.is_number_float()) { + return int_equals_float_exactly(a, b); + } + if (a.is_number_float() && b.is_number_integer()) { + return int_equals_float_exactly(b, a); + } + if (a.is_array() && b.is_array()) { + if (a.size() != b.size()) { + return false; + } + for (std::size_t i = 0; i < a.size(); ++i) { + if (!param_values_equal(a[i], b[i])) { + return false; // recurse -> NaN[] handled element-wise (ROS params <= 1-D) + } + } + return true; + } + return a == b; +} + +/// Minimum spacing between two parameter reads so that no more than `max_reads_per_tick` +/// of them happen per tick period. This is the whole enforcement of that budget: the reader +/// takes one target per pass and waits out the rest of its slot, so a gap of zero means no +/// budget at all. +/// +/// Computed in microseconds on purpose. A budget larger than the tick period expressed in +/// milliseconds (2000 reads against a 1000 ms tick) truncates to zero under integer +/// millisecond division, which silently removes the bound instead of tightening it. +/// Microseconds are not enough on their own either: a budget above a million reads per second +/// truncates to zero again, and a zero gap is a reader that never waits. The floor keeps the +/// loop paced no matter what an operator asks for, and it does so SILENTLY: a budget inside the +/// accepted range is taken as written, and whether the floor bites depends on the tick period, +/// which from_json never sees. `max_reads_per_tick: 100000` (the ceiling) against +/// `tick_interval_ms: 1` therefore buys 1000 round trips per tick period, not 100000, with +/// nothing logged - the floor caps the effective budget BELOW the request, which is the safe +/// direction for a load bound, so this is a limit and not a defect. +/// +/// Rounded UP, because the achievable rate is `1 / gap` and not `reads / period`: flooring the +/// division makes the gap shorter than the budget allows and the rate correspondingly higher. +/// `tick_interval_ms: 1` with `max_reads_per_tick: 600` - both accepted configurations - floors +/// to a 1 us gap and 1000 round trips per tick period against a documented bound of 600. A knob +/// an operator lowers to protect a node must never be overshot; rounding up can only spend the +/// budget more slowly than asked, which is the safe direction for a bound. +inline std::chrono::microseconds read_gap(int tick_interval_ms, int max_reads_per_tick) { + const auto reads = static_cast(max_reads_per_tick > 0 ? max_reads_per_tick : 1); + const auto period_us = static_cast(tick_interval_ms) * 1000; + return std::chrono::microseconds(std::max(1, (period_us + reads - 1) / reads)); +} + +/// Parsed param-drift detector configuration. See the design doc for the model. +struct ParamDriftConfig { + bool baseline_capture = true; ///< self-capture drift on unlisted params + std::map expect; ///< param_name -> absolute expected value + std::vector ignore_globs; ///< param-name globs never flagged (self-capture set only) + int max_reads_per_tick = 8; ///< bound on blocking param reads per tick (coverage latency lever) + /// Upper bound on the configurable budget. A request beyond this is not a tuning choice, it is + /// a typo or a unit mix-up: even at the floor gap of one microsecond the reader cannot issue a + /// million service round-trips per tick, so accepting it would only mean pretending. + static constexpr std::int64_t kMaxReadsPerTickCeiling = 100000; + std::vector warnings; ///< non-fatal config problems, surfaced by the detector's logger + + /// Parse the per-detector config the plugin delivers (config["detectors"]["param_drift"], + /// see detector_config.hpp): keys `baseline` (bool), `expect` (object of param_name -> + /// expected value, flattened from the gateway's nested delivery), `ignore` (native JSON + /// string array). A present-but-wrong-typed `baseline`/`ignore`/`expect` is NOT silently + /// dropped: it records a warning (a mistyped `ignore` would otherwise re-enable drift alerts + /// on suppressed params with no diagnostic trail). + static ParamDriftConfig from_json(const nlohmann::json & detector_cfg) { + ParamDriftConfig cfg; + if (detector_cfg.contains("baseline")) { + if (detector_cfg["baseline"].is_boolean()) { + cfg.baseline_capture = detector_cfg["baseline"].get(); + } else { + cfg.warnings.push_back("'baseline' must be a boolean; ignoring it and keeping the default (true)"); + } + } + if (detector_cfg.contains("ignore")) { + if (detector_cfg["ignore"].is_array()) { + for (const auto & g : detector_cfg["ignore"]) { + if (g.is_string()) { + cfg.ignore_globs.push_back(g.get()); + } else { + cfg.warnings.push_back("'ignore' entries must be strings; skipping a non-string entry"); + } + } + } else { + cfg.warnings.push_back("'ignore' must be a string array (e.g. [\"*_stamp\"]); ignoring it"); + } + } + if (detector_cfg.contains("max_reads_per_tick")) { + // Range-check BEFORE narrowing. is_number_integer() is true across the whole 64-bit range + // and get() truncates, so 4294967297 would arrive as 1 and -4294967295 would arrive as + // 1 too - a NEGATIVE request silently passing a positive-only guard, and a budget quietly + // eight times smaller than the default. Both were accepted without a warning. + const auto & value = detector_cfg["max_reads_per_tick"]; + const std::int64_t wide = value.is_number_integer() ? value.get() : 0; + if (value.is_number_integer() && wide > 0 && wide <= kMaxReadsPerTickCeiling) { + cfg.max_reads_per_tick = static_cast(wide); + } else { + cfg.warnings.push_back("'max_reads_per_tick' must be an integer in 1.." + + std::to_string(kMaxReadsPerTickCeiling) + "; keeping the default (8)"); + } + } + if (detector_cfg.contains("expect")) { + if (detector_cfg["expect"].is_object()) { + // The gateway un-nests dotted config keys, so a pin on a dotted param name + // (e.g. expect.qos_overrides./scan.reliability) arrives as a nested object. + // Flatten it back to the full dotted param name so it matches the observed key. + flatten_expect(detector_cfg["expect"], "", cfg.expect, &cfg.warnings); + } else { + cfg.warnings.push_back("'expect' must be an object of param_name -> expected value; ignoring it"); + } + } + // A wrong TYPE on the four keys above already warns; a wrong NAME used to not, even + // though it fails the same way. `ignore_globs` (the struct's own field name) instead + // of `ignore` leaves the globs empty and GRAPH_PARAM_DRIFT firing on exactly the + // params the operator meant to suppress. + collect_unknown_detector_keys(detector_cfg, known_keys(), cfg.warnings); + // Configurations that parse cleanly and can never do anything. Silence here is the worst + // outcome: the detector looks configured and reports nothing, forever. + if (!cfg.baseline_capture && cfg.expect.empty()) { + cfg.warnings.push_back( + "'baseline' is false and 'expect' is empty, so this detector will never read a " + "parameter or report anything; set 'expect' or leave 'baseline' enabled"); + } + if (!cfg.baseline_capture && !cfg.ignore_globs.empty()) { + cfg.warnings.push_back( + "'ignore' only applies to self-captured parameters, and 'baseline' is false, so it has " + "no effect here"); + } + // A glob made of nothing but '*' matches every parameter name, so self-capture is suppressed + // wholesale. At runtime that is indistinguishable from a detector that is working and finding + // nothing - the silent failure this detector exists to rule out, reproduced in its own config. + const bool catch_all_ignore = + std::any_of(cfg.ignore_globs.begin(), cfg.ignore_globs.end(), [](const std::string & g) { + return !g.empty() && g.find_first_not_of('*') == std::string::npos; + }); + if (cfg.baseline_capture && catch_all_ignore) { + if (cfg.expect.empty()) { + cfg.warnings.push_back( + "'ignore' matches every parameter name and 'expect' is empty, so this detector will " + "never report anything; narrow the pattern"); + } else { + cfg.warnings.push_back( + "'ignore' matches every parameter name, so self-captured drift is fully suppressed " + "and only the 'expect' pins can report"); + } + } + return cfg; + } + + /// Config keys this detector reads. Anything else under detectors.param_drift is a typo. + static const std::set & known_keys() { + static const std::set keys{"baseline", "ignore", "max_reads_per_tick", "expect"}; + return keys; + } + + /// Flatten a (possibly nested) `expect` object back to dotted param-name keys. The gateway + /// splits dotted config keys into nested objects; ROS param values are scalars/arrays (never + /// objects), so the recursion stops at the expected value and the joined key is the full + /// param name (e.g. {"qos_overrides":{"/scan":{"reliability":v}}} -> "qos_overrides./scan.reliability"). + static void flatten_expect(const nlohmann::json & obj, const std::string & prefix, + std::map & out, + std::vector * warnings = nullptr) { + for (const auto & item : obj.items()) { + const std::string key = prefix.empty() ? item.key() : prefix + "." + item.key(); + if (item.value().is_object()) { + if (item.value().empty() && warnings != nullptr) { + // An empty object carries no expected value, so the pin the operator wrote silently + // becomes nothing at all. Say so rather than dropping it. + warnings->push_back("'expect." + key + "' has no value; ignoring that pin"); + } + flatten_expect(item.value(), key, out, warnings); + } else if (!out.emplace(key, item.value()).second && warnings != nullptr) { + // Two config paths flattened to the same parameter name (e.g. `a: {b: 1}` next to + // `a.b: 2`). emplace keeps the first, so the other pin is silently lost. + warnings->push_back("'expect." + key + "' is pinned twice; keeping the first value"); + } + } + } +}; + +/// Pure, ROS-free drift core. Holds per-app captured baselines and computes this app's +/// drifted param descriptors from an observed param map. One instance per detector. +/// Aggregation (raise/clear across apps) moves to the detector. +class ParamDriftPolicy { + public: + explicit ParamDriftPolicy(ParamDriftConfig cfg) : cfg_(std::move(cfg)) { + } + + const ParamDriftConfig & config() const { + return cfg_; + } + + /// Per-value budget inside the description. Small enough that several drifted entries fit under + /// AggregatedFault's cap, large enough to still show a recognisable value. Public so a test can + /// pin the budget rather than a loose descriptor length that would pass at twice the size. + static constexpr std::size_t kMaxRenderedValueChars = 64; + + /// Returns this app's drifted descriptors (empty = clean). Captures the app's baseline + /// the first time it is called with capture_allowed=true (so a not-yet-armed node is + /// never baselined pre-settle). expect rules are absolute and fire regardless. + /// `pinned_out`, when given, receives how many of the returned descriptors came from an + /// `expect` pin. Those are always the leading entries, so a caller can tell an + /// operator-declared violation from self-captured drift without parsing the text back. It + /// must not parse: a parameter's own value can contain any substring, including the markers + /// this function writes, so text sniffing would misclassify. + std::vector evaluate(const std::string & app_id, const std::map & observed, + bool capture_allowed, std::size_t * pinned_out = nullptr) { + std::vector drifted; + for (const auto & [name, expected] : cfg_.expect) { + const auto it = observed.find(name); + if (it != observed.end() && !param_values_equal(it->second, expected)) { + drifted.push_back(app_id + ":" + safe_name(name) + " expected=" + compact(expected) + + " got=" + compact(it->second)); + } + } + if (pinned_out != nullptr) { + *pinned_out = drifted.size(); + } + if (cfg_.baseline_capture && capture_allowed) { + auto & base = baselines_[app_id]; // creates empty on first sight + for (const auto & [name, value] : observed) { + if (cfg_.expect.count(name) > 0 || is_ignored(name)) { + continue; + } + const auto bit = base.find(name); + if (bit == base.end()) { + base.emplace(name, value); // new param -> capture silently (not a drift) + continue; + } + // No `first` check here: when this app had no baseline at all, every name takes the + // capture branch above, so reaching this line already means the baseline pre-existed. + if (!param_values_equal(value, bit->second)) { + drifted.push_back(app_id + ":" + safe_name(name) + " baseline=" + compact(bit->second) + + " got=" + compact(value)); + } + } + } + return drifted; + } + + /// App vanished from the graph: drop its baseline so a later reappearance re-captures + /// instead of drifting against a stale baseline. + void forget(const std::string & app_id) { + baselines_.erase(app_id); + } + + private: + bool is_ignored(const std::string & name) const { + for (const auto & g : cfg_.ignore_globs) { + if (glob_match(g, name)) { + return true; + } + } + return false; + } + + /// Render a parameter value for a fault description. Node-supplied values are arbitrary, and + /// this is the first detector to put them into fault text at all, so two things must hold. + /// + /// ASCII only. The aggregated description is capped by cutting at a BYTE offset, and a cut + /// through a multi-byte UTF-8 sequence produces a description the gateway's own DTO writer + /// cannot serialize - GET /faults then answers 500 instead of listing the fault. `ensure_ascii` + /// escapes non-ASCII, so any byte offset is a safe cut point. + /// + /// Never throws. A value whose bytes are not valid UTF-8 makes a plain dump() throw, and that + /// throw escapes into the plugin's per-detector guard, so the detector would neither raise nor + /// clear for as long as the value is present - a CONFIRMED fault that can never heal. The + /// replace handler substitutes U+FFFD instead. + /// + /// Long values are elided: one array-valued parameter can otherwise consume the whole + /// description budget and hide every other drifted entry. + /// + /// A parameter the node DECLARED and never SET is rendered from the marker rather than dumped. + /// Dumping it would print `null`, and a NaN double prints `null` too - so one description said + /// the same thing about "the node never set this parameter" and "the value is NaN", which are + /// different faults with different remedies. See unset_param_value(). + static std::string compact(const nlohmann::json & v) { + if (is_unset_param(v)) { + return kUnsetParameterRendering; + } + return elide(v.dump(-1, ' ', /*ensure_ascii=*/true, nlohmann::json::error_handler_t::replace)); + } + + /// Same treatment for a parameter NAME. Names come verbatim from the watched node's + /// ListParameters reply and are never validated, so they carry exactly the two hazards values do: + /// a name whose bytes are not valid UTF-8, or a valid multi-byte name split by the aggregated + /// description's byte-offset cut, makes the gateway's JSON writer throw. That answers 500 on + /// GET /faults and takes every OTHER fault down with it, since they share the response. + static std::string safe_name(const std::string & name) { + return elide(nlohmann::json(name).dump(-1, ' ', /*ensure_ascii=*/true, nlohmann::json::error_handler_t::replace)); + } + + /// Shorten to the budget while keeping BOTH ends. Head-only truncation renders two different + /// values identically whenever they share a long prefix - the common case for robot_description + /// and for nav2 footprint arrays, where only a late element differs - so the fault would fire + /// correctly and then prove nothing about what changed. + static std::string elide(std::string text) { + if (text.size() <= kMaxRenderedValueChars) { + return text; + } + const std::size_t keep = kMaxRenderedValueChars - 3; + const std::size_t head = (keep + 1) / 2; + return text.substr(0, head) + "..." + text.substr(text.size() - (keep - head)); + } + + ParamDriftConfig cfg_; + std::map> baselines_; // app_id -> (param -> value) +}; + +} // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/package.xml b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/package.xml index ee975482..615d6b3b 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/package.xml +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/package.xml @@ -13,6 +13,10 @@ ros2_medkit_gateway ros2_medkit_msgs rclcpp + + rcl_interfaces nlohmann-json-dev lifecycle_msgs tf2_msgs @@ -25,6 +29,9 @@ rosgraph_msgs std_msgs sensor_msgs + + rcutils launch_testing_ament_cmake diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/param_drift_detector.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/param_drift_detector.cpp new file mode 100644 index 00000000..6a9b378f --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/param_drift_detector.cpp @@ -0,0 +1,1125 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ros2_medkit_gateway/core/providers/introspection_provider.hpp" +#include "ros2_medkit_gateway/core/transports/parameter_transport.hpp" +#include "ros2_medkit_gateway/ros2/transports/ros2_parameter_transport.hpp" +#include "ros2_medkit_graph_watchdog/aggregated_fault.hpp" +#include "ros2_medkit_graph_watchdog/detector.hpp" +#include "ros2_medkit_graph_watchdog/detector_config_keys.hpp" +#include "ros2_medkit_graph_watchdog/detector_registry.hpp" +#include "ros2_medkit_graph_watchdog/graph_fault_codes.hpp" +#include "ros2_medkit_graph_watchdog/param_drift_policy.hpp" + +namespace ros2_medkit_graph_watchdog { + +namespace { +// Fault severity scale: SEVERITY_WARN=1, SEVERITY_ERROR=2 (ros2_medkit_msgs/msg/Fault.msg). +// The detector reports the drift and does not grade its impact: whether a changed parameter +// matters is a question about the robot, not about the graph, so triage belongs to whoever +// reads the fault. WARN keeps it visible without claiming the run is broken. +constexpr std::uint8_t kParamDriftSeverity = ros2_medkit_msgs::msg::Fault::SEVERITY_WARN; +constexpr double kParamServiceTimeoutSec = 0.5; // per-read timeout, same as the lifecycle GetState read +constexpr double kNegativeCacheTtlSec = 10.0; // unreachable-node backoff +constexpr int kForgetGrace = 2; // missed-sweep grace before forgetting, mirrors WarmupTracker + +// What one transport call costs the WATCHED node, in parameter-service round trips. +// +// Nothing in the build ties these literals to the transport they describe. Compiling the gateway's +// Ros2ParameterTransport into the plugin (see PARAMETER_TRANSPORT_SOURCES in CMakeLists.txt) only +// guarantees that the constants and the code are the same VERSION - a round trip added inside the +// transport would leave every one of them stale and still compile. +// +// Two DIFFERENT tests are needed to hold them true, and neither can do the other's job: +// +// - test/e2e/test_param_roundtrip_e2e.test.py measures what the TRANSPORT really costs, by running +// it against a node that answers the parameter services by hand and counts every request it is +// asked to serve. It reads none of the literals below - it drives the two transport functions +// over the configurations endpoint with this detector switched OFF - so it catches a round trip +// being added to or removed from the transport, and it catches nothing about these numbers. Set +// kBaselineReadRoundTrips to 4 and it still passes. +// - the CHARGE, i.e. that these literals still match that measurement, is held by the timing cases +// in test/test_param_drift_integration.cpp (ABaselineVisitIsChargedTwoRoundTripsAndNotTheColdStartPair, +// TheBudgetIsSpentInRoundTripsNotCalls, AnUndeclaredPinIsChargedLessThanOneThatResolves). They +// time the reader against a configured budget, which is a function of what it charges, so a +// literal moved here moves the elapsed time and they go red. +// +// Every OTHER test in this package proves the detector's arithmetic GIVEN these numbers, and the +// stand-in transport they use counts CALLS. +// +// A baseline visit is one list_parameters(): a list request, then one batched get for every name +// it returned (ros2_parameter_transport.cpp:397 and :420). +constexpr int kBaselineReadRoundTrips = 2; +// The FIRST list_parameters() against a given node costs two MORE, because it primes the +// transport's defaults cache before doing its own read (:364 -> :1158 list + :1190 get). That +// cache has no TTL (:1163 - entries are only evicted under LRU pressure), so the extra pair is a +// one-off per node per gateway lifetime and is deliberately NOT charged here: this knob bounds a +// SUSTAINED sweep, and charging every read for a cost paid once per node would throttle the whole +// run to cover the first rotation. The README states the cold-contact excess instead. +// +// An expect visit is one get_parameter() per pinned name, and each of those is three round trips: +// a name-scoped list, a get, and a describe (:520, :553, :585). The describe is unconditional on +// the success path - the block at :584 has no enclosing guard, only the USE of its result at :586 +// is guarded - so it is part of the price whether or not the descriptor is wanted. There is no +// cold-contact pair here: get_parameter never calls cache_default_values, so a `baseline: false` +// sweep pays the same three on its first read of a node as on every later one. +constexpr int kExpectReadRoundTrips = 3; +// A pin the node does not declare costs ONE round trip: the name-scoped list comes back empty and +// the transport returns NOT_FOUND without reaching the get or the describe (:538). `expect` pins +// are documented as checked on every node that HAS the parameter, so on a mixed graph most nodes +// answer NOT_FOUND for most pins, and charging those the full success price would throttle the +// sweep three times harder than the load it creates. NOT_FOUND has a second, rarer path that costs +// two (:571): the name-scoped list DOES find the name and the get then comes back with no value +// for it, which the GetParameters contract allows, so the transport stops one round trip later. +// The error code is the same on both paths and the caller cannot tell which it took, so the CHARGE +// below is the higher of them and deliberately overstates the common case: a bound may overstate a +// cost, it must never understate one. Both are measured in test_param_roundtrip_e2e. +// +// A parameter the node DECLARES and has not set is not either of those. rclcpp answers one with a +// PARAMETER_NOT_SET value rather than an absent one, so the transport gets a parameter back, pays +// for the unconditional describe and returns SUCCESS carrying null - three round trips, charged as +// kExpectReadRoundTrips. Charging it two would understate what the node pays. +constexpr int kExpectNotFoundRoundTrips = 2; + +// Completed, FAILED read attempts against one armed app before we say so. Counted in attempts and +// never in elapsed ticks. An app the round-robin sweep has not reached yet has made no attempt at +// all, and on a large graph it waits many ticks for its turn: reporting that "no read has +// succeeded" for it would be a bringup warning about a perfectly healthy node, which is what an +// elapsed-tick threshold produced. Ten consecutive failures is past any single dropped reply and +// squarely into "this node does not answer". +constexpr int kUnreadReportAfterFailures = 10; +// A FLOOR in ticks, spent alongside the attempt counts and never instead of them. +// +// An attempt is not a duration. The reader paces itself at max_reads_per_tick round trips per tick +// period, so what an attempt costs in wall clock is read_gap x its charge - and the whole horizon +// therefore scales with the load knob. At the accepted ceiling of 100000 the gap floors at one +// microsecond and sixty-one failed attempts are spent in a couple of milliseconds, so the very +// first ticks of a run write off any node whose parameter service DDS has not finished discovering +// yet. The README's own remedy for a slow sweep - raise max_reads_per_tick - is exactly what +// collapses it. +// +// Ticks are the one quantity the budget does not scale, so they are the floor. Used as a +// CONJUNCTION with the attempt count this can only ever DELAY a report or a give-up, never bring +// one forward, so it does not reintroduce the failure mode that counting attempts was adopted to +// fix: a tick horizon that EXPIRES on a large graph. The values mirror the attempt counts, which at +// the default tick period is ten seconds before an app is named and a minute before it stops +// holding back a clear. +constexpr int kUnreadReportAfterTicks = kUnreadReportAfterFailures; +// Missed sweeps a node may be absent for before we treat it as restarted and drop its captured +// baseline. Deliberately the SAME window WarmupTracker absorbs (kDefaultForgetGrace): that header +// documents that DDS discovery routinely drops a node from a single poll without it restarting, so +// re-baselining on the first miss would re-capture the node's CURRENT - possibly already drifted - +// values on ordinary churn. A confirmed fault would then heal while the parameter is still wrong +// and could never fire again, because the new baseline IS the wrong value. The cost of waiting is +// that a respawn faster than this window keeps the old baseline and reports the operator's change +// as drift until they revert it or the node stays away longer. +constexpr int kBaselineForgetGrace = 2; +// Upper bound on the configurable prune grace. At the 1 s default tick this is already an hour of +// absence before a finding is dropped; beyond it the value is a typo or a unit mix-up, not a choice. +constexpr std::int64_t kMaxPruneGrace = 3600; +// Completed, FAILED read attempts after which an app stops holding back a clear. +// +// A clear asserts that nothing is drifting anywhere, and that is only true if we actually looked +// everywhere. An app the sweep has never ATTEMPTED therefore blocks the clear INDEFINITELY: no +// amount of elapsed time turns an unmeasured node into a measured one, and a horizon in ticks is +// guaranteed to expire on a graph whose rotation is longer than the horizon - the detector would +// then report health it never measured, on exactly the large graphs where the sweep is slowest. +// +// What does justify giving up is evidence: this many attempts were made against the node and every +// one of them failed. That is a node with no working parameter service (a non-rclcpp participant, +// parameter services disabled, a wedged executor), and there is no per-node opt-out for this +// detector, so without a bound one such node would keep a fixed drift reported as CONFIRMED for +// the lifetime of the gateway, with a description naming something that is no longer true. Past +// the bound the app is declared unmeasurable, logged once, and excluded from the blocking set. +// +// The count only grows while the app stays unread and is dropped the moment a read succeeds, so an +// app already given up on can never silently re-enter the blocking set - and one that does become +// readable leaves it through the front door, by being read. +constexpr int kUnmeasurableAfterFailures = 60; +// The tick floor for the give-up, spent alongside the attempt count. See kUnreadReportAfterTicks. +constexpr int kUnmeasurableAfterTicks = kUnmeasurableAfterFailures; +// Consecutive ticks a clear may be withheld before the reason is put in the log. +// +// Withholding is the right thing to do - health that was never measured is not health - but it is +// indistinguishable from the outside from a detector that is simply working and finding nothing: +// no fault is raised, no fault is cleared, and a GRAPH_PARAM_DRIFT already in the store just never +// heals. An operator has no way to tell "the graph is being covered" from "the graph has not been +// covered for the last twenty minutes and never will be", so the detector has to say which. Once +// per episode, and only past a horizon a healthy rotation on a large graph comfortably fits inside. +constexpr int kWithheldClearReportTicks = 60; +// Consecutive ticks an app may stay non-armed while holding a frozen finding. Freezing is right for +// a pause - the drift is still real and the gate only means "do not raise about an unsettled +// entity". But a node that leaves `active` for good is never re-evaluated (not armed) and never +// pruned (still present), so without this the aggregate re-raises its stale finding forever, even +// after an operator fixes the parameter. Mirrors the prune horizon. +constexpr int kFrozenHoldTicks = 60; +// Characters one app may contribute to the aggregated description. Without a per-app bound a single +// node drifting on many parameters fills the whole 480-char cap by itself, and every other app - +// including one whose violation an operator explicitly pinned - is cut regardless of the ordering. +// Ordering decides who goes first; this is what makes going second still mean something. Three apps +// fit comfortably, which is the case worth optimising for: the aggregate exists to say WHICH nodes +// drifted, and the per-parameter detail is a hint, not the record. +constexpr std::size_t kMaxAppDetailChars = 150; + +/// One app the reader has been told to read, tagged with the token of the arming EPISODE it +/// belongs to. +/// +/// The token, not the app id and not the fqn, is what the reader compares before publishing. Both +/// of those are stable across a de-arm and a re-arm, and tick() rewrites `targets` from scratch +/// every tick, so an app whose lifecycle gate closed and re-opened inside one (blocking, seconds +/// long) read window would pass an identity check and have a value sampled mid-transition published +/// as its next reading - which becomes its BASELINE, a false drift no later read can heal, and +/// exactly what the fence exists to prevent. A token is issued once per arming episode from a +/// monotonic counter and never reused, so an interrupted episode can never be mistaken for the one +/// that followed it. +struct ReadTarget { + std::string app_id; + std::string fqn; + std::uint64_t token = 0; +}; + +/// An armed app with no usable read, and the EVIDENCE we have about why. +/// +/// `failed_attempts` is the number of consecutive completed read attempts against this app that +/// failed. Zero means the round-robin sweep has not reached it yet - the app is waiting its turn, +/// nothing is known about it, and nothing may be said about it either. +struct UnreadApp { + std::string app_id; + int failed_attempts = 0; +}; + +/// One-shot latches for the two things we say about an app that will not answer, plus how long the +/// app has been in that state. Kept per app and dropped the moment the app leaves the unread set, +/// so a run of silence is reported once rather than every tick, and an app that starts answering +/// again can be reported afresh if it later stops. +struct UnreadLatch { + bool reported = false; ///< the "no read has succeeded" warning has been logged + bool gave_up = false; ///< the unmeasurable warning has been logged + /// Consecutive ticks this app has been armed with no usable read. The FLOOR half of both + /// horizons: see kUnreadReportAfterTicks for why an attempt count alone is not a duration. + int ticks = 0; +}; + +// Reading a node's parameters is a BLOCKING service round-trip (up to kParamServiceTimeoutSec +// each, and observed to hang PAST that on a real Nav2 node stuck mid-transition, since the +// underlying DDS recv does not always honour the spin timeout). Doing it inline on the tick +// thread stalls the WHOLE detection loop - node_death, tf_stale, everything - the moment ONE +// node's parameter service is slow. So the reads run on a dedicated background thread and the +// tick thread only ever reads the cache. State shared between the two threads lives here, +// behind a shared_ptr so a reader still inside a read keeps it alive. Teardown joins the +// thread rather than detaching it, and shuts the transport down first so a blocking read +// returns instead of holding the join (see the destructor); a detached reader would still be +// making service calls while rclcpp tears the context down, which aborts rather than throws. +struct SharedIo { + std::mutex mutex; + std::condition_variable cv; + bool stop = false; + bool reader_stopped = false; ///< set by the reader when it leaves the loop (no more rcl ops) + bool baseline_capture = true; ///< snapshot of policy config (set-once at reader start) + std::vector expect_names; ///< pinned params to read when baseline_capture is false + /// The ROS-neutral base, not the concrete ROS 2 transport. The detector builds the real one by + /// default; a test can inject a stand-in to control read latency, force an empty batch, or fail + /// construction outright - none of which is reachable through a live node. + std::unique_ptr transport; // reader-thread only + std::vector targets; // armed apps to read, this episode + std::uint64_t next_token = 1; ///< monotonic; never reused + std::map> observed; // app_id -> last usable read + std::set have; ///< app_ids with a usable cached read (distinct from "read empty") + /// app_id -> consecutive read attempts that COMPLETED and failed. Written by the reader under + /// this mutex, read by the tick thread under it. An app absent from this map and absent from + /// `have` has never been attempted at all, and that is the distinction the clear rests on: `have` + /// says "we looked and it answered", an entry here says "we looked and it did not", and neither + /// says "we have not looked yet". Erased on the first successful read. + std::map failed_reads; + /// Pacing for the reader thread (set-once at reader start, alongside baseline_capture). + /// `max_reads_per_tick` service round trips at the watched nodes per `tick_interval_ms` - one + /// transport call is several of those, see reader_loop(). + int max_reads_per_tick = 8; + int tick_interval_ms = kDefaultTickIntervalMs; +}; + +/// Blocking read of `fqn`'s parameters into `observed`. Runs ONLY on the reader thread (it is +/// the sole user of io.transport, so the transport needs no lock of its own here). Returns true +/// when `observed` is a usable, complete view; false when the node should be skipped (a non- +/// NOT_FOUND transport error, or a stop mid-read) so evaluate() never sees a partial view and +/// spuriously clears a real drift. Mirrors the original inline read_params semantics exactly. +/// +/// `round_trips` accumulates what this visit is CHARGED against the WATCHED node in +/// parameter-service round trips - not transport calls, which are 1-3 round trips each, and never +/// below what the node really paid (see kExpectNotFoundRoundTrips). reader_loop turns it into the +/// pacing debt for the visit, so the knob bounds the load a node really sees. +bool read_params_io(SharedIo & io, const std::string & fqn, std::map & observed, + int & round_trips) { + if (io.baseline_capture) { + round_trips += kBaselineReadRoundTrips; + const auto result = io.transport->list_parameters(fqn); // one list, then one batched get + if (!result.success || !result.data.is_array() || result.data.empty()) { + // An empty array is a SUCCESSFUL response that carries no view - the transport documents it + // as legitimate, e.g. when a parameter declaration races the list. Treating it as "nothing + // drifts here" is what the contract above exists to prevent: evaluate() would compare + // against no observations at all, find nothing, and the tick would clear a real drift. + // Every ROS 2 node declares use_sim_time, so a genuinely empty parameter set is not a case + // worth trading that away for. + return false; + } + for (const auto & p : result.data) { + if (p.contains("name") && p.contains("value")) { + // Through param_entry_value, not straight off "value": the entry also carries the TYPE, and + // for a parameter the node declared and never set that is the only thing separating it from + // a NaN double - both of which are `null` by the time a description is written. See + // param_entry_value() and unset_param_value(). + observed.emplace(p["name"].get(), param_entry_value(p)); + } + } + return true; + } + for (const auto & name : io.expect_names) { + { + std::lock_guard lk(io.mutex); + if (io.stop) { + return false; // shutting down mid-read: incomplete view -> skip (no spurious clear) + } + } + const auto result = io.transport->get_parameter(fqn, name); + // Charged from the OUTCOME rather than blind, because a name the node never declared costs it + // a fraction of a successful read (see kExpectNotFoundRoundTrips). A call that throws is not + // charged at all - reader_loop's own floor of one slot covers that, and a throw here means the + // context is being torn down, not that the sweep is running hot. + round_trips += result.error_code == ros2_medkit_gateway::ParameterErrorCode::NOT_FOUND ? kExpectNotFoundRoundTrips + : kExpectReadRoundTrips; + if (result.success && result.data.contains("value")) { + // Same reason as the baseline branch above: a SUCCESS carrying null is a parameter the node + // declared and never set, and it must not reach the description as the `null` a NaN prints. + observed.emplace(name, param_entry_value(result.data)); + } else if (result.error_code != ros2_medkit_gateway::ParameterErrorCode::NOT_FOUND) { + return false; // unreadable this tick -> skip, do not risk a spurious clear + } + } + return true; +} + +/// Background loop: round-robins the tick thread's current `targets`, doing one blocking read +/// at a time and publishing the result into the cache. A hung read stalls only THIS thread, not +/// the tick loop; on stop it shuts the transport down and exits, releasing the shared_ptr. +/// +/// PACED at max_reads_per_tick service ROUND TRIPS per tick period - the unit the watched node +/// actually pays in, not transport calls, which are 1-3 round trips each (see +/// kBaselineReadRoundTrips / kExpectReadRoundTrips / kExpectNotFoundRoundTrips). Without pacing +/// this loop issues round-trips +/// back to back for the whole life of the gateway - tick() re-arms `targets` with every present +/// app on every tick, so there is always work and the cv predicate is always satisfied. Every one +/// of those round trips is served on the customer node's own executor. The short-circuit paths +/// turn it into a pure busy spin: a node in the transport's negative cache returns +/// SERVICE_UNAVAILABLE immediately, and `baseline: false` with an empty `expect` does zero IO and +/// returns true, so with all targets in either state the thread burns a core doing nothing. The +/// budget is also what the documented coverage latency of +/// ceil(round_trips_per_visit * node_count / max_reads_per_tick) ticks is computed from. +/// +/// The cursor is POSITIONAL, not an identity. `idx` indexes a vector the tick thread rewrites from +/// scratch every tick, in sorted app-id order, so a target's slot is not stable: an app arriving +/// that sorts earlier shifts every later one down a slot, and one leaving shifts them up, which +/// re-reads an app just visited or jumps past the app that was due. Nobody is starved by that - +/// `idx` still advances by one per read, so every index of the CURRENT list is visited within +/// targets.size() reads - but the rotation is not a guaranteed round of exactly one read per app, +/// and under continuous node-name churn an individual app's turn can keep moving. Nothing the +/// detector SAYS about an app is allowed to rest on how long that app has gone unread; see +/// kUnreadReportAfterFailures and kUnmeasurableAfterFailures, which count failed ATTEMPTS. +/// +/// Every completed visit also records its OUTCOME against the app (see `failed_reads`), which is +/// what lets the tick thread tell "this node will not answer" from "the sweep has not got to it +/// yet" without appealing to how much time has passed. +void reader_loop(std::shared_ptr io) { + std::size_t idx = 0; + for (;;) { + ReadTarget target; + std::chrono::microseconds min_gap{0}; + { + std::unique_lock lk(io->mutex); + io->cv.wait(lk, [&] { + return io->stop || !io->targets.empty(); + }); + if (io->stop) { + break; + } + target = io->targets[idx % io->targets.size()]; + ++idx; + min_gap = read_gap(io->tick_interval_ms, io->max_reads_per_tick); + } + const auto read_started = std::chrono::steady_clock::now(); + std::map observed; + bool ok = false; + int round_trips = 0; + try { + ok = read_params_io(*io, target.fqn, observed, round_trips); + } catch (const std::exception &) { + // A transport/rclcpp call can throw when the context is invalidated during shutdown, or + // on a malformed service path. An escape would std::terminate the process (the reader has + // no outer handler unlike the tick loop), so swallow it: drop this read and let the loop + // re-check stop. Matches the guard the plugin puts around each detector->tick(). + ok = false; + } + { + std::unique_lock lk(io->mutex); + if (io->stop) { + break; + } + // Record the outcome only if this arming EPISODE is still current. The read ran with the + // lock released and can take seconds, during which the tick thread may have dropped this + // app - because its lifecycle gate closed, or because prune_vanished forgot it - and may + // even have re-armed it since. Publishing anyway is not merely stale, it is harmful in + // three ways: a value sampled mid-transition becomes the app's baseline when it re-arms, + // which is a false drift that can never heal; a read landing after the prune re-inserts an + // entry whose miss counter is gone, so no later prune can reach it and the maps grow + // without bound under node-name churn; and a failure charged to an episode that is over + // pushes an app that is now perfectly readable towards being given up on. + // + // Comparing the token rather than the app id is what closes the de-arm/re-arm case: the id + // and the fqn are both stable across it, so either would let a mid-transition sample land. + const bool current_episode = std::any_of(io->targets.begin(), io->targets.end(), [&target](const ReadTarget & t) { + return t.token == target.token; + }); + if (current_episode) { + if (ok) { + io->observed[target.app_id] = std::move(observed); + io->have.insert(target.app_id); + io->failed_reads.erase(target.app_id); + } else { + // An ATTEMPT that completed and failed. This - not elapsed time - is the only thing + // that ever justifies giving up on an app and letting a clear through without it. + ++io->failed_reads[target.app_id]; + // And the app stops counting as MEASURED. A cached read is evidence about the instant it + // was taken, not a standing licence: once a read fails, what is cached says nothing about + // the node's current parameters. Leaving the app in `have` kept it in the evaluated set + // for ever off a snapshot from before it went silent, so it never joined the unread set, + // never held back a clear, and neither unread warning could ever fire about it - the + // detector reported "nothing is drifting anywhere" on a node it had not read in hours. + // `observed` is deliberately kept: it is only ever read together with `have`, and the + // next successful read overwrites it. + io->have.erase(target.app_id); + } + } + // Hold the budget: wait out whatever is left of this visit's slots. The budget is spent in + // service ROUND TRIPS at the watched node - not in node visits, and not in transport calls + // either: one list_parameters() is two round trips there and one get_parameter() is three on + // a name the node declares, and an `expect` visit makes one of the latter per pinned name. + // Charging a slot per visit, + // or even per call, would let the sweep put several times the promised load on a node while + // the knob an operator lowered to protect it reports being honoured. Charge what was + // actually spent. Waiting on the cv (not sleeping) keeps stop responsive, so shutdown is not + // delayed by the pacing. + const auto owed = min_gap * std::max(1, round_trips); + const auto elapsed = std::chrono::steady_clock::now() - read_started; + if (elapsed < owed) { + io->cv.wait_for(lk, owed - elapsed, [&] { + return io->stop; + }); + } + if (io->stop) { + break; + } + } + } + // Signal that the reader has left the loop and will do no more rcl ops. A pre-shutdown + // callback waits on this (see tick()) so the thread is idle BEFORE rclcpp invalidates the + // context - a blocking rcl call caught mid-context-teardown would SIGABRT, not throw, so it + // cannot merely be try/catch'd. The transport itself is torn down later, in the destructor. + std::lock_guard lk(io->mutex); + io->reader_stopped = true; + io->cv.notify_all(); +} +} // namespace + +/// Self-capturing parameter-drift detector. See the package design doc. +/// Parameter reads run on a background thread (see SharedIo) so a slow/stuck node's blocking +/// read can never stall the tick loop; tick() only evaluates the cache. +class ParamDriftDetector : public Detector { + public: + ParamDriftDetector() : policy_(ParamDriftConfig{}) { + } + + bool set_parameter_transport_factory_for_test(const ParameterTransportFactory & factory) override { + transport_factory_ = factory; + return true; + } + ~ParamDriftDetector() override { + if (io_) { + // Deregister the pre-shutdown callback first (it holds a weak_ptr to io_, so it is safe + // either way, but leaving a stale callback on the context is untidy). No-op if it already + // ran during a normal shutdown. + if (context_) { + context_->remove_pre_shutdown_callback(pre_shutdown_handle_); + } + { + std::lock_guard lk(io_->mutex); + io_->stop = true; + } + io_->cv.notify_all(); + // Shut the transport down FIRST: it interrupts any in-flight blocking read (its spin has + // a bounded timeout, see Ros2ParameterTransport::shutdown) so the reader returns promptly. + // Then JOIN - never detach: a detached reader still doing a service call races rclcpp + // teardown and SIGABRTs. The join here runs before the node/context are torn down (the + // plugin destroys detectors before rclcpp shutdown), so the reader finishes cleanly. + if (io_->transport) { + io_->transport->shutdown(); + } + if (reader_thread_.joinable()) { + reader_thread_.join(); + } + } + } + ParamDriftDetector(const ParamDriftDetector &) = delete; + ParamDriftDetector & operator=(const ParamDriftDetector &) = delete; + ParamDriftDetector(ParamDriftDetector &&) = delete; + ParamDriftDetector & operator=(ParamDriftDetector &&) = delete; + + std::string id() const override { + return "param_drift"; + } + + /// Distinct app ids this detector holds ANY per-app bookkeeping for, across both its own maps + /// and the ones it shares with the reader thread. + /// + /// Every one of these is keyed by app id, and node names are not a bounded set: discovery has no + /// hidden-node filter, so each `ros2` CLI invocation appears once under a name nothing will ever + /// use again. A map that no prune path reaches therefore grows for the life of the process, and + /// nothing about a leak is visible in a fault, a log line or a description - which is why it + /// needs a number a test can assert on rather than a behaviour a test can observe. + std::size_t tracked_count_for_test() const override { + std::set ids; + const auto collect = [&ids](const auto & container) { + for (const auto & entry : container) { + ids.insert(entry.first); + } + }; + collect(misses_); + collect(drifted_); + collect(reported_); + collect(pinned_counts_); + collect(unread_latches_); + collect(unarmed_ticks_); + collect(bound_fqn_); + if (io_) { + std::lock_guard lk(io_->mutex); + collect(io_->observed); + collect(io_->failed_reads); + ids.insert(io_->have.begin(), io_->have.end()); + for (const auto & t : io_->targets) { + ids.insert(t.app_id); + } + } + return ids.size(); + } + + void configure(const nlohmann::json & config) override { + policy_ = ParamDriftPolicy{ParamDriftConfig::from_json(config)}; + // prune_grace is injected into EVERY detector's config by the plugin and is exempt from the + // unknown-key warning, so ignoring it here would accept the key with no warning and no + // effect - the exact silent failure the README says cannot happen. + own_warnings_.clear(); + prune_grace_ = kForgetGrace; + if (config.contains("prune_grace")) { + // Range-check on the WIDE value, like max_reads_per_tick. get() truncates first, so + // 4294967296 would arrive as 0 and drop findings on the first missed sweep - the flapping + // prune_grace exists to prevent - and 2^63-1 would arrive as -1 and be dropped in silence. + const auto & value = config["prune_grace"]; + const std::int64_t wide = value.is_number_integer() ? value.get() : -1; + if (wide >= 0 && wide <= kMaxPruneGrace) { + prune_grace_ = static_cast(wide); + } else { + own_warnings_.push_back("'prune_grace' must be an integer in 0.." + std::to_string(kMaxPruneGrace) + + "; keeping " + std::to_string(kForgetGrace)); + } + } + // Plugin-injected: the reader thread paces itself at max_reads_per_tick service round trips + // per tick period, so it needs to know how long a tick period is. + if (config.contains("tick_interval_ms") && config["tick_interval_ms"].is_number_integer() && + config["tick_interval_ms"].get() > 0) { + tick_interval_ms_ = config["tick_interval_ms"].get(); + } + warnings_logged_ = false; + } + + void tick(DetectorContext & ctx) override { + if (!ctx.gateway_node || !ctx.snapshot) { + return; + } + // Lazily build the shared IO + reader thread on the first tick (configure() has no node). + // + // `io_` is assigned LAST, once the transport and the thread both exist. Assigning it first + // and filling it in afterwards means any throw from this block - std::thread under resource + // pressure, an rclcpp throw while building the transport's hidden node, bad_alloc - leaves a + // half-built `io_` behind. The plugin's per-detector guard logs that throw once, and every + // later tick then sees a non-null `io_`, skips this block, and runs against a detector that + // has no reader at all: `have` stays empty forever. Nothing distinguishes that from a healthy + // detector at any call site. Leaving `io_` null instead means the next tick simply retries. + if (!io_) { + auto fresh = std::make_shared(); + fresh->baseline_capture = policy_.config().baseline_capture; + fresh->max_reads_per_tick = policy_.config().max_reads_per_tick; + fresh->tick_interval_ms = tick_interval_ms_; + for (const auto & [name, _] : policy_.config().expect) { + fresh->expect_names.push_back(name); + } + fresh->transport = transport_factory_ ? transport_factory_(ctx.gateway_node) + : std::make_unique( + ctx.gateway_node, kParamServiceTimeoutSec, kNegativeCacheTtlSec); + reader_thread_ = std::thread(reader_loop, fresh); + io_ = std::move(fresh); + // Register a pre-shutdown callback: it runs while the context is STILL valid (before + // rclcpp invalidates it, even on a SIGINT), signals the reader to stop and waits (bounded) + // for it to leave its loop. That guarantees the reader is not mid-blocking-rcl-call when + // the context dies - which would SIGABRT (rcl aborts, it does not throw, so it cannot be + // caught). The bounded wait keeps a genuinely hung read from hanging shutdown forever. + context_ = ctx.gateway_node->get_node_base_interface()->get_context(); + std::weak_ptr weak_io = io_; + pre_shutdown_handle_ = context_->add_pre_shutdown_callback([weak_io] { + auto io = weak_io.lock(); + if (!io) { + return; + } + std::unique_lock lk(io->mutex); + io->stop = true; + io->cv.notify_all(); + io->cv.wait_for(lk, std::chrono::seconds(2), [&] { + return io->reader_stopped; + }); + }); + } + + log_config_warnings_once(ctx); + + // Present apps (id + fqn) from the entity snapshot. + std::vector> apps; // (app.id, fqn) + for (const auto & a : ctx.snapshot->apps) { + const std::string fqn = a.effective_fqn(); // bound_fqn or derived; see app.hpp + if (!fqn.empty()) { + apps.emplace_back(a.id, fqn); + } + } + std::sort(apps.begin(), apps.end()); + + forget_rebound_apps(apps); + prune_vanished(apps); + + // Armed targets (fast, NO I/O): only capture/evaluate a node that is armed + lifecycle-ok + // The blocking read for each is done off-thread by reader_loop. + const std::string self_fqn = ctx.gateway_node->get_fully_qualified_name(); + std::vector> armed; + std::vector denied; // gate-denied apps still inside the frozen hold + for (const auto & [app_id, fqn] : apps) { + if (fqn == self_fqn) { + // The GATEWAY's own node, matched on its fully qualified name. Reading it would have the + // process interrogate itself through the hidden client node the transport spins, and its + // own parameters would then start reporting drift. + continue; + } + if (reliability_allows(ctx.gate, app_id)) { + unarmed_ticks_.erase(app_id); + armed.emplace_back(app_id, fqn); + } else { + if (++unarmed_ticks_[app_id] > kFrozenHoldTicks) { + // Held long enough that this is not a pause: release the finding so a fixed parameter can + // stop being reported. It re-raises from a fresh read if the app ever arms again. Past + // this point the app also stops holding back a clear: the hold is what bounds an app the + // gate will never open for, and without a bound one permanently inactive node would keep + // GRAPH_PARAM_DRIFT from ever healing, with no read it could ever make to get out of it. + drifted_.erase(app_id); + pinned_counts_.erase(app_id); + } else { + denied.push_back(app_id); + } + // Do NOT erase this app's drift. The aggregate raise is gated on the aggregate's own + // source, never on app_id, so an unarmed app's finding is not "suppressed anyway" - + // dropping it here is the only thing that would suppress it, and dropping it feeds the + // CLEAR path, which carries no gate at all. A routine lifecycle pause would then heal a + // drift that is still present. The entry stays frozen: not re-evaluated while unarmed, + // not removed either. Removal is the prune path's job, when the app actually goes away. + continue; + } + } + + // Hand the reader thread the current armed set, then evaluate drift from the CACHED reads. + std::map> cached; + std::vector unread; // armed apps with no usable read yet - see emit_aggregated + /// Gate-denied apps with no usable read either. The gate decides whether we may RAISE about an + /// entity; it says nothing about whether we have measured it, and an app that is denied is one + /// the reader is not even allowed to target. "The round-robin has not reached it" and "the gate + /// has not armed it" are the same state - nothing is known - and the clear may not be emitted + /// out of either. Without this, every app is denied for the first `warmup_cycles` ticks after a + /// gateway or plugin start, the armed set is empty, and the detector clears on every one of + /// those ticks before it has issued a single round trip. + std::vector denied_unmeasured; + { + std::lock_guard lk(io_->mutex); + // Carry an app's arming token forward only while it stays armed WITHOUT interruption. An app + // missing from the previous target list is starting a new episode and gets a fresh token, so + // a read still in flight from its last one cannot publish into it. The fqn is part of the + // key for the same reason: a re-bind to a different node under the same app id is a new + // episode too, even though the id never changed. + std::map, std::uint64_t> previous; + for (const auto & t : io_->targets) { + previous.emplace(std::make_pair(t.app_id, t.fqn), t.token); + } + std::vector next; + next.reserve(armed.size()); + for (const auto & [app_id, fqn] : armed) { + const auto pit = previous.find(std::make_pair(app_id, fqn)); + next.push_back(ReadTarget{app_id, fqn, pit != previous.end() ? pit->second : io_->next_token++}); + } + io_->targets = std::move(next); + for (const auto & [app_id, fqn] : armed) { + (void)fqn; + auto it = io_->observed.find(app_id); + if (it != io_->observed.end() && io_->have.count(app_id) > 0) { + cached[app_id] = it->second; // copy out under the lock; evaluate below without it held + } else { + const auto fit = io_->failed_reads.find(app_id); + unread.push_back(UnreadApp{app_id, fit != io_->failed_reads.end() ? fit->second : 0}); + } + } + for (const auto & app_id : denied) { + if (io_->have.count(app_id) == 0) { + denied_unmeasured.push_back(app_id); + } + } + } + io_->cv.notify_one(); + + // Advance the per-app unread bookkeeping BEFORE the blocking set is built: the give-up is a + // conjunction of failed attempts and elapsed ticks, so the tick count has to already exist when + // the set that depends on it is computed. + advance_unread_latches(unread); + + // Apps that hold back a clear: no usable read, and no grounds to stop waiting for one. An app + // the sweep has not attempted (failed_attempts == 0) is in here for as long as that stays true, + // however long the rotation takes - see kUnmeasurableAfterFailures. + std::vector blocking; + for (const auto & app : unread) { + if (!given_up_on(app)) { + blocking.push_back(app.app_id); + } + } + blocking.insert(blocking.end(), denied_unmeasured.begin(), denied_unmeasured.end()); + + for (const auto & [app_id, observed] : cached) { + std::size_t pinned = 0; + const auto d = policy_.evaluate(app_id, observed, /*capture_allowed=*/true, &pinned); + if (!d.empty()) { + drifted_[app_id] = d; + pinned_counts_[app_id] = pinned; + } else { + drifted_.erase(app_id); + pinned_counts_.erase(app_id); + } + } + + report_unmatched_pins_once(ctx, cached, blocking.empty()); + emit_aggregated(ctx, unread, blocking); + } + + private: + /// Name `expect` pins that no app declares, once each. A pin whose parameter name is misspelled + /// comes back NOT_FOUND from every read, and that is deliberately carved out as "not a drift" - + /// a node is not at fault for a parameter it never had. The cost is that such a pin is inert + /// forever, and at runtime inert is indistinguishable from a pin that is checked and passes + /// every time. That is the same silent failure this detector exists to rule out, so it has to + /// say something. + /// + /// Said only once the sweep has COVERED the graph, for the same reason a clear is: only one node + /// is visited per rotation slot, so a perfectly good pin is legitimately unseen for as long as + /// the sweep has not reached the node that declares it. Telling an operator to go and fix a name + /// that is already correct is the same defect as clearing a fault we never measured, and a + /// threshold in ticks fails here in exactly the same way - it expires on the large graphs where + /// the rotation is longest. + /// + /// `swept` is the clear's own condition: every armed app has either been read or been given up + /// on after repeated failures. `cached` non-empty is still required on top of it, because a graph + /// where every app was given up on is swept in name only and says nothing about any pin. + void report_unmatched_pins_once(const DetectorContext & ctx, + const std::map> & cached, + bool swept) { + if (cached.empty() || !swept) { + return; + } + for (const auto & [pin, expected] : policy_.config().expect) { + (void)expected; + const bool declared = std::any_of(cached.begin(), cached.end(), [&pin](const auto & entry) { + return entry.second.count(pin) != 0; + }); + if (declared) { + unmatched_pins_reported_.erase(pin); // a later disappearance is worth saying again + continue; + } + // Node check FIRST: latching without logging would silence the pin for the whole run. + if (!ctx.gateway_node || !unmatched_pins_reported_.insert(pin).second) { + continue; + } + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "param_drift: no app declares '%s', the parameter pinned under 'expect', so that " + "pin can never report a drift; check the name against the node's own parameters", + pin.c_str()); + } + } + + void log_config_warnings_once(const DetectorContext & ctx) { + if (warnings_logged_ || !ctx.gateway_node) { + return; + } + warnings_logged_ = true; + for (const auto & w : policy_.config().warnings) { + RCLCPP_WARN(ctx.gateway_node->get_logger(), "param_drift config: %s", w.c_str()); + } + // Keys the DETECTOR parses rather than the policy (prune_grace) surface here, so an operator + // finds them in the same place and a rejected value is never silent. + for (const auto & w : own_warnings_) { + RCLCPP_WARN(ctx.gateway_node->get_logger(), "param_drift config: %s", w.c_str()); + } + } + + /// Per-app unread bookkeeping for THIS tick: drop the entries of apps that have left the unread + /// set, and advance the tick counter of the ones still in it. + /// + /// The drop is load-bearing and is why this must run on every tick, including ticks where nothing + /// is unread. The latches are one-shot per RUN of failures, not once per process: an app that + /// goes quiet, recovers and goes quiet again has failed twice, and the second time is worth + /// saying to the operator who fixed it the first time. A latch left behind silences that app for + /// the life of the gateway, and silence is also what a working latch looks like. + void advance_unread_latches(const std::vector & unread) { + std::set still_unread; + for (const auto & app : unread) { + still_unread.insert(app.app_id); + } + for (auto it = unread_latches_.begin(); it != unread_latches_.end();) { + it = still_unread.count(it->first) == 0 ? unread_latches_.erase(it) : std::next(it); + } + for (const auto & app : unread) { + ++unread_latches_[app.app_id].ticks; + } + } + + /// Consecutive ticks `app_id` has been armed with no usable read. Zero when it is not in the + /// unread set at all. + int unread_ticks(const std::string & app_id) const { + const auto it = unread_latches_.find(app_id); + return it != unread_latches_.end() ? it->second.ticks : 0; + } + + /// Whether this app has stopped holding back a clear. BOTH counters have to be past the bound: + /// the attempts, because only evidence justifies giving up, and the ticks, because an attempt is + /// not a duration and the attempt horizon alone is scaled by the load knob (kUnreadReportAfterTicks). + /// Strictly past both, matching the give-up warning: at exactly the bound the app still blocks. + bool given_up_on(const UnreadApp & app) const { + return app.failed_attempts > kUnmeasurableAfterFailures && unread_ticks(app.app_id) > kUnmeasurableAfterTicks; + } + + /// Name the apps that are holding back a clear, once each, so the reason a GRAPH_PARAM_DRIFT + /// is not healing is readable instead of having to be inferred. + /// + /// Only apps we ATTEMPTED and failed to read are ever named. An app the round-robin has not + /// reached yet is silent for a reason that is entirely ours, and saying "no parameter read has + /// succeeded for it" would be a bringup warning about a healthy node on every graph whose + /// rotation outlasts the threshold. Reported once per app per run of failures; the latches are + /// created and dropped by advance_unread_latches, which tick() runs first. + void report_unread_once(const DetectorContext & ctx, const std::vector & unread) { + for (const auto & app : unread) { + if (app.failed_attempts <= 0) { + continue; // never attempted: there is nothing to report and nothing to give up on + } + const auto lit = unread_latches_.find(app.app_id); + if (lit == unread_latches_.end()) { + continue; // advance_unread_latches owns this map; it always has an entry for an unread app + } + auto & latch = lit->second; + if (app.failed_attempts >= kUnreadReportAfterFailures && latch.ticks >= kUnreadReportAfterTicks && + !latch.reported && ctx.gateway_node) { + latch.reported = true; + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "param_drift: %d consecutive parameter reads of '%s' have failed; its parameters " + "are unverified and GRAPH_PARAM_DRIFT will not clear while that is true", + app.failed_attempts, app.app_id.c_str()); + } + // Strictly past the bound, matching the blocking set built in tick(): at exactly the bound + // the app still blocks, so announcing that we gave up on it would be premature. + if (given_up_on(app) && !latch.gave_up && ctx.gateway_node) { + latch.gave_up = true; + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "param_drift: giving up on '%s' after %d failed parameter reads; it no longer " + "holds back GRAPH_PARAM_DRIFT from clearing, and its parameters stay unverified", + app.app_id.c_str(), kUnmeasurableAfterFailures); + } + } + } + + /// Say, once per episode, that a clear is being withheld and why. + /// + /// Withholding is correct - health that was never measured is not health - but from the outside + /// it is indistinguishable from a detector that is working and finding nothing: no raise, no + /// clear, and a GRAPH_PARAM_DRIFT already in the store simply never heals. The apps holding it + /// back are frequently ones report_unread_once may not name, because nothing has been ATTEMPTED + /// against them (they are waiting their turn, or the gate has not armed them), so without this + /// the log is silent about the one thing an operator needs in order to act. + void report_withheld_clear(const DetectorContext & ctx, const std::vector & blocking) { + ++withheld_ticks_; + if (withheld_ticks_ <= kWithheldClearReportTicks || withheld_reported_ || !ctx.gateway_node) { + return; + } + withheld_reported_ = true; + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "param_drift: nothing has been found drifting for %d consecutive ticks, but " + "GRAPH_PARAM_DRIFT is being withheld from clearing because %zu app(s) still have no " + "usable parameter read, starting with '%s'. Until every armed app has been read or " + "given up on, the graph has not been measured and no clear is emitted; raise " + "'max_reads_per_tick' or shorten 'tick_interval_ms' if the sweep is simply too slow " + "for the size of this graph", + withheld_ticks_, blocking.size(), blocking.front().c_str()); + } + + /// Drop everything an app's PREVIOUS binding left behind when its fqn changes underneath it. + /// + /// The reader already treats (app_id, fqn) as the identity of an arming episode, so a read in + /// flight from the old binding cannot publish into the new one - but nothing reset the state the + /// old binding had already published. The new node's first reading was then compared against a + /// baseline a DIFFERENT node happened to hold, which is a drift no later read can heal because + /// the reference itself belongs somewhere else, and it inherited the old node's failure count, + /// so a perfectly readable node could be given up on after a handful of its own attempts. + void forget_rebound_apps(const std::vector> & apps) { + for (const auto & [app_id, fqn] : apps) { + const auto it = bound_fqn_.find(app_id); + if (it == bound_fqn_.end()) { + bound_fqn_.emplace(app_id, fqn); + continue; + } + if (it->second == fqn) { + continue; + } + it->second = fqn; + policy_.forget(app_id); + drifted_.erase(app_id); + pinned_counts_.erase(app_id); + unread_latches_.erase(app_id); + if (io_) { + std::lock_guard lk(io_->mutex); + io_->observed.erase(app_id); + io_->have.erase(app_id); + io_->failed_reads.erase(app_id); + io_->targets.erase(std::remove_if(io_->targets.begin(), io_->targets.end(), + [&app_id](const ReadTarget & t) { + return t.app_id == app_id; + }), + io_->targets.end()); + } + } + } + + /// Two different horizons, on purpose. + /// + /// The BASELINE is dropped once the app has been absent from kBaselineForgetGrace + 1 + /// consecutive sweeps - long enough that the absence is no longer explained by the discovery + /// churn WarmupTracker documents. Dropping it on the FIRST missed sweep would re-capture the + /// node's current, possibly already drifted values on an ordinary dropped poll: the confirmed + /// fault would heal while the parameter is still wrong and could never fire again, because the + /// new baseline IS the wrong value. The cost is the other direction - a respawn faster than the + /// window keeps the pre-restart baseline and reports the operator's own change as drift until + /// they revert it. See kBaselineForgetGrace, and the README's recovery-path paragraph. + /// + /// The FINDING is dropped only after prune_grace consecutive absences. Removing it on the first + /// missed sweep would clear the aggregated fault on any one-tick discovery gap, which is + /// flapping, not healing. + /// + /// The two horizons are INDEPENDENTLY configurable and prune_grace can be the shorter one - `0` + /// and `1` are both inside the documented range - so the baseline forget cannot simply wait for + /// the churn window and hope the counter survives that long. Dropping the counter is what makes + /// any later cleanup unreachable: with prune_grace below the churn window the forget would never + /// run for any app, ever, leaving both maps growing without bound under node-name churn and every + /// returning node compared against a stale baseline. So the forget happens at the churn window OR + /// on the tick the counter is dropped, whichever comes first, and a latch keeps it one-shot. + void prune_vanished(const std::vector> & apps) { + std::set present; + for (const auto & [id, fqn] : apps) { + present.insert(id); + } + for (auto it = misses_.begin(); it != misses_.end();) { + if (present.count(it->first) > 0) { + it->second.misses = 0; + it->second.baseline_forgotten = false; + ++it; + continue; + } + ++it->second.misses; + const bool dropping = it->second.misses > prune_grace_; + if (!it->second.baseline_forgotten && (it->second.misses >= kBaselineForgetGrace + 1 || dropping)) { + // Past the churn-absorbing window (or out of ticks in which to do it at all), so treat this + // as a restart rather than a dropped poll: forget the baseline and the cached read so the + // node's return starts from a fresh capture. + it->second.baseline_forgotten = true; + policy_.forget(it->first); + if (io_) { + std::lock_guard lk(io_->mutex); + io_->observed.erase(it->first); + io_->have.erase(it->first); + io_->failed_reads.erase(it->first); + // And from targets, in the SAME critical section. tick() rewrites targets further down, + // so leaving the app there would let a read completing inside this tick republish the + // values just erased - they would become the baseline on return, which is the false, + // unhealable drift the reader's episode fence exists to prevent. + io_->targets.erase(std::remove_if(io_->targets.begin(), io_->targets.end(), + [&it](const ReadTarget & t) { + return t.app_id == it->first; + }), + io_->targets.end()); + } + } + if (dropping) { + drifted_.erase(it->first); + pinned_counts_.erase(it->first); + // The frozen-hold counter goes with the finding it holds. Nothing else erases it off the + // arm path, and discovery has no hidden-node filter, so every short-lived `ros2` CLI node + // would otherwise leave one entry behind for the life of the process. + unarmed_ticks_.erase(it->first); + // Same for the remembered binding: it is written for every present app and nothing else + // reaches it, so it leaks under node-name churn exactly as the counter above did. + bound_fqn_.erase(it->first); + it = misses_.erase(it); + } else { + ++it; + } + } + for (const auto & id : present) { + misses_.emplace(id, AbsenceCounter{}); + } + } + + /// Level-triggered aggregation: one graph-level fault carrying every app's drift, through the + /// same helper qos_mismatch and orphan use. Going through AggregatedFault rather than building + /// the text here is what keeps three properties this detector previously got wrong: the fault + /// is raised under graph_source_id() (a constant, owned entity - a host Component id would + /// match no entity scope at all, so the fault would be reachable from no endpoint), the + /// description stops being built once it exceeds the cap instead of concatenating the whole + /// graph first, and the cap itself is the helper's constant rather than a second copy of 480. + /// + /// `order` carries the priority: entries an operator explicitly pinned with `expect` come + /// first, then self-captured drift, and within each group the newly detected apps precede the + /// ones already reported. Without that, a capped description can hide exactly the two things + /// worth reading - the pin the operator declared, and whatever just changed. + void emit_aggregated(DetectorContext & ctx, const std::vector & unread, + const std::vector & blocking) { + // A clear asserts that nothing is drifting anywhere. That is only true if we actually LOOKED + // everywhere. drifted_ is built from cached reads alone, so an app whose parameter service + // never answers contributes nothing to it - and without this guard an empty drifted_ would + // be reported as health the detector never measured, every tick, forever. AggregatedFault + // decides raise-versus-clear from emptiness alone, so the guard has to live here: when there + // is nothing to report AND something we could not read, emit nothing at all. + // + // A raise is never withheld: a drift found on the apps that did answer is real either way. + // Every tick, including ticks with nothing unread: that call is what resets the latches. + report_unread_once(ctx, unread); + if (drifted_.empty() && !blocking.empty()) { + report_withheld_clear(ctx, blocking); + return; + } + withheld_ticks_ = 0; + withheld_reported_ = false; + std::map affected; + for (const auto & [app, items] : drifted_) { + std::string detail; + for (const auto & item : items) { + detail += (detail.empty() ? "" : "; ") + item; + } + if (detail.size() > kMaxAppDetailChars) { + // Keep both ends: the head names the app and its first drifted parameter, the tail keeps the + // count of what followed readable rather than ending mid-token. + const std::size_t keep = kMaxAppDetailChars - 3; + const std::size_t head = (keep + 1) / 2; + detail = detail.substr(0, head) + "..." + detail.substr(detail.size() - (keep - head)); + } + affected.emplace(app, detail); + } + aggregated_.emit_ordered(ctx, affected, emit_order()); + reported_ = drifted_; + } + + /// App ids in the order their details should appear in the description. Keys of `affected` + /// missing here are appended by the helper, so a gap degrades the ordering rather than + /// dropping an app. + std::vector emit_order() const { + std::vector pinned_new, pinned_old, baseline_new, baseline_old; + for (const auto & [app, items] : drifted_) { + (void)items; + const bool is_new = reported_.count(app) == 0; + const auto pit = pinned_counts_.find(app); + const bool pinned = pit != pinned_counts_.end() && pit->second > 0; + auto & bucket = pinned ? (is_new ? pinned_new : pinned_old) : (is_new ? baseline_new : baseline_old); + bucket.push_back(app); + } + std::vector order; + order.reserve(drifted_.size()); + for (const auto * bucket : {&pinned_new, &pinned_old, &baseline_new, &baseline_old}) { + order.insert(order.end(), bucket->begin(), bucket->end()); + } + return order; + } + + ParamDriftPolicy policy_; + int tick_interval_ms_ = kDefaultTickIntervalMs; // plugin-injected; paces the reader thread + std::shared_ptr io_; // shared with the background reader thread + std::thread reader_thread_; + rclcpp::Context::SharedPtr context_; // for the pre-shutdown callback + rclcpp::PreShutdownCallbackHandle pre_shutdown_handle_; + /// Consecutive missing sweeps for one app, plus the one-shot latch that keeps the baseline + /// forget from repeating once the churn window has passed. + struct AbsenceCounter { + int misses = 0; + bool baseline_forgotten = false; + }; + + std::map misses_; // app.id -> absence bookkeeping + std::map> drifted_; // app.id -> its drifted descriptors + std::map> reported_; // drifted_ as of the previous emit + std::map pinned_counts_; // app.id -> leading `expect` descriptors + std::map unread_latches_; // app.id -> which unread warnings were logged + std::set unmatched_pins_reported_; // expect pins already reported as declared by no app + std::map unarmed_ticks_; // app.id -> consecutive ticks the gate denied it + std::map bound_fqn_; // app.id -> the fqn it was last seen bound to + int withheld_ticks_ = 0; // consecutive ticks a clear has been held back + bool withheld_reported_ = false; // the withheld-clear warning fired for this episode + int prune_grace_ = kForgetGrace; // plugin-injected; absences before a finding is dropped + std::vector own_warnings_; // config problems the detector itself found + ParameterTransportFactory transport_factory_; // empty = build the real ROS 2 transport + AggregatedFault aggregated_{graph_fault_codes::kParamDrift, kParamDriftSeverity}; + bool warnings_logged_ = false; +}; + +REGISTER_DETECTOR(ParamDriftDetector, "param_drift") + +} // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py index 2198d4af..1f65aea9 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py @@ -31,9 +31,13 @@ on PYTHONPATH once the workspace's install/setup.bash is sourced. """ +import json import os import time +from launch import LaunchDescription +from launch.actions import TimerAction +import launch_testing.actions import requests from ros2_medkit_test_utils.constants import get_test_port from ros2_medkit_test_utils.launch_helpers import ( @@ -42,16 +46,13 @@ create_gateway_node, ) -from launch import LaunchDescription -from launch.actions import TimerAction -import launch_testing.actions - API_BASE_PATH = '/api/v1' def create_watchdog_test_launch( *, detector_params=None, + extra_gateway_params=None, demo_nodes=None, port=None, demo_delay=2.0, @@ -66,6 +67,11 @@ def create_watchdog_test_launch( keys, e.g. ``'plugins.graph_watchdog.detectors.param_drift.mode'``), merged on top of the ``plugins``/``plugins.graph_watchdog.path`` parameters this factory always sets. + extra_gateway_params : dict or None + Extra ROS parameters for the gateway node itself rather than for the + plugin (e.g. ``'parameter_service_negative_cache_sec'``). Kept separate + from `detector_params` so a gateway-scope knob is not filed under a + plugin-scope name; both end up in the same node config. demo_nodes : list of str or None Node keys from ``ros2_medkit_test_utils.launch_helpers.DEMO_NODE_REGISTRY``. ``None`` means no demo nodes. @@ -95,6 +101,8 @@ def create_watchdog_test_launch( } if detector_params: params.update(detector_params) + if extra_gateway_params: + params.update(extra_gateway_params) gateway_node = create_gateway_node(port=port, extra_params=params) @@ -155,7 +163,7 @@ def graph_watchdog_plugin_path(): return path -def wait_until_watchdog_armed(port, timeout=60.0, interval=0.5): +def wait_until_watchdog_armed(port, timeout=60.0, interval=0.5, app_id=None): """Poll ``GET /api/v1/x-medkit-watchdog`` until the plugin is live and armed. Any assertion that a fault is ABSENT must gate on this first. Absence is the @@ -173,6 +181,28 @@ def wait_until_watchdog_armed(port, timeout=60.0, interval=0.5): ``global_state == 'armed'`` additionally proves the bringup grace elapsed, i.e. the gate would have permitted a raise during the window that follows. + `app_id` narrows that from "some app is armed" to "THIS app is armed", which + is what a test that is about to perturb one specific node needs. The gate is + what decides whether a detector may target an app at all (param_drift builds + its read set from ``reliability_allows`` - see its tick()), so an armed + entry is the precondition for the very first read of that node: gating on it + means a perturbation can no longer land before the detector was even + permitted to look, and a stack that comes up late fails HERE, naming the + node, instead of surfacing later as an unexplained missing fault. + + The check mirrors ``ReliabilityGate::allows_raise``: the payload's + per-entity ``state`` is ``armed`` only when the warmup has elapsed AND the + lifecycle watcher considers the node ok, which is exactly the conjunction + that permits a raise. The boolean ``armed`` field alone is only the warmup + half. + + What this does NOT prove is that a detector has already READ the app - no + e2e surface exposes that. The plugin's only route reports gate state, and a + detector's level-triggered PASSED for a fault that does not exist yet is + dropped by the fault_manager (see ``report_fault_event``), so a clear is + invisible until something has been raised. Callers that need a captured + baseline must still allow the sweep a window after this returns. + Parameters ---------- port : int @@ -181,26 +211,49 @@ def wait_until_watchdog_armed(port, timeout=60.0, interval=0.5): Maximum time to wait in seconds. interval : float Sleep between retries in seconds. + app_id : str or None + App id (as the gateway derives it from the ROS node name) that must be + present in ``entities`` with ``state == 'armed'``. ``None`` only + requires some entity plus a global ``armed`` state. Returns ------- bool - ``True`` once the plugin reports at least one entity and a global - ``armed`` state, ``False`` on timeout. + ``True`` once the plugin reports a global ``armed`` state together with + at least one entity (or with `app_id` armed, when given), ``False`` on + timeout. """ base = f'http://127.0.0.1:{port}{API_BASE_PATH}' deadline = time.monotonic() + timeout + last_seen = 'GET /x-medkit-watchdog was never answered at all' while time.monotonic() < deadline: try: response = requests.get(f'{base}/x-medkit-watchdog', timeout=5) - if response.status_code == 200: + if response.status_code != 200: + # Worth naming rather than folding into the generic message: the route is + # registered by the plugin itself, so a 404 from a gateway that is otherwise + # answering says the .so never loaded. + last_seen = f'HTTP {response.status_code} from GET /x-medkit-watchdog' + else: status = response.json().get('x-medkit-watchdog', {}) - if status.get('entities') and status.get('global_state') == 'armed': + last_seen = json.dumps(status) + entities = status.get('entities') or [] + if app_id is None: + armed = bool(entities) + else: + armed = any( + e.get('id') == app_id and e.get('state') == 'armed' for e in entities) + if armed and status.get('global_state') == 'armed': return True - except requests.exceptions.RequestException: - pass + except requests.exceptions.RequestException as exc: + last_seen = f'GET /x-medkit-watchdog failed: {exc}' time.sleep(interval) + # The caller turns this into an assertion failure, but only it knows what it + # was waiting for; the gate state itself is the evidence for WHY, and it is + # gone once the launch tears down. Print it while it still exists. + print(f'wait_until_watchdog_armed(app_id={app_id!r}) timed out after {timeout}s; ' + f'last watchdog status: {last_seen}') return False diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_config_plumbing_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_config_plumbing_e2e.test.py new file mode 100644 index 00000000..9ee26ead --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_config_plumbing_e2e.test.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Config-plumbing e2e: proves NESTED plugin config reaches a live detector +through the REAL gateway. + +A C++ unit test cannot prove this - it hand-builds already-nested JSON and +calls a detector's configure() directly, bypassing the real config-delivery +path (extract_plugin_config -> detector_config.hpp -> +ParamDriftConfig::from_json) entirely. That bypass is exactly what hid the +pre-existing bug this test guards against a regression of: detector_config.hpp +used to read a FLAT dotted-key shape while the gateway delivers NESTED, and +even after fixing that, param_drift's own `expect` sub-object was read via a +flat "expect." prefix scan and silently dropped through the real gateway. + +Uses a param_drift `expect` rule, which fires on a STATIC graph (no runtime +drift needed - expect rules are absolute, see param_drift_policy.hpp and +test_param_drift_integration.cpp's ExpectRuleFlagsWrongFromStart). This +exercises BOTH config layers at once: the `expect` field (extract_detector_config +-> ParamDriftConfig::from_json) and `mode` (detector_config.hpp). + +Runs as THREE separate CTest targets (see CMakeLists.txt), each launching its +OWN gateway+fault_manager+demo-node stack (config is read once at plugin +set_context() time, so proving both layers needs two distinct launches, not +two test methods sharing one). The WATCHDOG_E2E_SCENARIO env var selects +which launch + assertion runs in this process: + +- "positive": plugins.graph_watchdog.detectors.param_drift.expect.use_sim_time + = True, with NO mode override. The demo node runs with its real default + use_sim_time=False, an immediate mismatch -> GRAPH_PARAM_DRIFT must raise. + If the `expect` nested reader (layer 2) were still broken, `expect` would + be silently dropped and this would never fire. +- "mode_off": same expect mismatch, PLUS + plugins.graph_watchdog.detectors.param_drift.mode = "off" -> GRAPH_PARAM_DRIFT + must never appear. If the `mode` nested reader (layer 1, detector_config.hpp) + were still broken, "off" would never be recognized and this detector would + keep raising despite the operator disabling it. +- "mode_off_yaml_bool": same, but `mode` is the YAML 1.1 boolean a BARE `off` + actually parses to. This is the form operators write, and it reaches the + plugin as a JSON bool, never a string. + +Both "off" scenarios assert an ABSENCE, so both first gate on the plugin's own +GET /x-medkit-watchdog route (harness.wait_until_watchdog_armed). Without that +gate the assertion passes just as well on a stack that never came up: the +gateway survives a plugin that fails to dlopen and a REST server that cannot +bind, both still exit 0, and poll_faults returns None on any transport error. +""" + +import os +import sys +import unittest + +import launch_testing + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from harness import ( # noqa: E402 + create_watchdog_test_launch, + poll_faults, + wait_until_watchdog_armed, +) + +from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES, get_test_port # noqa: E402 + +# No default on purpose. A default makes this file FAIL OPEN: if the CTest ENVIRONMENT property +# never reaches the process - an edit that drops the variable from one of the three targets, +# someone running launch_test on the file directly - then generate_test_description() also skips +# its `mode` override, so the launch and the assertions degrade together into a second copy of the +# positive scenario and CTest still reports 1/1 passed. A KeyError here is loud and immediate. +SCENARIO = os.environ['WATCHDOG_E2E_SCENARIO'] +PORT = get_test_port() + +# Fast tick cadence + short warmup so the whole story (gateway/fault_manager +# startup, demo-node discovery, per-app arm, global bringup grace - see the +# plugin README's "Reliability (bringup-quiesce)") comfortably fits inside +# the poll_faults() timeouts below without any hand-tuned pre-sleep: polling +# itself absorbs demo_delay + discovery + warmup, retrying every interval +# until the assertion's own timeout. +TICK_INTERVAL_MS = 200 +WARMUP_CYCLES = 3 + +# The two shapes `mode` actually arrives in. A QUOTED "off" in YAML stays a +# STRING; a BARE `off` is YAML 1.1, which rcl_yaml_param_parser types as BOOL +# False (as do `Off`, `OFF`, `no`, `n`). `off` bare is what operators write, so +# both shapes have to disable the detector - see parse_detector_mode(). +_MODE_OFF_STRING = 'off' +_MODE_OFF_YAML_BOOL = False + + +def generate_test_description(): + detector_params = { + 'plugins.graph_watchdog.tick_interval_ms': TICK_INTERVAL_MS, + 'plugins.graph_watchdog.warmup_cycles': WARMUP_CYCLES, + 'plugins.graph_watchdog.detectors.param_drift.baseline': False, + 'plugins.graph_watchdog.detectors.param_drift.expect.use_sim_time': True, + } + if SCENARIO == 'mode_off': + detector_params['plugins.graph_watchdog.detectors.param_drift.mode'] = _MODE_OFF_STRING + elif SCENARIO == 'mode_off_yaml_bool': + detector_params['plugins.graph_watchdog.detectors.param_drift.mode'] = _MODE_OFF_YAML_BOOL + + return create_watchdog_test_launch( + detector_params=detector_params, + demo_nodes=['temp_sensor'], + port=PORT, + ) + + +class TestConfigPlumbingPositive(unittest.TestCase): + """Positive control: the nested `expect` config reaches param_drift.""" + + def test_expect_rule_raises_from_nested_config(self): + fault = poll_faults(PORT, 'GRAPH_PARAM_DRIFT', timeout=60.0) + self.assertIsNotNone( + fault, + 'GRAPH_PARAM_DRIFT never raised - the nested `expect` config was ' + 'not delivered to param_drift through the real gateway', + ) + + +class TestConfigPlumbingModeOff(unittest.TestCase): + """Mode suppression: the nested `mode` config disables the detector.""" + + def test_mode_off_suppresses_detector(self): + # Gate on the plugin's own route BEFORE asserting absence. Without this the + # assertion is vacuous by construction: a stack that never came up produces + # exactly the same "no fault" result, and every failure mode on this launch + # (plugin fails to dlopen, REST server cannot bind) still exits 0. This runs + # in its own process against its own gateway on its own port, so nothing in + # the positive scenario covers it either. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed gate - the plugin did not load ' + 'or its tick loop never ran, so an absent fault proves nothing', + ) + # Proving absence means waiting out the full window: a detector that + # (regression) ignored `mode` would have raised well within it. + fault = poll_faults(PORT, 'GRAPH_PARAM_DRIFT', timeout=20.0) + self.assertIsNone( + fault, + 'GRAPH_PARAM_DRIFT raised despite ' + 'detectors.param_drift.mode="off" - the nested `mode` config was ' + 'not delivered to (or not honored by) the plugin', + ) + + +class TestConfigPlumbingModeOffYamlBool(unittest.TestCase): + """A BARE `off` disables the detector, not just a quoted one.""" + + def test_yaml_boolean_off_suppresses_detector(self): + # `mode: off` written bare in YAML never reaches the plugin as a string: + # rcl_yaml_param_parser applies YAML 1.1 and types it BOOL False. A reader + # that only accepts strings therefore leaves the detector in its `raise` + # default, and the detector the operator just disabled keeps pushing + # GRAPH_PARAM_DRIFT at the robot with nothing in the log to explain it. + # This is the shape operators actually write, so it gets its own launch. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed gate - the plugin did not load ' + 'or its tick loop never ran, so an absent fault proves nothing', + ) + fault = poll_faults(PORT, 'GRAPH_PARAM_DRIFT', timeout=20.0) + self.assertIsNone( + fault, + 'GRAPH_PARAM_DRIFT raised despite a bare ' + 'detectors.param_drift.mode: off - the YAML boolean form of "off" ' + 'was not honored by the plugin', + ) + + +# Each CTest target launches this file with one scenario, so only that scenario's case may +# run. Marking the other two as skipped would report them as skipped on every single run, +# which is exactly how a scenario that has genuinely stopped working would look. Removing +# them from the module instead means each run reports one case, and a missing result is a +# real failure rather than an expected line of output. +_SCENARIO_CASES = { + 'positive': 'TestConfigPlumbingPositive', + 'mode_off': 'TestConfigPlumbingModeOff', + 'mode_off_yaml_bool': 'TestConfigPlumbingModeOffYamlBool', +} +if SCENARIO not in _SCENARIO_CASES: + raise RuntimeError( + f'WATCHDOG_E2E_SCENARIO={SCENARIO!r} is not one of {sorted(_SCENARIO_CASES)}; ' + 'the CTest target and this file disagree about which scenarios exist') +for _scenario, _case_name in _SCENARIO_CASES.items(): + if _scenario != SCENARIO: + del globals()[_case_name] + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + """Verify the gateway/fault_manager/demo stack exits cleanly.""" + + def test_exit_codes(self, proc_info): + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'Process {info.process_name} exited with {info.returncode}', + ) diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_drift_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_drift_e2e.test.py new file mode 100644 index 00000000..cd527d00 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_drift_e2e.test.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""param_drift e2e for the mode operators actually get: self-capturing BASELINE drift. + +The three config-plumbing scenarios all set ``baseline: false`` and assert only that a fault +appears or does not, so until this file existed the detector's DEFAULT and headline behaviour - +capture a parameter's value, notice when it changes at runtime - had no end-to-end coverage, and +no scenario asserted a clear at all. The whole "Closing the loop" section of the README rests on +the clear reaching the fault_manager and healing there. + +What this drives, all real: a gateway process with the plugin ``.so`` loaded, a real +fault_manager, a real ROS node whose parameter this test changes over the real parameter service, +and the operator-visible ``GET /api/v1/faults`` surface. + +Unlike test_orphan_e2e.test.py, the node here MUST be spun: the detector reads its parameters over +a service, and an unspun node answers nothing, so the detector would report it as never read +instead of capturing a baseline. + +Everything below is anchored to the plugin's own GET /x-medkit-watchdog route +(harness.wait_until_watchdog_armed) rather than to wall-clock time from process start. This file +both asserts an ABSENCE and then perturbs a parameter, and each needs the gate for its own reason: +an absence proves nothing about a stack that never came up, and the detector may not read a node +before the gate arms it, so a perturbation that lands first turns the DRIFTED value into the +baseline and no fault can ever raise. +""" + +import os +import sys +import threading +import unittest + +import launch_testing +import rclpy +from rclpy.node import Node + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +# I100 as well as E402: `harness` is only importable because of the sys.path line above, so this +# import cannot be moved up to where the alphabetical order would put it. +from harness import ( # noqa: E402, I100 + create_watchdog_test_launch, + poll_cleared, + poll_faults, + wait_until_watchdog_armed, +) + +from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES, get_test_port # noqa: E402 + +PORT = get_test_port() + +# Fast tick + short warmup so the whole story fits inside the poll timeouts without a hand-tuned +# pre-sleep - same rationale as the other e2e files here. max_reads_per_tick is raised because the +# real graph carries the gateway, the fault_manager and this test's node, and the sweep is +# round-robin: at the default budget the node under test waits several ticks for its turn. +TICK_INTERVAL_MS = 200 +WARMUP_CYCLES = 3 +FAULT_CODE = 'GRAPH_PARAM_DRIFT' +# The ROS node name, which is also the App id the gateway derives for a node in the root +# namespace - so it is both what this test spins up and what the watchdog gate reports as an +# entity. Used for the arming gate and for the "the fault names the node" assertion, which must +# not be able to drift apart. +TARGET_NODE = 'pd_e2e_target' +PARAM_NAME = 'speed_limit' +BASELINE_VALUE = 1.0 +DRIFTED_VALUE = 0.2 + +# How long the detector gets to capture a baseline, measured from the moment the gate armed the +# target (NOT from process start - bringup is absorbed by the gate, which is the point of it). +# Sized to exceed the parameter transport's 10 s negative-cache TTL, so a first read that timed +# out and backed the node off is retried and still lands inside the window. The C++ integration +# suite sizes its own poll bound off the same constant for the same reason. +SILENT_CAPTURE_SEC = 15.0 + + +def generate_test_description(): + detector_params = { + 'plugins.graph_watchdog.tick_interval_ms': TICK_INTERVAL_MS, + 'plugins.graph_watchdog.warmup_cycles': WARMUP_CYCLES, + 'plugins.graph_watchdog.detectors.param_drift.max_reads_per_tick': 32, + } + return create_watchdog_test_launch( + detector_params=detector_params, + demo_nodes=None, + port=PORT, + # The clear must reach HEALED, i.e. leave the default active-fault query, which the + # fault_manager only does when healing is enabled - see harness.py and the README's + # "Closing the loop". + healing_enabled=True, + ) + + +class TestParamDriftE2e(unittest.TestCase): + """Baseline drift raises through the real stack, and restoring the value heals it.""" + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._node = Node(TARGET_NODE) + cls._node.declare_parameter(PARAM_NAME, BASELINE_VALUE) + cls._stop = threading.Event() + cls._spin = threading.Thread(target=cls._spin_until_stopped, daemon=True) + cls._spin.start() + + @classmethod + def _spin_until_stopped(cls): + # The node has to answer parameter-service calls for the detector to read it at all. + while not cls._stop.is_set() and rclpy.ok(): + rclpy.spin_once(cls._node, timeout_sec=0.1) + + @classmethod + def tearDownClass(cls): + cls._stop.set() + cls._spin.join(timeout=5.0) + cls._node.destroy_node() + rclpy.shutdown() + + def test_01_runtime_change_raises_and_names_the_parameter(self): + # Gate on the target being ARMED before anything below runs. Two distinct things go wrong + # without it, and neither is visible in the result: + # + # - The absence assertion is vacuous against a stack that never came up. A plugin that + # fails to dlopen, a REST server that cannot bind - the gateway survives both and exits + # 0, and poll_faults swallows every transport error and returns None, which is exactly + # what assertIsNone wants to see. + # - Worse, the flip below is only a drift if a baseline was captured first, and the + # detector may not target a node until the gate arms it. On a slow runner a fixed + # pre-window elapses during bringup, set_parameters lands before the first read, the + # DRIFTED value silently becomes the baseline, and no fault can ever raise - the run + # then dies at the 60 s poll below blaming the detector for a race in this test. + # + # Anchoring both to the gate makes bringup cost time here, where the failure message + # names the real cause, instead of eating the window that is supposed to cover the read. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} as an armed entity - the plugin did not ' + f'load, its tick loop never ran, or the gateway never discovered the node. The ' + f'detector reads only apps the gate arms, so nothing below would mean anything', + ) + + # Give the detector time to capture the baseline BEFORE changing anything. Capturing + # must be silent, so a fault appearing here at all would mean the detector reports its + # own first read as drift. + self.assertIsNone( + poll_faults(PORT, FAULT_CODE, timeout=SILENT_CAPTURE_SEC), + 'GRAPH_PARAM_DRIFT was raised before any parameter changed, so baseline capture is ' + 'not silent', + ) + + self._node.set_parameters( + [rclpy.parameter.Parameter(PARAM_NAME, rclpy.Parameter.Type.DOUBLE, DRIFTED_VALUE)]) + + fault = poll_faults(PORT, FAULT_CODE, timeout=60.0) + self.assertIsNotNone( + fault, + f'{FAULT_CODE} never raised after {PARAM_NAME} changed from {BASELINE_VALUE} to ' + f'{DRIFTED_VALUE} at runtime', + ) + description = fault.get('description', '') + self.assertIn(PARAM_NAME, description) + self.assertIn(TARGET_NODE, description) + + def test_02_restoring_the_value_heals_the_fault(self): + # "Cleared" is an absence too, and it is the default state of a fault that was never + # raised: on the stack test_01 just failed against, poll_cleared answers True on its + # first request. Confirm the fault is actually THERE first, so what this measures is a + # heal and not a fault that never happened. test_01 raised it and the drifted value is + # still set, so it is present now or the level-triggered raise is not level-triggered. + self.assertIsNotNone( + poll_faults(PORT, FAULT_CODE, timeout=30.0), + f'{FAULT_CODE} is not active at the start of the heal test, so there is nothing here ' + f'that could heal and the clear below would pass without measuring anything', + ) + + # This is the path the README tells operators to rely on: put the value back and the fault + # goes away on its own. It only works if the detector keeps emitting a level-triggered + # PASSED and the fault_manager counts those toward healing. + self._node.set_parameters( + [rclpy.parameter.Parameter(PARAM_NAME, rclpy.Parameter.Type.DOUBLE, BASELINE_VALUE)]) + + self.assertTrue( + poll_cleared(PORT, FAULT_CODE, timeout=60.0), + f'{FAULT_CODE} did not heal after {PARAM_NAME} was restored to {BASELINE_VALUE}', + ) + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + """Verify the gateway/fault_manager stack exits cleanly.""" + + def test_exit_codes(self, proc_info): + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'Process {info.process_name} exited with {info.returncode}', + ) diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_roundtrip_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_roundtrip_e2e.test.py new file mode 100644 index 00000000..26d30171 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_roundtrip_e2e.test.py @@ -0,0 +1,658 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""What one parameter read really costs the node that serves it. + +``param_drift`` charges its ``max_reads_per_tick`` budget in parameter-service ROUND TRIPS +and not in transport calls, because one call is several round trips at the watched node: +``kBaselineReadRoundTrips`` (2), ``kExpectReadRoundTrips`` (3) and +``kExpectNotFoundRoundTrips`` (2) in ``src/detectors/param_drift_detector.cpp``. Every other +test in this package proves the detector's arithmetic GIVEN those numbers, and the stand-in +transport they use counts CALLS, so not one of them can tell whether the TRANSPORT still costs +what those numbers say. Add a fourth service call inside ``Ros2ParameterTransport`` and the +whole suite stays green while the load bound this feature sells quietly becomes a lie. + +This test measures the transport. It runs the REAL one - the gateway compiles the same +``ros2_parameter_transport.cpp`` the plugin does (see ``PARAMETER_TRANSPORT_SOURCES`` in +CMakeLists.txt) - against a node that answers the parameter services BY HAND and counts every +request it is asked to serve. The counts are what the node paid, not what anything claims it +paid. + +What it does NOT do, and must not be relied on for, is check the detector's own literals. It +never reads them: ``param_drift`` is switched OFF here and the reads are driven over HTTP, so +setting ``kBaselineReadRoundTrips`` to 4 and ``kExpectReadRoundTrips`` to 6 with the transport +untouched leaves this file green. What catches a CHARGE that has drifted away from the +measurement is the timing suite in ``test/test_param_drift_integration.cpp`` +(``ABaselineVisitIsChargedTwoRoundTripsAndNotTheColdStartPair``, +``TheBudgetIsSpentInRoundTripsNotCalls``, ``AnUndeclaredPinIsChargedLessThanOneThatResolves``), +which times the reader against a configured budget. The two halves are complementary and +neither is redundant: this file pins the cost, those pin the charge against it. + +The gateway is launched here with a 5 s per-call timeout and the negative cache disabled, while +the detector builds its transport with 0.5 s and a 10 s cache. That difference does not affect +anything measured below: a round-trip count on the SUCCESS path is the number of requests the +transport issues, which is the same whatever the deadline on each of them is. The generous +settings exist only so a slow first DDS discovery cannot turn a success into a TIMEOUT, or lock +a probe out for a minute after one early attempt, and so make the first-contact measurements +repeatable. + +Why the reads are driven over HTTP rather than by the detector. The constants describe two +functions, ``Ros2ParameterTransport::list_parameters`` and ``::get_parameter``, and +``GET /{entity}/configurations`` / ``GET /{entity}/configurations/{id}`` invoke exactly those, +once per backing node, synchronously, on demand (``ConfigurationManager`` delegates 1:1). That +is what makes a count attributable to ONE call. The detector reaches the same two functions +from a background round-robin sweep that never stops and exposes no visit boundary, so driving +it would only ever yield an increment pattern smeared over the sweep's pacing - a weaker claim +about the same code. ``param_drift`` is therefore switched OFF here, so the only parameter +traffic the probe node sees is the traffic this test asks for. + +The four costs pinned below, and how they relate to what the detector charges: + +* cold baseline visit = 4 - ``list_parameters`` primes the transport's defaults cache before + its own read (list + get, then list + get). That cache has no TTL, so this is a one-off per + node per gateway lifetime, and the detector deliberately does NOT charge it. +* warm baseline visit = 2 - matches ``kBaselineReadRoundTrips``. +* resolved ``expect`` pin = 3 - matches ``kExpectReadRoundTrips`` (name-scoped list, get, + describe; describe is unconditional on the success path). +* unresolved ``expect`` pin = 1 - the name-scoped list comes back empty and the transport + returns NOT_FOUND before the get. The detector charges 2 for this case ON PURPOSE: a second, + rarer NOT_FOUND path costs two round trips and the error code cannot tell the two apart, so + it charges the higher of them. A bound may overstate a cost, never understate one - so 1 here + is the real cost and is ALLOWED to be below the charge. Do not "fix" either number to match + the other. +* a listed name the get returns no value for = 2 - that second NOT_FOUND path, measured, so the + charge of 2 is anchored from above as well as below. +* a DECLARED BUT UNSET parameter = 3, and a SUCCESS rather than a NOT_FOUND. rclcpp answers one + with a ``PARAMETER_NOT_SET`` value, not with an absent one, so the transport gets a parameter + back and pays for the describe like any other resolved pin. It is charged 3, which is right. + This case is pinned separately because the two-round-trip path above was previously described + as being it. +* FIRST resolved ``expect`` pin against an untouched node = 3, the same as any later one. + ``get_parameter`` never primes the defaults cache, so the cold-contact pair is a property of + ``list_parameters`` and of ``baseline: true`` mode, not of reading a node for the first time. + Measured against a second probe that nothing has listed. +""" + +import os +import sys +import threading +import time +import unittest + +import launch_testing +from rcl_interfaces.msg import ( + ParameterDescriptor, + ParameterType, + ParameterValue, + SetParametersResult, +) +from rcl_interfaces.srv import ( + DescribeParameters, + GetParameters, + GetParameterTypes, + ListParameters, + SetParameters, + SetParametersAtomically, +) +import rclpy +from rclpy.executors import SingleThreadedExecutor +from rclpy.node import Node +import requests + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from harness import create_watchdog_test_launch # noqa: E402 + +from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES, get_test_port # noqa: E402 + +PORT = get_test_port() +API_BASE = f'http://127.0.0.1:{PORT}/api/v1' + +PROBE_NODE = 'pd_rt_probe' +# A SECOND probe, identical but never listed. The cold-contact pair belongs to +# list_parameters (which primes the transport's defaults cache) and not to get_parameter +# (which does not), so proving that needs a node whose cache is still unprimed when the pinned +# read reaches it - which the first probe stops being the moment test_02 lists it. +PIN_PROBE_NODE = 'pd_rt_pinprobe' +# Two declared parameters, so the batched get after a full list carries more than one name and +# a regression that silently drops the batch cannot pass by returning a single value. +DECLARED_PARAMS = {'probe_gain': 2.5, 'probe_margin': 0.5} +# A name the probe never declares: the name-scoped list answers empty and the transport must +# stop there. +UNDECLARED_PARAM = 'pd_rt_no_such_parameter' +# A name the probe lists and then returns NO VALUE for. This is the second, rarer NOT_FOUND path +# and the whole justification for charging NOT_FOUND two round trips rather than one: the +# name-scoped list answers, so the transport goes on to the get, and only the get finds there is +# nothing there. The error code is the same NOT_FOUND as the one-round-trip case and the caller +# cannot tell which it took, so the charge takes the higher of the two. +# +# The GetParameters contract does not require one value per requested name, so a reply that simply +# omits it is well formed - and it is the only reply that reaches this branch. See NOT_SET_PARAM +# for the case this was previously described as, which behaves quite differently. +NO_VALUE_PARAM = 'pd_rt_no_value_returned' +# A name the probe lists and answers with a PARAMETER_NOT_SET value: a parameter that is DECLARED +# BUT UNSET, which is what rclcpp's own parameter service replies for one. That reply is a SUCCESS +# carrying a null value, not a NOT_FOUND: the transport gets a parameter back, goes on to the +# describe, and returns it. It therefore costs 3 and is charged 3 like any other resolved pin. +# Pinned here because the 2-round-trip NOT_FOUND above used to be documented as this case, which +# would have made the charge for a resolved-but-null read an understatement of what the node pays. +# Measured on the SECOND probe, whose full list nothing asserts on; adding either name to the +# first would change what test_02 and test_03 read back. +NOT_SET_PARAM = 'pd_rt_declared_but_unset' + +# Per-request HTTP patience. Generous enough for a first request that races gateway startup, short +# enough that several of them still fit inside the ctest timeout with the retry deadlines below. +HTTP_TIMEOUT = 15 + +# The six services rclcpp's AsyncParametersClient requires before it reports ready. Only three +# of them are ever expected to carry traffic; the rest exist so wait_for_service() succeeds, +# and are counted anyway - a transport that started calling one of them would show up here +# rather than hide inside a total. +SERVICE_NAMES = ( + 'list_parameters', + 'get_parameters', + 'describe_parameters', + 'get_parameter_types', + 'set_parameters', + 'set_parameters_atomically', +) + + +def _double(value): + """Build a well-formed DOUBLE ``ParameterValue``.""" + return ParameterValue(type=ParameterType.PARAMETER_DOUBLE, double_value=float(value)) + + +class _RequestFailed: + """Stand-in for a response that never arrived. + + ``_measure`` is called from retry loops that decide what to do next from + ``response.status_code``, and the HTTP call it wraps is reachable in precisely the state + those loops exist for: a read that degenerates and outlives the client's patience. Letting + ``requests`` raise out of it aborts the retry with a traceback rather than retrying, and none + of the diagnostics the loop was written to print ever run - the failure report is then a stack + trace about a socket instead of a statement about round trips. + """ + + status_code = None + + def __init__(self, exc): + self.text = f'{type(exc).__name__}: {exc}' + + def json(self): + raise AssertionError(f'there is no response body to parse: {self.text}') + + +class ParameterRequestCounter(Node): + """A node that serves the parameter services by hand and counts what it is asked for. + + Its own rclcpp/rclpy parameter services are DISABLED: their traffic is unobservable from + the outside, and counting is the entire point. The hand-rolled replacements answer on the + names rclpy would have used, so the transport's ``AsyncParametersClient`` finds them + exactly as it finds a normal node's. + + Every reply is well formed. A malformed one would send the transport down an early-return + path, and the test would then be counting failures instead of the cost of a successful + read - a green run that proves nothing. + """ + + def __init__(self, node_name, listed_extras=()): + super().__init__(node_name, start_parameter_services=False) + self._lock = threading.Lock() + self._counts = dict.fromkeys(SERVICE_NAMES, 0) + # Names the list reports beyond DECLARED_PARAMS. NO_VALUE_PARAM is omitted from the get + # reply entirely; NOT_SET_PARAM is answered with a PARAMETER_NOT_SET value. Those two + # replies take the transport down different paths and cost the node different amounts. + self._listed_extras = tuple(listed_extras) + fqn = self.get_fully_qualified_name() + self._services = [ + self.create_service(ListParameters, f'{fqn}/list_parameters', self._on_list), + self.create_service(GetParameters, f'{fqn}/get_parameters', self._on_get), + self.create_service( + DescribeParameters, f'{fqn}/describe_parameters', self._on_describe), + self.create_service( + GetParameterTypes, f'{fqn}/get_parameter_types', self._on_types), + self.create_service(SetParameters, f'{fqn}/set_parameters', self._on_set), + self.create_service( + SetParametersAtomically, + f'{fqn}/set_parameters_atomically', + self._on_set_atomically), + ] + + def snapshot(self): + """Per-service request counts as of now.""" + with self._lock: + return dict(self._counts) + + def _bump(self, service): + with self._lock: + self._counts[service] += 1 + + def _on_list(self, request, response): + self._bump('list_parameters') + names = sorted(list(DECLARED_PARAMS) + list(self._listed_extras)) + if request.prefixes: + # rcl's prefix rule: a name matches when it IS the prefix or lives under it. The + # transport's name-scoped probe passes the bare parameter name, so an undeclared + # name matches nothing and the reply is legitimately empty. + names = [ + name for name in names + if any(name == p or name.startswith(f'{p}.') for p in request.prefixes) + ] + response.result.names = names + return response + + def _on_get(self, request, response): + self._bump('get_parameters') + # NO_VALUE_PARAM is omitted from the reply rather than answered NOT_SET. The service + # contract does not require one value per requested name, and an omitted value is the only + # reply that reaches the transport's second NOT_FOUND branch - the one that costs two round + # trips and that kExpectNotFoundRoundTrips is sized for. + response.values = [ + _double(DECLARED_PARAMS[name]) if name in DECLARED_PARAMS + else ParameterValue(type=ParameterType.PARAMETER_NOT_SET) + for name in request.names + if name != NO_VALUE_PARAM + ] + return response + + def _on_describe(self, request, response): + self._bump('describe_parameters') + response.descriptors = [ + ParameterDescriptor(name=name, type=ParameterType.PARAMETER_DOUBLE) + for name in request.names + ] + return response + + def _on_types(self, request, response): + self._bump('get_parameter_types') + response.types = [ + ParameterType.PARAMETER_DOUBLE if name in DECLARED_PARAMS + else ParameterType.PARAMETER_NOT_SET + for name in request.names + ] + return response + + def _on_set(self, request, response): + self._bump('set_parameters') + response.results = [ + SetParametersResult(successful=False, reason='probe is read-only') + for _ in request.parameters + ] + return response + + def _on_set_atomically(self, request, response): + self._bump('set_parameters_atomically') + response.result = SetParametersResult(successful=False, reason='probe is read-only') + return response + + +def generate_test_description(): + detector_params = { + # The detector whose constants this test pins is switched OFF so that every request the + # probe counts is one this test asked for. Its background sweep would otherwise read the + # probe on its own schedule and no count would be attributable to a single call. `off` + # keeps the detector out of the plugin's tick list entirely (see + # graph_watchdog_plugin.cpp), so it never even builds a transport. + 'plugins.graph_watchdog.detectors.param_drift.mode': 'off', + } + gateway_params = { + # A generous per-call timeout keeps a slow first DDS discovery from turning a real + # SUCCESS path into a TIMEOUT. Disabling the negative cache means an early attempt made + # before the probe's services are discoverable cannot lock the node out for a minute + # (the default TTL): it just fails, costs the probe nothing, and test_02 retries while + # first contact is still ahead of it. + 'parameter_service_timeout_sec': 5.0, + 'parameter_service_negative_cache_sec': 0.0, + } + return create_watchdog_test_launch( + detector_params=detector_params, + extra_gateway_params=gateway_params, + demo_nodes=None, + port=PORT, + ) + + +class TestParameterRoundTripCost(unittest.TestCase): + """One transport call, counted at the node that serves it.""" + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._probe = ParameterRequestCounter(PROBE_NODE) + cls._pin_probe = ParameterRequestCounter( + PIN_PROBE_NODE, listed_extras=(NO_VALUE_PARAM, NOT_SET_PARAM)) + cls._executor = SingleThreadedExecutor() + cls._executor.add_node(cls._probe) + cls._executor.add_node(cls._pin_probe) + cls._stop = threading.Event() + cls._spin = threading.Thread(target=cls._spin_until_stopped, daemon=True) + cls._spin.start() + + @classmethod + def _spin_until_stopped(cls): + while not cls._stop.is_set() and rclpy.ok(): + cls._executor.spin_once(timeout_sec=0.1) + + @classmethod + def tearDownClass(cls): + cls._stop.set() + cls._spin.join(timeout=5.0) + cls._executor.remove_node(cls._probe) + cls._executor.remove_node(cls._pin_probe) + cls._executor.shutdown() + cls._probe.destroy_node() + cls._pin_probe.destroy_node() + rclpy.shutdown() + + # -- helpers ---------------------------------------------------------------------------- + + def _measure(self, url, probe=None): + """GET `url` and return the response together with what it cost `probe`. + + The probe's counters are bumped inside the service callback, which completes before + the reply reaches the transport and therefore long before the HTTP response returns. + The settle window after the response is not there to catch a laggard: it is there so + that a request arriving from anywhere ELSE lands inside the delta and breaks the + assertion, instead of silently shifting the next measurement's baseline. + """ + probe = probe if probe is not None else self._probe + before = probe.snapshot() + try: + response = requests.get(url, timeout=HTTP_TIMEOUT) + except requests.exceptions.RequestException as exc: + # A timed-out or dropped request is a legitimate outcome here, not a test error: see + # _RequestFailed. The delta is still taken, because a request that failed at the + # CLIENT may well have cost the probe something at the far end, and the retry loops + # check exactly that before deciding first contact is still ahead of them. + response = _RequestFailed(exc) + time.sleep(1.0) + after = probe.snapshot() + delta = {name: after[name] - before[name] for name in SERVICE_NAMES} + return response, delta + + @staticmethod + def _total(counts): + return sum(counts.values()) + + def _assert_round_trips(self, delta, expected, what): + self.assertEqual( + self._total(delta), expected, + f'{what} cost the node {self._total(delta)} parameter-service round trips, not ' + f'{expected}. Per service: {delta}. Either a round trip was added to or removed ' + f'from Ros2ParameterTransport, or the charge for this case in ' + f'param_drift_detector.cpp no longer matches what the node actually pays.') + + # -- tests ------------------------------------------------------------------------------ + + def test_01_the_probes_are_discovered_and_nothing_reads_them_unasked(self): + """Every later delta is only attributable if the probes are otherwise untouched.""" + wanted = {PROBE_NODE, PIN_PROBE_NODE} + # The retry deadlines in this file are chosen so that ALL of them together, plus the + # single-shot cases between them, expire well inside the ctest timeout. A test written to + # diagnose a slow-discovery path is worthless if ctest kills it before it can print. + deadline = time.monotonic() + 60.0 + seen = set() + while time.monotonic() < deadline: + try: + response = requests.get(f'{API_BASE}/apps', timeout=5) + if response.status_code == 200: + seen = {item.get('id') for item in response.json().get('items', [])} + if wanted <= seen: + break + except requests.exceptions.RequestException: + pass + time.sleep(0.5) + + self.assertTrue( + wanted <= seen, + f'the gateway never discovered {sorted(wanted - seen)} as apps, so no configurations ' + f'request can reach them. Apps seen: {sorted(seen)}') + + before = (self._probe.snapshot(), self._pin_probe.snapshot()) + time.sleep(3.0) + after = (self._probe.snapshot(), self._pin_probe.snapshot()) + self.assertEqual( + before, after, + f'something is reading parameters from a probe on its own: {before} -> {after}. ' + f'Every round-trip count below would then include traffic this test did not ask for.') + for name, counts in ((PROBE_NODE, after[0]), (PIN_PROBE_NODE, after[1])): + self.assertEqual( + self._total(counts), 0, + f'{name} had already served {self._total(counts)} parameter requests before this ' + f'test made any: {counts}. The first-contact measurements below need to be the ' + f'FIRST reads ever made against these nodes.') + + def test_02_a_cold_baseline_visit_costs_four_round_trips(self): + """First contact pays for priming the transport's defaults cache as well as the read.""" + url = f'{API_BASE}/apps/{PROBE_NODE}/configurations' + deadline = time.monotonic() + 45.0 + while True: + response, delta = self._measure(url) + if response.status_code == 200: + break + # A failure BEFORE the probe served anything is a transport that has not discovered + # the services yet: it fails inside wait_for_service, costs the node nothing, and + # the next attempt is still first contact. A failure that DID cost something has + # spent the cold visit, and the measurement below can no longer be made. + self.assertEqual( + self._total(delta), 0, + f'the first configurations read failed with HTTP {response.status_code} after ' + f'costing the probe {self._total(delta)} round trips ({delta}), so first ' + f'contact is gone and the cold cost can no longer be measured') + self.assertLess( + time.monotonic(), deadline, + f'GET {url} never succeeded; last status {response.status_code}: ' + f'{response.text[:400]}') + time.sleep(1.0) + + names = {item.get('name') for item in response.json().get('items', [])} + self.assertEqual( + names, set(DECLARED_PARAMS), + f'the read did not come back with the parameters the probe declares ({names}), so ' + f'the transport did not take its SUCCESS path and the count below is the price of a ' + f'failure, not of a read') + + self._assert_round_trips( + delta, 4, + 'a COLD baseline visit (first list_parameters against the node: the defaults-cache ' + 'priming list+get, then the second list+get the read itself makes)') + # Per service, not just the total. The four is two list+get PAIRS and nothing else; a + # change that swapped one of them for a describe or a get_parameter_types would leave the + # total at four and mean something quite different about what the node pays. + self.assertEqual( + (delta['list_parameters'], delta['get_parameters']), (2, 2), + f'a cold baseline visit is two list+get pairs - the defaults-cache priming one and ' + f'the read itself - and this was not: {delta}') + + def test_03_a_warm_baseline_visit_costs_two_round_trips(self): + """The defaults cache has no TTL, so the cold pair is paid once per node, not per read.""" + url = f'{API_BASE}/apps/{PROBE_NODE}/configurations' + response, delta = self._measure(url) + self.assertEqual(response.status_code, 200, response.text[:400]) + names = {item.get('name') for item in response.json().get('items', [])} + self.assertEqual(names, set(DECLARED_PARAMS), response.text[:400]) + + self._assert_round_trips( + delta, 2, + 'a WARM baseline visit (kBaselineReadRoundTrips: one list, then one batched get of ' + 'every name it returned)') + self.assertEqual( + (delta['list_parameters'], delta['get_parameters']), (1, 1), + f'a warm baseline visit is exactly one list and one batched get, and this was not - a ' + f'total of two says nothing about WHICH two: {delta}') + + def test_04_a_resolved_expect_pin_costs_three_round_trips(self): + """describe_parameters is unconditional on the success path, so the pin pays for it.""" + pin = 'probe_gain' + response, delta = self._measure(f'{API_BASE}/apps/{PROBE_NODE}/configurations/{pin}') + self.assertEqual(response.status_code, 200, response.text[:400]) + self.assertEqual( + response.json().get('data'), DECLARED_PARAMS[pin], + f'the pinned read did not return the value the probe serves, so the transport did ' + f'not take its SUCCESS path: {response.text[:400]}') + + self._assert_round_trips( + delta, 3, + 'a RESOLVED expect pin (kExpectReadRoundTrips: name-scoped list, get, describe)') + self.assertEqual( + delta['describe_parameters'], 1, + f'the descriptor round trip is what makes a resolved pin cost three rather than ' + f'two, and it did not happen: {delta}') + + def test_05_an_unresolved_expect_pin_costs_one_round_trip(self): + """Charged 2 by the detector on purpose; the real cost is 1 and is allowed to be lower. + + The transport answers NOT_FOUND from the name-scoped list alone and never reaches the + get or the describe. ``kExpectNotFoundRoundTrips`` is 2 because a SECOND NOT_FOUND path + - a parameter that is declared but unset - does cost two, and the error code cannot + distinguish them, so the constant takes the higher of the two. Overstating a load bound + is safe; understating it is not. This asserts the real cost, not the charge. + """ + response, delta = self._measure( + f'{API_BASE}/apps/{PROBE_NODE}/configurations/{UNDECLARED_PARAM}') + self.assertEqual( + response.status_code, 404, + f'a parameter the probe never declares must come back 404, not ' + f'{response.status_code}: {response.text[:400]}') + + self._assert_round_trips( + delta, 1, + 'an UNRESOLVED expect pin (the name-scoped list returns no names and the transport ' + 'stops there)') + self.assertEqual( + delta['get_parameters'], 0, + f'the transport read a value for a parameter its own name-scoped list said the node ' + f'does not have: {delta}') + + def test_06_a_pinned_read_has_no_cold_contact_surcharge(self): + """The cold pair belongs to list_parameters, not to a read in general. + + ``get_parameter`` never calls ``cache_default_values``, so an ``expect``-only sweep + (``baseline: false``) never primes that cache and never pays the extra pair - the first + pinned read of a node costs the same 3 as every one after it. The second probe exists + purely to have a node whose defaults cache is still unprimed at this point; the first + one stopped being one the moment test_02 listed it. + """ + pin = 'probe_gain' + before = self._pin_probe.snapshot() + self.assertEqual( + self._total(before), 0, + f'{PIN_PROBE_NODE} has already been read ({before}), so this is no longer a ' + f'first-contact measurement') + + url = f'{API_BASE}/apps/{PIN_PROBE_NODE}/configurations/{pin}' + deadline = time.monotonic() + 45.0 + while True: + response, delta = self._measure(url, probe=self._pin_probe) + if response.status_code == 200: + break + self.assertEqual( + self._total(delta), 0, + f'the first pinned read of {PIN_PROBE_NODE} failed with HTTP ' + f'{response.status_code} after costing it {self._total(delta)} round trips ' + f'({delta}), so first contact is gone and the cost can no longer be measured') + self.assertLess( + time.monotonic(), deadline, + f'GET {url} never succeeded; last status {response.status_code}: ' + f'{response.text[:400]}') + time.sleep(1.0) + + self.assertEqual( + response.json().get('data'), DECLARED_PARAMS[pin], + f'the pinned read did not return the value the probe serves, so the transport did ' + f'not take its SUCCESS path: {response.text[:400]}') + self._assert_round_trips( + delta, 3, + 'the FIRST expect pin read of a node (same as any later one: get_parameter does not ' + 'prime the defaults cache, so there is no cold-contact pair in expect-only mode)') + + def test_07_a_listed_name_with_no_value_costs_two_round_trips(self): + """The rarer NOT_FOUND path, and the reason the charge for NOT_FOUND is 2 and not 1. + + test_05 measures the COMMON NOT_FOUND: the name-scoped list comes back empty and the + transport stops there, for one round trip. This is the other one. The node lists the name, + so the transport goes on to the get, and the get comes back with no value for it - two + round trips, and the same NOT_FOUND error code as the one-round-trip case. + ``kExpectNotFoundRoundTrips`` is 2 because that is the higher of the two and the transport + cannot tell a caller which it took: a load bound may overstate a cost, never understate it. + + Without this case the charge was anchored from one side only. test_05 says the charge may + legitimately exceed the real cost, which on its own is an argument for any number at all; + it takes this measurement to say why the number is exactly two. + """ + response, delta = self._measure( + f'{API_BASE}/apps/{PIN_PROBE_NODE}/configurations/{NO_VALUE_PARAM}', + probe=self._pin_probe) + self.assertEqual( + response.status_code, 404, + f'a parameter the probe lists but returns no value for must come back 404, not ' + f'{response.status_code}: {response.text[:400]}') + + self._assert_round_trips( + delta, 2, + 'a LISTED name the get returns no value for (the name-scoped list finds the name, the ' + 'get finds no value, and the transport returns NOT_FOUND before the describe)') + self.assertEqual( + (delta['list_parameters'], delta['get_parameters'], delta['describe_parameters']), + (1, 1, 0), + f'the two round trips are the list and the get, and the describe must not happen - a ' + f'NOT_FOUND that reached the descriptor read would cost three, which is MORE than ' + f'kExpectNotFoundRoundTrips charges and would make the bound an understatement: ' + f'{delta}') + + def test_08_a_declared_but_unset_pin_is_a_success_costing_three(self): + """A declared-but-unset parameter is NOT the two-round-trip NOT_FOUND path. + + This is worth its own case because it is what that path used to be documented as. A + parameter a node has declared and not set is answered by rclcpp's parameter service with a + ``PARAMETER_NOT_SET`` VALUE, not with an absent one, so the transport gets a parameter + back, goes on to the unconditional describe, and returns SUCCESS carrying null. It costs + the node three round trips, the same as any other resolved pin, and ``expect`` charges it + three - which is right. Had the charge of 2 really been sized for this case it would have + been an UNDERSTATEMENT of what the node pays, and the whole point of that constant is that + it may overstate and must never understate. + """ + response, delta = self._measure( + f'{API_BASE}/apps/{PIN_PROBE_NODE}/configurations/{NOT_SET_PARAM}', + probe=self._pin_probe) + self.assertEqual( + response.status_code, 200, + f'a parameter the node declares but has not set is a value the node HAS (an unset ' + f'one), so the read succeeds: {response.status_code}, {response.text[:400]}') + self.assertIsNone( + response.json().get('data'), + f'the unset parameter came back carrying a value: {response.text[:400]}') + + self._assert_round_trips( + delta, 3, + 'a DECLARED BUT UNSET pin (list, get, describe - the same success path as any other ' + 'resolved pin, which is why expect charges it kExpectReadRoundTrips and not ' + 'kExpectNotFoundRoundTrips)') + self.assertEqual( + delta['describe_parameters'], 1, + f'the descriptor round trip is what makes this a three rather than a two, and it did ' + f'not happen: {delta}') + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + """Verify the gateway/fault_manager stack exits cleanly.""" + + def test_exit_codes(self, proc_info): + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'Process {info.process_name} exited with {info.returncode}', + ) diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_integration.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_integration.cpp new file mode 100644 index 00000000..ea89e1f0 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_integration.cpp @@ -0,0 +1,3693 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Drives the REAL mechanism: a target node exposes parameters over DDS, the detector +// reads them via Ros2ParameterTransport, and raises/clears through a real ReportFault +// service round-trip to a fake fault_manager. NOT a full-gateway e2e; this proves the +// detector end to end within its own scope. +// +// The detector is snapshot-driven: tick() early-returns unless ctx.snapshot is set, iterates +// ctx.snapshot->apps (id + fqn), and raises ONE aggregated fault whose source_id is always +// graph_source_id() - the constant "graph_watchdog", the entity the plugin owns. It never +// depends on what the snapshot contains. So every test seeds the snapshot via set_apps() and +// asserts on the "graph_watchdog" source, never a per-node FQN and never a component id. +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ros2_medkit_gateway/core/providers/introspection_provider.hpp" +#include "ros2_medkit_gateway/core/transports/parameter_transport.hpp" +#include "ros2_medkit_graph_watchdog/aggregated_fault.hpp" +#include "ros2_medkit_graph_watchdog/detector.hpp" +#include "ros2_medkit_graph_watchdog/detector_registry.hpp" +#include "ros2_medkit_graph_watchdog/reliability_gate.hpp" + +using ros2_medkit_gateway::App; +using ros2_medkit_gateway::Component; +using ros2_medkit_gateway::IntrospectionInput; +using ros2_medkit_graph_watchdog::AggregatedFault; +using ros2_medkit_graph_watchdog::DetectorContext; +using ros2_medkit_graph_watchdog::DetectorMode; +using ros2_medkit_graph_watchdog::reliability_allows; +using ros2_medkit_graph_watchdog::ReliabilityGate; +using ReportFault = ros2_medkit_msgs::srv::ReportFault; +using namespace std::chrono_literals; + +namespace { +// The detector class is file-local in param_drift_detector.cpp but self-registers via +// REGISTER_DETECTOR, which runs when that .cpp is linked into this test. Pull an +// instance from the registry (no production factory needed). +std::unique_ptr make_param_drift() { + for (auto & d : ros2_medkit_graph_watchdog::DetectorRegistry::instance().create_all()) { + if (d->id() == "param_drift") { + return std::move(d); + } + } + return nullptr; +} + +/// Runs `fn` when it leaves scope, on EVERY exit path - including the one a failed ASSERT takes. +/// +/// The teardown this file needs is not optional cleanup. A node removed from the fixture's +/// executor only on the success path is still registered with a RUNNING MultiThreadedExecutor when +/// an ASSERT above destroys it, and the process SIGABRTs on a dangling callback group: a readable +/// assertion failure turns into a crash that says nothing about what went wrong. The same applies +/// to a joinable std::thread, where the failure mode is std::terminate. +template +class ScopeExit { + public: + explicit ScopeExit(FnT fn) : fn_(std::move(fn)) { + } + ~ScopeExit() { + // A destructor must not throw: an escape here during stack unwinding from a failed assertion + // is std::terminate, which is strictly worse than the leak this guard exists to prevent. + try { + fn_(); + } catch (const std::exception &) { + // Swallowed on purpose, per the note above. + } + } + ScopeExit(const ScopeExit &) = delete; + ScopeExit & operator=(const ScopeExit &) = delete; + ScopeExit(ScopeExit &&) = delete; + ScopeExit & operator=(ScopeExit &&) = delete; + + private: + FnT fn_; +}; + +template +ScopeExit on_scope_exit(FnT fn) { + return ScopeExit(std::move(fn)); +} + +/// Captures rcutils log output for as long as it is alive and restores the console handler on every +/// exit path. Leaving the process-global handler installed would swallow the log output of every +/// later case in this binary, so the restore cannot be left to the success path. +/// +/// One implementation for the whole file. The rcutils handler CONTRACT hands over a caller-supplied +/// format string and a va_list - it is not a choice the handler makes - so `vsnprintf` here is the +/// one place in this file that cannot use a literal format. Written out per test, that was six +/// copies of the suppression and six guard types; written once, it is one. +/// +/// The handler is a plain C function pointer with no user-data slot, so the live capture is reached +/// through a file-static. It is written by the test thread and read by whichever thread logs (the +/// detector's reader thread does), hence the atomic. +class LogCapture { + public: + LogCapture() { + active().store(this); + rcutils_logging_set_output_handler(&LogCapture::handler); + } + ~LogCapture() { + rcutils_logging_set_output_handler(rcutils_logging_console_output_handler); + active().store(nullptr); + } + LogCapture(const LogCapture &) = delete; + LogCapture & operator=(const LogCapture &) = delete; + LogCapture(LogCapture &&) = delete; + LogCapture & operator=(LogCapture &&) = delete; + + /// How many captured lines contain `needle`. + int count(const std::string & needle) const { + std::lock_guard lk(mutex_); + return static_cast(std::count_if(lines_.begin(), lines_.end(), [&needle](const std::string & line) { + return line.find(needle) != std::string::npos; + })); + } + + private: + static std::atomic & active() { + static std::atomic current{nullptr}; + return current; + } + + static void handler(const rcutils_log_location_t * /*location*/, int /*severity*/, const char * /*name*/, + rcutils_time_point_value_t /*timestamp*/, const char * format, va_list * args) { + char buf[1024]; + va_list copy; + va_copy(copy, *args); + // The format string arrives from the logging call site through the handler signature, so there + // is no literal to write here and no other way to reach the arguments than the va_list. The + // build runs -Werror=format=2; GCC exempts va_list-taking formatters from -Wformat-nonliteral, + // clang does not, so the CI clang-tidy job fails on this line alone. A pragma rather than a + // lint-suppression comment, because the diagnostic is raised as an ERROR and clang-tidy does + // not honour suppression comments for those. Scoped to the single call. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" + vsnprintf(buf, sizeof(buf), format, copy); +#pragma GCC diagnostic pop + va_end(copy); + LogCapture * capture = active().load(); + if (capture == nullptr) { + return; + } + std::lock_guard lk(capture->mutex_); + capture->lines_.emplace_back(buf); + } + + mutable std::mutex mutex_; + std::vector lines_; +}; + +/// Adapts a body that wants the request and the response by reference to the shape rclcpp requires. +/// +/// create_service deduces the callback by comparing its argument tuple against +/// `std::function, std::shared_ptr)>` EXACTLY, so a handler +/// cannot declare those parameters by reference however little it needs a copy of them. Stating +/// that shape once here keeps it out of every fake service below. +template +std::function, std::shared_ptr)> +service_callback(BodyT body) { + return [body = std::move(body)](std::shared_ptr req, + std::shared_ptr resp) { + body(*req, *resp); + }; +} + +/// The four parameter services a hand-rolled probe node needs beyond list/get, purely so the +/// transport's `AsyncParametersClient` reports ready. +/// +/// wait_for_service() on that client waits for ALL SIX; a node missing any of them is answered +/// SERVICE_UNAVAILABLE and negative-cached before a single request reaches its handlers. A test +/// about what a READ costs or how long one may take is then measuring a node that was never read at +/// all - which is exactly how a first attempt at the per-read-bound case below passed for the wrong +/// reason. These four only ever have to exist; nothing in this file drives them. +std::vector add_idle_parameter_services(const rclcpp::Node::SharedPtr & node, + const std::string & fqn) { + return { + node->create_service( + fqn + "/describe_parameters", service_callback( + [](const rcl_interfaces::srv::DescribeParameters::Request & req, + rcl_interfaces::srv::DescribeParameters::Response & resp) { + for (const auto & name : req.names) { + rcl_interfaces::msg::ParameterDescriptor d; + d.name = name; + d.type = rcl_interfaces::msg::ParameterType::PARAMETER_DOUBLE; + resp.descriptors.push_back(d); + } + })), + node->create_service( + fqn + "/get_parameter_types", service_callback( + [](const rcl_interfaces::srv::GetParameterTypes::Request & req, + rcl_interfaces::srv::GetParameterTypes::Response & resp) { + resp.types.assign(req.names.size(), + rcl_interfaces::msg::ParameterType::PARAMETER_DOUBLE); + })), + node->create_service( + fqn + "/set_parameters", service_callback( + [](const rcl_interfaces::srv::SetParameters::Request & req, + rcl_interfaces::srv::SetParameters::Response & resp) { + for (std::size_t i = 0; i < req.parameters.size(); ++i) { + rcl_interfaces::msg::SetParametersResult r; + r.successful = false; + r.reason = "probe is read-only"; + resp.results.push_back(r); + } + })), + node->create_service( + fqn + "/set_parameters_atomically", + service_callback( + [](const rcl_interfaces::srv::SetParametersAtomically::Request & /*req*/, + rcl_interfaces::srv::SetParametersAtomically::Response & resp) { + resp.result.successful = false; + resp.result.reason = "probe is read-only"; + })), + }; +} +} // namespace + +/// A parameter transport under the test's control. The detector's reads are blocking service +/// round-trips against a live node, which makes three behaviours unreachable from a real ROS graph: +/// a read that is still in flight when its app is de-armed, a SUCCESS response that carries no +/// parameters at all, and a transport that cannot be constructed. Each of those guards a real +/// failure mode, so each needs a stand-in to be provable. +class ScriptedTransport : public ros2_medkit_gateway::ParameterTransport { + public: + bool is_self_node(const std::string & /*node*/) const override { + return false; + } + ros2_medkit_gateway::ParameterResult list_parameters(const std::string & node) override { + throw_if_scripted(node); + if (is_unreadable(node)) { + ros2_medkit_gateway::ParameterResult r; + r.success = false; // a node with no working parameter service - no live node can be made to + r.error_code = ros2_medkit_gateway::ParameterErrorCode::SERVICE_UNAVAILABLE; + return r; // answer this way on demand, which is why the stand-in has to + } + enter_read(node); + ros2_medkit_gateway::ParameterResult r; + r.success = true; + { + // Under `m_`, all of it. enter_read() releases the lock when it returns, so reading + // empty_batch_ out here is a plain bool racing set_empty_batch() on the test thread against + // the reader thread - which TSan fails the whole integration test for, and which is a real + // race and not a test artefact: the stand-in stands in for a transport the reader calls + // concurrently with the test reconfiguring it. + std::lock_guard lk(m_); + if (empty_batch_) { + r.data = nlohmann::json::array(); // SUCCESS carrying no view - a legitimate transport reply + return r; + } + const auto it = node_params_.find(node); + if (it != node_params_.end()) { + r.data = nlohmann::json::array(); + for (const auto & [name, value] : it->second) { + r.data.push_back(nlohmann::json{{"name", name}, {"value", value}, {"type", "double"}}); + } + return r; + } + } + r.data = nlohmann::json::array({{{"name", "gain"}, {"value", value_for(node)}, {"type", "double"}}}); + return r; + } + ros2_medkit_gateway::ParameterResult get_parameter(const std::string & node, const std::string & name) override { + throw_if_scripted(node); + if (is_unreadable(node)) { + ros2_medkit_gateway::ParameterResult r; + r.success = false; + r.error_code = ros2_medkit_gateway::ParameterErrorCode::SERVICE_UNAVAILABLE; + return r; + } + enter_read(node); + { + std::lock_guard lk(m_); + if (restrict_declared_) { + const auto per_node = node_declared_.find(node); + const auto & names = per_node != node_declared_.end() ? per_node->second : declared_; + if (names.count(name) == 0) { + ros2_medkit_gateway::ParameterResult r; + r.success = false; + r.error_code = ros2_medkit_gateway::ParameterErrorCode::NOT_FOUND; + return r; + } + } + } + ros2_medkit_gateway::ParameterResult r; + r.success = true; + r.data = nlohmann::json{{"name", name}, {"value", value_for(node)}, {"type", "double"}}; + return r; + } + ros2_medkit_gateway::ParameterResult set_parameter(const std::string & /*node*/, const std::string & /*name*/, + const nlohmann::json & /*value*/) override { + return {}; + } + ros2_medkit_gateway::ParameterResult list_own_parameters() override { + return {}; + } + ros2_medkit_gateway::ParameterResult get_own_parameter(const std::string & /*name*/) override { + return {}; + } + ros2_medkit_gateway::ParameterResult get_default(const std::string & /*node*/, + const std::string & /*name*/) override { + return {}; + } + ros2_medkit_gateway::ParameterResult list_defaults(const std::string & /*node*/) override { + return {}; + } + bool is_node_available(const std::string & /*node*/) const override { + return true; + } + void shutdown() override { + std::lock_guard lk(m_); + blocked_ = false; // never let a teardown wait on a gate the test forgot to open + blocked_nodes_.clear(); + gate_.notify_all(); + } + + void block() { + std::lock_guard lk(m_); + blocked_ = true; + } + void release() { + std::lock_guard lk(m_); + blocked_ = false; + gate_.notify_all(); + } + /// Hold reads of ONE node. The reader thread is single-threaded, so blocking everything stops + /// the whole loop: a test that needs to observe the reader moving past a held read has to leave + /// the other targets running. + void block_node(const std::string & node) { + std::lock_guard lk(m_); + blocked_nodes_.insert(node); + } + void release_node(const std::string & node) { + std::lock_guard lk(m_); + blocked_nodes_.erase(node); + gate_.notify_all(); + } + /// Wait until the reader is actually inside a read, so the test never races the thread. + bool wait_until_reading(int at_least) { + std::unique_lock lk(m_); + return entered_.wait_for(lk, 5s, [&] { + return calls_ >= at_least; + }); + } + /// Wait until a read is actually HELD on the gate. The call counter cannot express this: a read + /// that entered just before block_node() is counted but runs to completion, so a test keying off + /// the count can proceed while nothing is held at all. + bool wait_until_parked() { + std::unique_lock lk(m_); + return parked_cv_.wait_for(lk, 5s, [this] { + return parked_ > 0; + }); + } + /// Default value for every node. + void set_value(double v) { + value_.store(v); + } + /// Override the value of ONE node. Needed whenever a test drifts a single app: the shared value + /// would drift every target at once, and the assertion would then pass on the wrong app. + void set_value(const std::string & node, double v) { + std::lock_guard lk(m_); + node_values_[node] = v; + } + /// Replace the whole parameter set ONE node answers `list_parameters` with. Left unset, the + /// stand-in reports a single `gain`, which is enough to drift a node but nowhere near enough to + /// fill the detector's per-app share of the aggregated description - so a test about which + /// entries survive that cap cannot be written with it at all. Per-node for the same reason + /// set_value and set_declared are: one shared set makes every node answer alike, and the point + /// here is that different nodes contribute different amounts of text. + void set_params(const std::string & node, std::map params) { + std::lock_guard lk(m_); + node_params_[node] = std::move(params); + } + /// Make one node permanently unreadable, whatever else the stand-in does. + void set_unreadable(const std::string & node) { + std::lock_guard lk(m_); + unreadable_.insert(node); + } + /// Undo set_unreadable. A test that proves an unreadable node cannot CLEAR a fault needs the + /// positive control that a readable one still can, or the silence it asserts is satisfied by a + /// detector that stopped emitting anything at all. + void set_readable(const std::string & node) { + std::lock_guard lk(m_); + unreadable_.erase(node); + } + /// Make one node's reads THROW. rclcpp can throw out of a read when the context is invalidated + /// mid-call, or on a malformed service path; a live node cannot be asked to do it on demand. + void set_throwing(const std::string & node) { + std::lock_guard lk(m_); + throwing_.insert(node); + } + bool wait_until_threw(int at_least) { + std::unique_lock lk(m_); + return threw_.wait_for(lk, 5s, [&] { + return throws_ >= at_least; + }); + } + /// How many reads have been REJECTED because their node is unreadable. `calls()` cannot express + /// this: an unreadable node fails before enter_read, so it is never counted there. A test whose + /// subject is a THRESHOLD on failed attempts has to pace itself off the attempts themselves - + /// keying off wall clock instead makes the assertion a function of the pacing budget, which is + /// the very coupling this counter exists to let a test pin. + int failures() { + std::lock_guard lk(m_); + return failures_; + } + bool wait_until_failed(int at_least) { + std::unique_lock lk(m_); + return failed_.wait_for(lk, 30s, [&] { + return failures_ >= at_least; + }); + } + void set_empty_batch(bool on) { + std::lock_guard lk(m_); + empty_batch_ = on; + } + /// Restrict which parameter names the node admits to having. Left unset, the stand-in answers + /// any name, which is precisely the behaviour no real node has: a real one returns NOT_FOUND for + /// anything it never declared, and that reply is deliberately not treated as a drift. + void set_declared(std::set names) { + std::lock_guard lk(m_); + declared_ = std::move(names); + restrict_declared_ = true; + } + /// Restrict which names ONE node admits to having, overriding the shared set for that node. + /// Same reason set_value has a per-node form: with a single shared set every node answers alike, + /// so a test about one node declaring a pin that the rest of the graph does not cannot be + /// written at all - and "the sweep has not reached the node that declares it yet" is exactly the + /// state the unmatched-pin warning must not fire in. + void set_declared(const std::string & node, std::set names) { + std::lock_guard lk(m_); + node_declared_[node] = std::move(names); + restrict_declared_ = true; + } + int calls() { + std::lock_guard lk(m_); + return calls_; + } + + private: + std::mutex m_; + std::condition_variable gate_; + std::condition_variable entered_; + std::condition_variable parked_cv_; + std::condition_variable threw_; + std::condition_variable failed_; + bool blocked_ = false; + bool empty_batch_ = false; + std::set unreadable_; + std::set blocked_nodes_; + std::set throwing_; + std::set declared_; + std::map> node_declared_; + bool restrict_declared_ = false; + std::map node_values_; + std::map> node_params_; + int calls_ = 0; + int parked_ = 0; + int throws_ = 0; + int failures_ = 0; + std::atomic value_{1.0}; + + bool is_unreadable(const std::string & node) { + std::lock_guard lk(m_); + if (unreadable_.count(node) == 0) { + return false; + } + ++failures_; + failed_.notify_all(); + return true; + } + + /// Throw out of the read itself, before anything is counted as a call. The throw has to leave + /// the transport the way a real one would - from inside the blocking call, not around it. + void throw_if_scripted(const std::string & node) { + { + std::lock_guard lk(m_); + if (throwing_.count(node) == 0) { + return; + } + ++throws_; + threw_.notify_all(); + } + throw std::runtime_error("scripted transport failure"); + } + + /// Count the call, then hold it for as long as the test blocks this node. `parked_` is + /// incremented only on the path that actually waits, so wait_until_parked() means a read is + /// held right now rather than merely started. + void enter_read(const std::string & node) { + std::unique_lock lk(m_); + ++calls_; + entered_.notify_all(); + const auto held = [this, &node] { + return blocked_ || blocked_nodes_.count(node) != 0; + }; + if (!held()) { + return; + } + ++parked_; + parked_cv_.notify_all(); + gate_.wait(lk, [&held] { + return !held(); + }); + --parked_; + } + + /// Read AFTER the gate, so a value the test sets while a read is held is the one that read + /// returns - which is how a stale in-flight sample is scripted. + double value_for(const std::string & node) { + std::lock_guard lk(m_); + const auto it = node_values_.find(node); + return it != node_values_.end() ? it->second : value_.load(); + } +}; + +class ParamDriftIntegrationTest : public ::testing::Test { + protected: + // Must run before the FIRST test's fixture is constructed: `exec_` is a fixture data + // member, so its default constructor (which touches the global rclcpp context to + // create an interrupt guard condition) runs before SetUp(). Doing rclcpp::init() only + // in SetUp() is too late for that first construction; SetUpTestSuite() runs earlier. + static void SetUpTestSuite() { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + } + + void SetUp() override { + gateway_ = std::make_shared("pd_it_gateway"); + target_ = std::make_shared("pd_it_target"); + target_->declare_parameter("speed_limit", 1.0); + sink_ = std::make_shared("pd_it_sink"); + srv_ = sink_->create_service( + "/fault_manager/report_fault", + service_callback([this](const ReportFault::Request & req, ReportFault::Response & resp) { + { + std::lock_guard lk(mtx_); + received_.push_back(req); + } + resp.accepted = true; // ReportFault.srv response field is `bool accepted` + })); + client_ = gateway_->create_client("/fault_manager/report_fault"); + exec_.add_node(gateway_); + exec_.add_node(target_); + exec_.add_node(sink_); + spin_ = std::thread([this]() { + exec_.spin(); + }); + ASSERT_TRUE(client_->wait_for_service(5s)); + } + void TearDown() override { + exec_.cancel(); + if (spin_.joinable()) { + spin_.join(); + } + exec_.remove_node(gateway_); + exec_.remove_node(target_); + exec_.remove_node(sink_); + client_.reset(); + srv_.reset(); + sink_.reset(); + target_.reset(); + gateway_.reset(); + } + + DetectorContext make_ctx(DetectorMode mode) { + DetectorContext ctx; + ctx.gateway_node = gateway_.get(); + ctx.node_mutex = &node_mutex_; + ctx.mode = mode; + ctx.gate = nullptr; // ungated by default; the gate-keying test wires a real gate in + ctx.fault_client = client_; + ctx.snapshot = &snapshot_; // tick() early-returns without a snapshot; each test seeds set_apps() + return ctx; + } + + // (Re)populate the owned entity snapshot the detector reads each tick. Uses the + // `App a; a.field = ...;` idiom (App has a RosBinding sub-struct, so designated/partial + // init trips -Wmissing-field-initializers under our strict flags). + // + // The aggregated fault's source_id does NOT depend on this snapshot: it is always + // graph_source_id(), the entity the plugin owns. SourceIdIgnoresTheHostComponent below seeds + // a component precisely to prove that, because a host Component id would match no entity + // scope and the fault would be reachable from no endpoint at all. + void set_apps(const std::vector> & id_fqn) { + snapshot_.apps.clear(); + for (const auto & [id, fqn] : id_fqn) { + App a; + a.id = id; + a.bound_fqn = fqn; + snapshot_.apps.push_back(a); + } + } + + // Wait until 's parameter service is discoverable, so the detector's first + // read succeeds and captures the baseline BEFORE any set_parameter. Without this, an + // early negative-cache miss (service not yet discovered) could make the first + // successful read land after the drift and capture the drifted value as the baseline. + bool wait_param_service(const std::string & node_fqn) { + std::lock_guard lk(node_mutex_); + auto c = gateway_->create_client(node_fqn + "/get_parameters"); + return c->wait_for_service(5s); + } + + // How many matching faults are currently in the log (for absence checks + snapshots). + std::size_t count_faults(const std::string & source_id, uint8_t event_type) { + std::lock_guard lk(mtx_); + std::size_t n = 0; + for (const auto & r : received_) { + if (r.source_id == source_id && r.fault_code == "GRAPH_PARAM_DRIFT" && r.event_type == event_type) { + ++n; + } + } + return n; + } + + // The description of the MOST RECENT FAILED report. That is what the fault store keeps: it + // overwrites the description on every report, so it is the only text an operator can read. A + // helper that scans the whole history can pass on a single transient report and hide the fact + // that the current record no longer names the entry. + std::string latest_failed_desc(const std::string & source_id) { + std::lock_guard lk(mtx_); + for (auto it = received_.rbegin(); it != received_.rend(); ++it) { + if (it->source_id == source_id && it->fault_code == "GRAPH_PARAM_DRIFT" && + it->event_type == ReportFault::Request::EVENT_FAILED) { + return it->description; + } + } + return {}; + } + + // The description of the FIRST failed report that mentions `needle`, or empty if there is none. + // + // Deliberately NOT the latest one. Any ordering rule keyed on an app being newly affected only + // decides the single report in which that app first appears: by the next tick the app has been + // reported, moves into the "already reported" bucket, and every ordering agrees again. Asserting + // on the latest record therefore passes under a wrong bucket order as soon as one more tick has + // gone by, which is what let the pinned-versus-self-captured cross combination go unexercised. + std::string first_failed_desc_containing(const std::string & source_id, const std::string & needle) { + std::lock_guard lk(mtx_); + for (const auto & r : received_) { + if (r.source_id == source_id && r.fault_code == "GRAPH_PARAM_DRIFT" && + r.event_type == ReportFault::Request::EVENT_FAILED && r.description.find(needle) != std::string::npos) { + return r.description; + } + } + return {}; + } + + // True if the CURRENT stored description names EVERY needle - used to prove one aggregated fault + // names all the drifting apps. Deliberately the latest record and not the history: the store + // overwrites the description on every report, so a history scan can pass on one transient report + // while the text an operator actually reads no longer names the entry. + bool latest_failed_desc_contains(const std::string & source_id, const std::vector & needles) { + const std::string desc = latest_failed_desc(source_id); + return !desc.empty() && std::all_of(needles.begin(), needles.end(), [&desc](const std::string & needle) { + return desc.find(needle) != std::string::npos; + }); + } + + // Tick until a NEW matching fault appears beyond `baseline_count` (i.e. arrived AFTER the + // trigger), or timeout. Requiring the fault to be new is what makes this a real proof: a + // detector whose comparison is gutted but still emits a spurious raise+clear during + // warm-up cannot satisfy it, because those land before the snapshot. + // + // The loop bound is 130 iterations (13s at 100ms) so the poll window EXCEEDS the + // transport's kNegativeCacheTtlSec (10s): the detector owns its transport (built from the + // registry) and the fixture cannot inject a shorter TTL, so a single timed-out read under + // ASan/TSan that negative-caches the target for 10s must not outlast the window. The happy + // path returns on first success, so responsive-node tests stay fast. + bool poll_for_new(const std::string & source_id, uint8_t event_type, std::size_t baseline_count, + ros2_medkit_graph_watchdog::Detector & det, DetectorContext & ctx) { + for (int i = 0; i < 130; ++i) { + det.tick(ctx); + std::this_thread::sleep_for(100ms); + if (count_faults(source_id, event_type) > baseline_count) { + return true; + } + } + return false; + } + + // Tick up to `iters` times, returning true as soon as `pred` holds. For non-count + // assertions (e.g. an aggregated description mentioning multiple apps). + template + bool poll_until(Pred pred, ros2_medkit_graph_watchdog::Detector & det, DetectorContext & ctx, int iters = 130) { + for (int i = 0; i < iters; ++i) { + det.tick(ctx); + std::this_thread::sleep_for(100ms); + if (pred()) { + return true; + } + } + return false; + } + + // EVIDENCE that every armed app has actually been read, to be taken before a test perturbs + // anything. Ticking a fixed number of times and hoping is not the same claim. + // + // The detector withholds its clear while any armed app is still unread (see emit_aggregated), so + // a PASSED event is not merely "nothing drifts" - it is the detector stating that it looked + // everywhere and found nothing. wait_param_service() cannot give that: it waits on the GATEWAY + // node's own client and says nothing about the transport's hidden one. Without this gate a first + // read that misses its window negative-caches the node for ten seconds, the first successful read + // lands after the parameter was changed, and the DRIFTED value silently becomes the baseline. + // + // Requires a NEW clear rather than any clear: a gate-denied app arms nothing, so a detector with + // an empty armed set clears on every tick, and "a clear exists" would then be satisfied before + // the app under test had been looked at even once. + bool wait_for_baseline_evidence(ros2_medkit_graph_watchdog::Detector & det, DetectorContext & ctx) { + const auto before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + return poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, before, det, ctx); + } + + rclcpp::Node::SharedPtr gateway_, target_, sink_; + rclcpp::Service::SharedPtr srv_; + rclcpp::Client::SharedPtr client_; + rclcpp::executors::MultiThreadedExecutor exec_; + std::thread spin_; + std::mutex mtx_, node_mutex_; + std::vector received_; + IntrospectionInput snapshot_; // owned entity snapshot the detector reads each tick +}; + +TEST_F(ParamDriftIntegrationTest, RuntimeSetRaisesThenRestoreClears) { + set_apps({{"pd_it_target", "/pd_it_target"}}); + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // defaults: baseline true, no expect + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_target")); // guarantee the baseline read succeeds first + // Evidence, not a fixed sleep: the clear only arrives once the target has actually been read. + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear ever arrived, so the target was never read and the change below would be " + "measured against no baseline at all"; + // Baseline capture alone MUST be silent (kills an "always-raise" detector). + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); + + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + target_->set_parameter(rclcpp::Parameter("speed_limit", 0.2)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + target_->set_parameter(rclcpp::Parameter("speed_limit", 1.0)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)); +} + +// The fault must be raised under the entity the plugin OWNS, never under the host Component, +// even when the snapshot carries one - which in a real gateway it always does. A host Component +// built by HostInfoProvider never sets `external`, and a non-external component does not claim +// its own bare id in a fault scope set, so a fault raised under it is listed by no entity +// endpoint at all: not /apps/graph_watchdog/faults, not /components//faults. It would show +// up only in the flat /faults list, which is also all the e2e polls - so nothing else here can +// catch this. Every other test in this file leaves `components` empty, which is exactly why the +// regression it guards went unnoticed. +TEST_F(ParamDriftIntegrationTest, SourceIdIgnoresTheHostComponent) { + Component host; + host.id = "pd-it-host"; + host.name = "pd-it-host"; + host.fqn = "pd-it-host"; + snapshot_.components.push_back(host); + + set_apps({{"pd_it_target", "/pd_it_target"}}); // speed_limit is declared in SetUp + + auto det = make_param_drift(); + det->configure(nlohmann::json{{"max_reads_per_tick", 4}}); + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_target")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear ever arrived, so the target was never read and no baseline exists to drift from"; + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + ASSERT_EQ(failed_before, 0u) << "baseline capture must be silent for the rest of this test to mean anything"; + target_->set_parameter(rclcpp::Parameter("speed_limit", 0.2)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // The discriminator: nothing was ever reported under the component id. + EXPECT_EQ(count_faults("pd-it-host", ReportFault::Request::EVENT_FAILED), 0u); + EXPECT_EQ(count_faults("pd-it-host", ReportFault::Request::EVENT_PASSED), 0u); +} + +TEST_F(ParamDriftIntegrationTest, ExpectRuleFlagsWrongFromStart) { + // A separate node that comes up already violating the production pin use_sim_time=false. + auto bad = std::make_shared("pd_it_badsim"); + bad->set_parameter(rclcpp::Parameter("use_sim_time", true)); + exec_.add_node(bad); + const auto bad_guard = on_scope_exit([this, bad] { + exec_.remove_node(bad); + }); + set_apps({{"pd_it_badsim", "/pd_it_badsim"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json{{"baseline", false}, {"expect", {{"use_sim_time", false}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_badsim")); // service up before we assert a read-driven raise + // Live use_sim_time (true) != expected (false) -> a NEW raise appears with no param set, + // on first read. A detector that never actually compares would produce no such fault. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); +} + +// A node whose parameter services EXIST (discovery succeeds) but which never answers them - the +// #531 unresponsive-node scenario. What has to hold is not a property of tick(): tick() does no IO +// at all in this design, so the time it takes says nothing about a read timeout, and the same +// assertion passes with the timeout raised to an hour. The bounded read is a property of the READER +// THREAD, and the way to see it is to require the sweep to get PAST the node that will not answer. +// +// The unresponsive app sorts FIRST, so the reader's round-robin reaches it before anything else. +// With the read bounded, it times out and the healthy app behind it is served, over and over. With +// the bound removed the reader parks on the very first target for good: the healthy app is never +// read, its `expect` violation is never seen, and neither assertion below can be satisfied. +// +// Teardown is the second half. The detector is destroyed while the fake service is STILL wedged - +// the test does not release it first - because that is the shape of a real shutdown: its destructor +// has to shut the transport down and JOIN the reader with a read outstanding. +TEST_F(ParamDriftIntegrationTest, AnUnresponsiveNodeDoesNotStallTheReader) { + const std::string kUnresp = "pd_it_aunresp"; // sorts before pd_it_target: first in the rotation + std::atomic stop{false}; + auto unresp = std::make_shared(kUnresp, rclcpp::NodeOptions().start_parameter_services(false)); + // list_parameters is the round-trip both read modes make first; it never replies. It loops on + // `stop` (not rclcpp::ok()) so the fake node can be released deterministically at the very end. + auto list_srv = unresp->create_service( + "/" + kUnresp + "/list_parameters", service_callback( + [&stop](const rcl_interfaces::srv::ListParameters::Request & /*req*/, + rcl_interfaces::srv::ListParameters::Response & /*resp*/) { + while (!stop.load()) { + std::this_thread::sleep_for(10ms); + } + })); + // get_parameters only needs to exist in the graph so discovery (wait_param_service) succeeds. + auto get_srv = unresp->create_service( + "/" + kUnresp + "/get_parameters", service_callback( + [](const rcl_interfaces::srv::GetParameters::Request & /*req*/, + rcl_interfaces::srv::GetParameters::Response & /*resp*/) {})); + rclcpp::executors::SingleThreadedExecutor unresp_exec; + unresp_exec.add_node(unresp); + std::thread unresp_spin([&unresp_exec] { + unresp_exec.spin(); + }); + // Declared BEFORE the detector so it is destroyed AFTER it: the wedged handler owns the fake + // node's only executor thread, so nothing can join that thread until `stop` is set, and the + // detector's own teardown must happen while the node is still refusing to answer. + const auto release_guard = on_scope_exit([&stop, &unresp_exec, &unresp_spin] { + stop.store(true); + unresp_exec.cancel(); + if (unresp_spin.joinable()) { + unresp_spin.join(); + } + }); + + set_apps({{kUnresp, "/" + kUnresp}, {"pd_it_target", "/pd_it_target"}}); + auto det = make_param_drift(); + // `expect` mode: the healthy target declares speed_limit=1.0 (see SetUp) and the pin says 0.0, so + // it violates from its very first read and no baseline warm-up is needed - which matters here, + // because the app that would hold up a warm-up is precisely the one that never answers. + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 100}, {"max_reads_per_tick", 8}, {"expect", {{"speed_limit", 0.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/" + kUnresp)); // services discoverable (but unresponsive) + ASSERT_TRUE(wait_param_service("/pd_it_target")); // the healthy app behind it + + // The sweep got past the node that will not answer. + ASSERT_TRUE(poll_until( + [this]() { + return latest_failed_desc_contains("graph_watchdog", {"pd_it_target", "got=1.0"}); + }, + *det, ctx)) + << "the healthy app behind the unresponsive one was never read, so the reader is parked on a " + "read that never returns: " + << latest_failed_desc("graph_watchdog"); + + // And it KEEPS getting past it. A description that names the new value can only come from a read + // taken after the change, i.e. from a rotation that came round again. + target_->set_parameter(rclcpp::Parameter("speed_limit", 7.25)); + EXPECT_TRUE(poll_until( + [this]() { + return latest_failed_desc_contains("graph_watchdog", {"got=7.25"}); + }, + *det, ctx)) + << "the healthy app was read once and never again, so the sweep stopped rotating: " + << latest_failed_desc("graph_watchdog"); + + // Nothing was invented about the node that never answered: it is skipped, never raised against. + EXPECT_EQ(latest_failed_desc("graph_watchdog").find(kUnresp), std::string::npos) + << "the unresponsive node was reported as drifted on a read that never completed"; + + // Teardown with the fake service still wedged. An unbounded read would leave the reader inside + // rcl when the destructor joins it, and the join would never return. + const auto teardown_started = std::chrono::steady_clock::now(); + det.reset(); + const auto teardown = std::chrono::steady_clock::now() - teardown_started; + EXPECT_LT(teardown, 8s) << "destroying the detector while a node was refusing to answer took " + << std::chrono::duration_cast(teardown).count() + << " ms: the reader could not be joined out of an unbounded read"; +} + +// Namespaced-node gate keying, with a REAL wired gate. The discriminator is LIFECYCLE +// suppression, NOT "FQN denied": the gate has two allow-paths - a KNOWN entity is allowed iff +// armed && lifecycle-ok; an UNKNOWN source_id is allowed once past the global bringup grace. +// So an unknown FQN key past grace returns TRUE. The teeth: a known+armed entity whose +// lifecycle is non-active is suppressed (false) via its app.id key, while the SAME string used +// as an unknown FQN key is allowed (true) because it bypasses per-entity lifecycle. The detector +// keys reliability_allows() by app.id; an FQN-keying detector would slip past the suppression. +// +// The negative window is not a hand-picked constant. Silence only means something if it lasted +// longer than a raise demonstrably needs, so the positive control below is TIMED - from the moment +// the gate opens to the moment the raise lands - and the window the negative phase watched for has +// to be at least that long. A 500 ms window next to a 13 s positive poll is not a discriminator, it +// is an accident of how long the negative loop happened to run. +TEST_F(ParamDriftIntegrationTest, NamespacedNodeGateKeyingLifecycleDiscriminator) { + // node n in namespace /ns: FQN /ns/n != app.id n + auto nsnode = std::make_shared("n", "/ns"); + nsnode->declare_parameter("gain", 1.0); + exec_.add_node(nsnode); + const auto nsnode_guard = on_scope_exit([this, nsnode] { + exec_.remove_node(nsnode); + }); + ASSERT_TRUE(wait_param_service("/ns/n")); + IntrospectionInput snap; + { + App a; + a.id = "n"; + a.bound_fqn = "/ns/n"; + snap.apps.push_back(a); + } + ReliabilityGate gate(3, gateway_.get(), &node_mutex_); + gate.update(snap, 5); + gate.update(snap, 8); // 3 elapsed -> n armed, global grace met + gate.set_lifecycle_state_for_test("n", "inactive"); + ASSERT_FALSE(reliability_allows(&gate, "n")); // app.id key: armed but lifecycle-inactive -> SUPPRESSED + ASSERT_TRUE(reliability_allows(&gate, "/ns/n")); // FQN key: unknown + past grace -> allowed (the bug path) + auto det = make_param_drift(); + det->configure(nlohmann::json{{"tick_interval_ms", 100}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + ctx.gate = &gate; + ctx.snapshot = &snap; + // NEGATIVE phase: correct app.id keying keeps the drifting node suppressed. Timed from the first + // tick, because that is when an FQN-keying detector would start capturing. + const auto negative_started = std::chrono::steady_clock::now(); + for (int i = 0; i < 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + nsnode->set_parameter(rclcpp::Parameter("gain", 0.2)); + for (int i = 0; i < 30; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + const auto negative_window = std::chrono::steady_clock::now() - negative_started; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); + // ^ an FQN-keying detector would see "/ns/n" allowed -> capture + raise; 0 proves app.id keying. + // POSITIVE control: flip lifecycle active -> the same node's drift now raises (proves the read/aggregate + // path works, so the 0 above was genuine app.id suppression, not a dead read). + const auto positive_started = std::chrono::steady_clock::now(); + gate.set_lifecycle_state_for_test("n", "active"); + ASSERT_TRUE(reliability_allows(&gate, "n")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "the node was never read once the gate opened, so the positive control below cannot time " + "anything"; // captures the current value (0.2) silently + const auto before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + nsnode->set_parameter(rclcpp::Parameter("gain", 0.9)); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, before, *det, ctx)); + const auto positive_latency = std::chrono::steady_clock::now() - positive_started; + + EXPECT_GE(negative_window, positive_latency) + << "the suppressed phase was watched for only " + << std::chrono::duration_cast(negative_window).count() + << " ms while an allowed node takes " + << std::chrono::duration_cast(positive_latency).count() + << " ms to arm, be read and raise: the silence above is the window being too short, not the " + "gate keying being right"; +} + +// A drift that is already reported must not be healed by the reliability gate closing on its +// app. The gate exists to hold back a RAISE about an entity that has not settled; it is not +// evidence that the parameter went back to its expected value. The aggregate's raise is gated on +// the aggregate's own source and never on the app, so dropping the app's finding when the gate +// denies it is the only thing that could clear it - straight into clear_fault, which carries no +// gate at all. A nav2 lifecycle_manager pausing a controller would then walk a still-present +// drift to HEALED and re-raise it on resume, flapping on every pause. +TEST_F(ParamDriftIntegrationTest, GateDenialDoesNotHealAPresentDrift) { + auto node = std::make_shared("g", "/gd"); + node->declare_parameter("gain", 1.0); + exec_.add_node(node); + const auto node_guard = on_scope_exit([this, node] { + exec_.remove_node(node); + }); + ASSERT_TRUE(wait_param_service("/gd/g")); + + IntrospectionInput snap; + { + App a; + a.id = "gd_frozen"; + a.bound_fqn = "/gd/g"; + snap.apps.push_back(a); + } + ReliabilityGate gate(3, gateway_.get(), &node_mutex_); + gate.update(snap, 5); + gate.update(snap, 8); // armed, global grace met + gate.set_lifecycle_state_for_test("gd_frozen", "active"); + ASSERT_TRUE(reliability_allows(&gate, "gd_frozen")); + + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + ctx.gate = &gate; + ctx.snapshot = &snap; + + for (int i = 0; i < 6; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + node->set_parameter(rclcpp::Parameter("gain", 0.2)); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // The gate closes while the parameter is still wrong. Nothing about the robot got better. + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + gate.set_lifecycle_state_for_test("gd_frozen", "inactive"); + ASSERT_FALSE(reliability_allows(&gate, "gd_frozen")); + for (int i = 0; i < 8; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "the gate closing healed a drift that is still present"; + EXPECT_NE(latest_failed_desc("graph_watchdog").find("gd_frozen"), std::string::npos) + << "the suppressed app vanished from the CURRENT description, so nothing tells the operator it " + "is still drifting: " + << latest_failed_desc("graph_watchdog"); + + // Positive control: the drift can still be cleared the only legitimate way, by fixing it. + gate.set_lifecycle_state_for_test("gd_frozen", "active"); + const auto passed_before2 = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + node->set_parameter(rclcpp::Parameter("gain", 1.0)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before2, *det, ctx)); +} + +// The other end of the freeze, and the one nothing covered. Freezing a finding is right for a +// PAUSE - the drift is still real and the gate only says "do not raise about an unsettled entity". +// It is wrong forever. A node that leaves `active` for good is never re-evaluated (not armed) and +// never pruned (still present in the graph), so without a release the aggregate keeps re-raising a +// finding nobody can act on: an operator who fixes the parameter and leaves the node deactivated +// watches GRAPH_PARAM_DRIFT stay CONFIRMED for the life of the gateway, naming a value that is no +// longer true. +// +// kFrozenHoldTicks is 60 consecutive unarmed ticks. Both ends of that are pinned here, because a +// release that fires too early is the flapping GateDenialDoesNotHealAPresentDrift forbids and a +// release that never fires is the latch above. Deleting the release leaves the suite green. +// +// The upper end is pinned by a SMALL number of further ticks rather than by a poll loop. The hold +// is a count the detector keeps privately, so the only thing that can hold the literal below to it +// is squeezing the release between "not yet after N" and "by N+3": a poll loop that ticks a +// hundred more times leaves the literal free to drift away from the constant it mirrors, and +// tripling kFrozenHoldTicks then stays green. +TEST_F(ParamDriftIntegrationTest, AFrozenFindingIsReleasedOnceTheHoldElapses) { + constexpr int kFrozenHoldTicks = 60; // mirrors the detector's constant; pinned from both sides below + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}}); + + IntrospectionInput snap; + { + App a; + a.id = "pd_it_hold"; + a.bound_fqn = "/pd_it_hold"; + snap.apps.push_back(a); + } + ReliabilityGate gate(3, gateway_.get(), &node_mutex_); + gate.update(snap, 5); + gate.update(snap, 8); // armed, global grace met + gate.set_lifecycle_state_for_test("pd_it_hold", "active"); + ASSERT_TRUE(reliability_allows(&gate, "pd_it_hold")); + + auto ctx = make_ctx(DetectorMode::Raise); + ctx.gate = &gate; + ctx.snapshot = &snap; + + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) << "the app was never read, so it has no baseline to drift from"; + ASSERT_NE(fake, nullptr); + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(9.0); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "the drift never raised, so there is no frozen finding whose release this test can observe"; + + // The gate closes and stays closed. No sleeps from here on: the hold is counted in TICKS, and + // while the app is unarmed the reader has no targets, so there is nothing to wait for. + gate.set_lifecycle_state_for_test("pd_it_hold", "inactive"); + ASSERT_FALSE(reliability_allows(&gate, "pd_it_hold")); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < kFrozenHoldTicks; ++i) { + det->tick(ctx); + } + // Reports are service round trips, so let the in-flight ones land before asserting that none of + // them was a clear. + std::this_thread::sleep_for(300ms); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "the frozen finding was released before the hold had elapsed, so an ordinary lifecycle " + "pause walks a still-present drift to HEALED"; + + // Three more ticks, no more. The release is due on the very next one. + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_GT(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "the finding stayed frozen past kFrozenHoldTicks, so a node that leaves 'active' for good " + "keeps GRAPH_PARAM_DRIFT raised on a parameter an operator may already have fixed"; + + // Positive control: releasing the finding is not the same as going deaf. The app arms again and + // the drift - which is still present - is re-raised from a fresh read. + gate.set_lifecycle_state_for_test("pd_it_hold", "active"); + const auto failed_after_release = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_after_release, *det, ctx)) + << "the released finding never came back once the app re-armed, even though the parameter is " + "still wrong: the release dropped the drift instead of re-deriving it"; +} + +// Node removal drives prune + clear e2e. Two nodes, one drifts and raises the aggregated +// fault; then the drifted node's App is REMOVED from the snapshot. After > kForgetGrace absent +// sweeps prune_vanished forgets it and drops it from drifted_, so the aggregate must CLEAR. +// Gutting prune_vanished to a no-op leaves the drift latched forever and fails this. +TEST_F(ParamDriftIntegrationTest, NodeRemovalPrunesAndClears) { + auto keep = std::make_shared("pd_it_keep"); + keep->declare_parameter("k", 1.0); + auto gone = std::make_shared("pd_it_gone"); + gone->declare_parameter("g", 1.0); + exec_.add_node(keep); + exec_.add_node(gone); + const auto nodes_guard = on_scope_exit([this, keep, gone] { + exec_.remove_node(keep); + exec_.remove_node(gone); + }); + set_apps({{"pd_it_keep", "/pd_it_keep"}, {"pd_it_gone", "/pd_it_gone"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_keep")); + ASSERT_TRUE(wait_param_service("/pd_it_gone")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear arrived, so at least one of the two apps was never read and has no baseline"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); + + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + gone->set_parameter(rclcpp::Parameter("g", 9.0)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // Remove the drifted node from the snapshot: after > kForgetGrace sweeps it is pruned, + // shedding its drift from the aggregate -> a clear must arrive. + set_apps({{"pd_it_keep", "/pd_it_keep"}}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)); +} + +// Forget raised path. Node drifts (aggregate raised) -> vanishes past grace (aggregate +// clears) -> reappears at a NEW healthy value distinct from its original baseline. Because +// forget() dropped its baseline, the first read after reappearance re-captures silently: NO +// stale/re-raised FAILED for it, and the aggregate stays CLEAR. If the baseline were NOT +// forgotten, the new healthy value would drift against the stale (original) baseline. +TEST_F(ParamDriftIntegrationTest, VanishedThenHealthyReappearanceStaysClear) { + auto keep = std::make_shared("pd_it_tbkeep"); + keep->declare_parameter("k", 1.0); + auto reap = std::make_shared("pd_it_reap"); + reap->declare_parameter("r", 1.0); + exec_.add_node(keep); + exec_.add_node(reap); + const auto nodes_guard = on_scope_exit([this, keep, reap] { + exec_.remove_node(keep); + exec_.remove_node(reap); + }); + set_apps({{"pd_it_tbkeep", "/pd_it_tbkeep"}, {"pd_it_reap", "/pd_it_reap"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_tbkeep")); + ASSERT_TRUE(wait_param_service("/pd_it_reap")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear arrived, so at least one of the two apps was never read and has no baseline"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); + + // Drift reap -> aggregate raised. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + reap->set_parameter(rclcpp::Parameter("r", 9.0)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // Vanish reap past grace -> pruned, aggregate clears. + set_apps({{"pd_it_tbkeep", "/pd_it_tbkeep"}}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)); + + // reap reappears at a NEW healthy value (5.0) - distinct from BOTH the original baseline + // (1.0) and the drifted value (9.0). This is deliberate: if forget() were missing, the + // stale baseline (1.0) would remain and 1.0 != 5.0 would raise a fresh spurious FAILED; with + // forget() in place, the stale baseline is gone and 5.0 is silently re-captured as the new + // baseline. Reusing 1.0 here would make the test pass vacuously even with a missing forget() + // (stale baseline == reappearance value looks "clean" either way). + reap->set_parameter(rclcpp::Parameter("r", 5.0)); // healthy, but distinct from the original baseline + set_apps({{"pd_it_tbkeep", "/pd_it_tbkeep"}, {"pd_it_reap", "/pd_it_reap"}}); + const auto failed_at_reappear = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + for (int i = 0; i < 8; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), failed_at_reappear) + << "stale/re-raised FAILED after healthy reappearance - baseline not forgotten on vanish"; + // The aggregate is currently clear: a fresh PASSED continues to flow. + const auto passed_reappear = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_reappear, *det, ctx)); +} + +// A SUCCESS response carrying an empty parameter array is a legitimate transport reply - the +// gateway transport documents it, for instance when a parameter declaration races the list. It is +// NOT evidence that nothing drifts: evaluate() would compare against no observations at all, find +// nothing, and the tick would clear a fault that is still true. No live node can be made to answer +// that way, which is why this needs the stand-in. +TEST_F(ParamDriftIntegrationTest, AnEmptyBatchDoesNotClearARaisedDrift) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + set_apps({{"pd_it_empty", "/pd_it_empty"}}); + for (int i = 0; i < 6; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for the rest of this test to mean anything"; + + // Drift it, and confirm the fault is genuinely up before testing what clears it. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(9.0); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // Now every read answers SUCCESS with nothing in it. The parameter is still 9.0. + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + fake->set_empty_batch(true); + for (int i = 0; i < 12; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "an empty batch was treated as a complete view and cleared a drift that is still present"; + + // Positive control: a real view showing the value restored does clear it. + fake->set_empty_batch(false); + fake->set_value(1.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)); +} + +// The reader releases the lock for the whole round trip, so a read can still be in flight when the +// tick thread drops its app. Publishing it anyway is not merely stale: prune_vanished has already +// forgotten the baseline and erased the cache, so the late sample lands in an empty cache and +// becomes the baseline the moment the app returns. The node is then compared against a value it +// held only while it was NOT a target - a drift no later read can heal, because the reference +// itself is wrong. This is the failure the still_a_target fence exists to prevent. +// +// Only reachable through the stand-in: a live read returns in milliseconds and nothing in a real +// graph holds one open across the three ticks the prune needs. +TEST_F(ParamDriftIntegrationTest, AReadLandingAfterItsAppWasDroppedIsNotPublished) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + // `keep` stays armed for the whole test. It is what makes the sequencing observable: the reader + // is a single thread, so a read of `keep` completing AFTER the held read of `fence` was released + // proves the fenced publish has already happened. With `fence` alone the loop would park on the + // cv and the test would have to race the very publish it is trying to observe. + const std::string kFence = "/pd_it_fence"; + const std::string kKeep = "/pd_it_keep"; + set_apps({{"pd_it_fence", kFence}, {"pd_it_keep", kKeep}}); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "both apps must be read and clean before the fence can be tested"; + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for the rest of this test to mean anything"; + + // Hold a read of `fence` open, and make it answer with a value the node never had while armed. + // value_for() is read after the gate, so the held call returns 9.0 when it is finally released. + fake->block_node(kFence); + ASSERT_TRUE(fake->wait_until_parked()) << "no read of the fenced app was ever held"; + fake->set_value(kFence, 9.0); + + // Drop `fence` from the graph for long enough to prune it: tick 1 takes it out of `targets`, + // tick 3 (kBaselineForgetGrace + 1) forgets its baseline and erases its cached read. + set_apps({{"pd_it_keep", kKeep}}); + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + + // Let the held read finish, then wait for a read of `keep` to prove the reader moved past it. + const int calls_before_release = fake->calls(); + fake->release_node(kFence); + ASSERT_TRUE(fake->wait_until_reading(calls_before_release + 1)) + << "the reader never got past the released read, so nothing was published either way"; + + // The app returns holding the value it always had. A fresh baseline is 1.0 and nothing drifts. + // Had the late 9.0 been published, it would be the baseline instead, and this read is the drift. + fake->set_value(kFence, 1.0); + set_apps({{"pd_it_fence", kFence}, {"pd_it_keep", kKeep}}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the returned app was never re-read"; + // ~2 s against a 250 ms charge per baseline visit (2 round trips x a 125 ms slot) over two + // targets: several fresh reads of `fence`. + for (int i = 0; i < 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "a read that completed after its app was dropped became the baseline on the app's return"; +} + +// A throw out of a read must not escape the reader thread. reader_loop runs on a std::thread with +// no outer handler, so an escape is std::terminate - the whole gateway process, not just the +// detector. rclcpp can throw from inside a blocking call when the context is invalidated mid-read, +// and the plugin's per-detector guard does not cover this thread. +// +// Deleting the reader's try/catch makes this test abort the test binary rather than fail it, which +// is exactly the point: today nothing in the suite ever makes a read throw. +TEST_F(ParamDriftIntegrationTest, AThrowingReadDoesNotKillTheReaderThread) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + const std::string kThrow = "/pd_it_throw"; + const std::string kKeep = "/pd_it_keep2"; + set_apps({{"pd_it_throw", kThrow}, {"pd_it_keep2", kKeep}}); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "both apps must be read and clean before the throw is introduced"; + ASSERT_NE(fake, nullptr); + + fake->set_throwing(kThrow); + ASSERT_TRUE(fake->wait_until_threw(2)) << "the reader never issued a read that throws"; + + // The surviving reader must still do its job: drift the OTHER app and require the raise. A + // reader that died silently would leave this polling until it times out. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(kKeep, 9.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "the reader stopped working after a read threw"; +} + +// The pre-shutdown callback is what stops the reader while the context is STILL valid. Without it +// the reader can be inside a blocking rcl call when rclcpp invalidates the context, and rcl aborts +// rather than throws - no try/catch reaches that. Nothing in the suite runs the callback, because +// shutting the global context down would take every remaining test with it. A context owned by +// this test drives the same code path in isolation. +TEST_F(ParamDriftIntegrationTest, ContextShutdownStopsTheReaderWhileTheDetectorIsStillAlive) { + auto own_context = std::make_shared(); + own_context->init(0, nullptr); + rclcpp::NodeOptions opts; + opts.context(own_context); + auto own_node = std::make_shared("pd_it_shutdown_gw", opts); + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + ctx.gateway_node = own_node.get(); // the callback is registered on THIS node's context + + set_apps({{"pd_it_shutdown", "/pd_it_shutdown"}}); + det->tick(ctx); + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_reading(1)) << "the reader never started"; + + // Returns only after the callback has waited for the reader to leave its loop, so the call + // count is stable from here on - unless nothing stopped the reader at all. + own_context->shutdown("test"); + + const int at_shutdown = fake->calls(); + std::this_thread::sleep_for(1s); // ~4 baseline visits at the default budget (2 round trips each) + EXPECT_EQ(fake->calls(), at_shutdown) + << "the reader was still issuing reads after the context shut down, so a blocking rcl call " + "could be in flight when the context is invalidated"; +} + +// The budget is spent in service ROUND TRIPS at the watched node - not in node visits, and not in +// transport calls either. One `get_parameter` is three round trips there (a name-scoped list, a +// get, and an unconditional describe), and an `expect` visit makes one per pinned name, so +// charging a slot per visit lets N pins put 3N times the promised load on one node, and charging a +// slot per call still lets it put three times that load, while the knob an operator lowered to +// protect the node reports being honoured either way. +// +// Four pins and a budget of ten round trips per 1000 ms tick period: one slot is 100 ms, so a visit +// costs 4 x 3 = 12 slots and owes 1200 ms. Visits land at ~0.0 s, ~1.2 s and ~2.4 s, and a fourth +// would need ~3.6 s, so a 3 s window holds exactly three visits: twelve calls. Charging per call +// owes 400 ms and fits eight visits (32 calls); charging per visit owes 100 ms and fits about +// thirty (120 calls). +TEST_F(ParamDriftIntegrationTest, TheBudgetIsSpentInRoundTripsNotCalls) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"baseline", false}, + {"tick_interval_ms", 1000}, + {"max_reads_per_tick", 10}, + {"expect", {{"a", 1.0}, {"b", 1.0}, {"c", 1.0}, {"d", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_budget", "/pd_it_budget"}}); + + const auto started = std::chrono::steady_clock::now(); + while (std::chrono::steady_clock::now() - started < 3s) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_NE(fake, nullptr); + const int calls = fake->calls(); + // Both bounds carry weight. Without the upper one, per-call and per-visit charging both pass. + // Without the lower one, so does a reader that paced itself into silence after a single visit - + // the test would then be proving that nothing happened, which any amount of over-pacing also + // satisfies. The gap between them is one visit wide on purpose: a fourth visit inside the window + // is the smallest observable under-charge, and it is excluded. + EXPECT_GE(calls, 12) << "issued " << calls + << " calls in ~3 s: the third visit never arrived, so the budget is being " + "spent slower than the knob promises"; + EXPECT_LE(calls, 15) << "issued " << calls + << " calls in ~3 s against a budget of 10 round trips per 1000 ms: a fourth " + "visit fit in the window, so a visit was charged less than the three " + "round trips each of its four pins really costs the node"; +} + +// The other half of charging what a read really costs: a pin the node does not declare is answered +// from the name-scoped list alone and never reaches the get or the describe, so it costs the node +// less than a pin that resolves. `expect` is documented as checked on every node that HAS the +// parameter, so on a mixed graph most nodes answer NOT_FOUND for most pins - charging those the +// full three round trips would throttle the sweep THREE TIMES as hard as the load it creates - the +// load being one round trip, not two: the charge of two is deliberately the higher of the two +// NOT_FOUND paths, while what the node really pays on the common one is a single name-scoped list. +// The coverage latency an operator computes from the knob would be wrong in the safe-looking +// direction. +// +// Four pins, none of them declared, a budget of eight round trips per 125 ms tick period: one slot +// is 15.625 ms, so a visit costs 4 x 2 = 8 slots and owes 125 ms. The ninth visit therefore starts +// at ~1.0 s. Charging NOT_FOUND the full three owes 187.5 ms and puts it at ~1.5 s; charging it one +// owes 62.5 ms and puts it at ~0.5 s. The band below excludes both. Timing the reads rather than +// counting them in a fixed window is what buys a quarter-second of margin on each side. +TEST_F(ParamDriftIntegrationTest, AnUndeclaredPinIsChargedLessThanOneThatResolves) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + // Declared-set restriction installed in the factory: the reader starts on the first tick and the + // very first visit must already be answered NOT_FOUND, or it is charged as a resolving read. + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_declared(std::set{}); // the node declares none of the pins + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"baseline", false}, + {"tick_interval_ms", 125}, + {"max_reads_per_tick", 8}, + {"expect", {{"a", 1.0}, {"b", 1.0}, {"c", 1.0}, {"d", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_notfound", "/pd_it_notfound"}}); + + // One tick arms the reader; it then round-robins its single target on its own, so the elapsed + // time to the 36th call is pure pacing and not the tick loop's cadence. + const auto started = std::chrono::steady_clock::now(); + det->tick(ctx); + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_reading(36)) << "the reader never issued nine visits, so it is paced " + "far slower than any charge this test can distinguish"; + const auto elapsed = std::chrono::steady_clock::now() - started; + const auto ms = std::chrono::duration_cast(elapsed).count(); + EXPECT_GT(ms, 750) << "nine visits took " << ms + << " ms: an undeclared pin was charged less than the round trip it does cost, " + "so the budget under-states the load the sweep puts on the node"; + EXPECT_LT(ms, 1250) << "nine visits took " << ms + << " ms: an undeclared pin was charged the full price of one that resolves, " + "so the sweep is throttled for a get and a describe it never issued"; +} + +// A clear says "nothing is drifting anywhere", and the only thing that can make that true is having +// LOOKED everywhere. Any horizon expressed in elapsed ticks eventually expires, and the rotation it +// is racing has no upper bound: with a 1 s tick, one read per tick and 120 apps, the derived give-up +// horizon ran out at tick 3612 while sixty apps had still never been contacted, so tick 3613 +// cleared GRAPH_PARAM_DRIFT on evidence that did not exist. Bigger graph, longer silence, MORE +// confidence - exactly backwards, and worst on the graphs where the sweep is slowest. +// +// 160 apps at one round trip per 200 ms tick period: a baseline visit owes two slots, i.e. 400 ms, +// so one rotation takes a minute. The tick loop below carries no sleep and runs far past the old +// horizon inside a fraction of that, so nearly the whole graph is uncontacted throughout. Nothing +// may be cleared, and nothing may be SAID about an app the sweep has not reached either - a warning +// that "no read has succeeded" for a node nobody has tried to read is the same defect in the log. +TEST_F(ParamDriftIntegrationTest, NoClearWhileTheSweepHasNotReachedEveryApp) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 200}, {"max_reads_per_tick", 1}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + for (int i = 0; i < 160; ++i) { + const std::string id = "pd_it_wide" + std::to_string(1000 + i); // uniform width -> sorted == creation order + apps.emplace_back(id, "/" + id); + } + set_apps(apps); + det->tick(ctx); // builds the transport and hands the reader all 160 targets + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_reading(1)) << "the reader never started, so nothing here is under test"; + + // No sleep: this is about tick COUNT, and the point is that no number of ticks can substitute for + // a read. 4000 is past the horizon the old derivation capped out at (3612) for any graph at all. + for (int i = 0; i < 4000; ++i) { + det->tick(ctx); + } + // The reader takes targets round-robin in order, and a baseline visit is exactly one call, so a + // call count below the app count means the tail of the graph was never contacted. This counts + // COVERAGE, not load - the stand-in cannot measure round trips and no bound here depends on it. + ASSERT_LT(fake->calls(), 160) << "the sweep got round all 160 apps inside the window, so " + "'uncontacted' was never the state under test"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), 0u) + << "cleared GRAPH_PARAM_DRIFT while most of the graph had never been contacted - that is " + "health the detector never measured, and it is what any horizon in ticks does once the " + "rotation outlasts it"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "nothing drifted, so nothing may be raised either"; + EXPECT_EQ(log.count("consecutive parameter reads of"), 0) + << "told the operator that reads of an app had failed when no read of it was ever attempted"; + EXPECT_EQ(log.count("giving up on"), 0) + << "gave up on an app the sweep had not reached, on the strength of elapsed ticks alone"; + + // Positive control: reduce the graph to an app that HAS been read and the clear flows at once. + // The zeroes above are the guard doing its job, not a detector that emits nothing. + set_apps({apps.front()}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "no clear even once every remaining app had been read, so the guard never releases"; +} + +// The other half of the same rule. Silence about an app is only ever reported once we have tried +// and failed - repeatedly - to read it, and the count that decides both warnings is a count of +// ATTEMPTS, so it cannot be reached by waiting. Both warnings fire once per run of failures, in +// order, and only about the app that will not answer. +// +// Two apps at eight round trips per 10 ms tick period: one slot is 1.25 ms and both a successful +// baseline visit and a failed one are charged two slots, so the ghost is retried roughly every +// 5 ms and the sixty-first failure lands inside a second. +TEST_F(ParamDriftIntegrationTest, TheUnreadWarningsRequireFailedAttemptsNotElapsedTicks) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_mute"); // before the reader can ever answer for it + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_heard", "/pd_it_heard"}, {"pd_it_mute", "/pd_it_mute"}}); + + for (int i = 0; i < 400 && log.count("giving up on") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_NE(fake, nullptr); + EXPECT_EQ(log.count("giving up on"), 1) << "the app that never answers was never given up on, so it " + "holds back every clear from here on"; + EXPECT_EQ(log.count("consecutive parameter reads of 'pd_it_mute'"), 1) + << "the app that never answers was never named, or was named more than once - the warning is " + "not latched"; + EXPECT_EQ(log.count("consecutive parameter reads of"), 1) + << "the healthy app was named alongside it, even though every read of it succeeded"; + + // Both are one-shot. Keep ticking well past the point they fired. + for (int i = 0; i < 100; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + EXPECT_EQ(log.count("consecutive parameter reads of"), 1) + << "the unread warning repeats every tick once its count is passed"; + EXPECT_EQ(log.count("giving up on"), 1) << "the give-up warning repeats every tick once its count is passed"; +} + +// A baseline visit is one list_parameters(), and that is TWO round trips at the node: the list, +// then the batched get of every name it returned. Charging it one slot - the old per-call rate - +// puts twice the promised load on every watched node in this detector's default mode. +// +// The other half of the rule is what is deliberately NOT charged. The first list_parameters() +// against a node costs two more, because the transport primes its defaults cache before its own +// read, and that cache has no TTL: the extra pair is a one-off per node for the life of the +// gateway. Charging four on every read would throttle the whole run to pay for the first rotation. +// +// One app, a budget of eight round trips per 125 ms tick period: one slot is 15.625 ms, so a visit +// costs 2 slots and owes 31.25 ms, putting the 33rd read at ~1.0 s. One slot per visit puts it at +// ~0.5 s and the cold-start four at ~2.0 s; the band excludes both. +TEST_F(ParamDriftIntegrationTest, ABaselineVisitIsChargedTwoRoundTripsAndNotTheColdStartPair) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 125}, {"max_reads_per_tick", 8}}); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_bl", "/pd_it_bl"}}); + + // One tick arms the reader; it round-robins its single target on its own from there, so the + // elapsed time is pure pacing rather than the tick loop's cadence. + const auto started = std::chrono::steady_clock::now(); + det->tick(ctx); + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_reading(33)) << "the reader never issued 33 baseline visits, so it is " + "paced far slower than any charge this test can distinguish"; + const auto elapsed = std::chrono::steady_clock::now() - started; + const auto ms = std::chrono::duration_cast(elapsed).count(); + EXPECT_GT(ms, 750) << "33 baseline visits took " << ms + << " ms: a visit was charged less than the list-plus-get it really costs the " + "node, so the budget under-states the load by half"; + EXPECT_LT(ms, 1250) << "33 baseline visits took " << ms + << " ms: a visit was charged the cold-contact price of four, which is paid " + "once per node for the life of the gateway, not on every read"; +} + +// The same condition that governs a clear governs the unmatched-`expect`-pin warning, and for the +// same reason: only one node is visited per rotation slot, so a pin that exactly one node declares +// is legitimately unseen until the sweep reaches that node. On any threshold counted in elapsed +// ticks the detector announces that "no app declares" it long before it has looked at the app that +// does - a warning that tells an operator to go and fix a name that is already correct. So the +// warning is withheld until every armed app has been read (or given up on) and at least one of +// them answered. +// +// Twenty-one apps, one pin, one round trip per 10 ms tick period. An undeclared pin is charged two +// round trips, so the twenty nodes that answer NOT_FOUND owe two slots each and the one node that +// declares the pin - it sorts last - is reached after forty slots, i.e. ~400 ms. The 140 ticks +// below span well over a second, so the sweep completes several times over with nothing reported, +// which is only true if the pin was actually MATCHED rather than merely not yet due. +TEST_F(ParamDriftIntegrationTest, APinOnlyTheLastNodeDeclaresIsNotReportedAsUnmatched) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + // Only the last-sorting node has the pin; the rest answer NOT_FOUND for it from the first read. + t->set_declared(std::set{}); + t->set_declared("/pd_it_pinlast", {"a_pin"}); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 10}, {"max_reads_per_tick", 1}, {"expect", {{"a_pin", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + for (int i = 0; i < 20; ++i) { + const std::string id = "pd_it_pin" + std::to_string(100 + i); // uniform width; all sort before "pinlast" + apps.emplace_back(id, "/" + id); + } + apps.emplace_back("pd_it_pinlast", "/pd_it_pinlast"); + set_apps(apps); + + for (int i = 0; i < 140; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + ASSERT_NE(fake, nullptr); + EXPECT_EQ(log.count("no app declares 'a_pin'"), 0) + << "told the operator no app declares a pin that the last node in the rotation declares " + "perfectly well - the warning fired before the sweep had read that node"; +} + +// A throw while building the shared IO must leave io_ null so the next tick retries. Assigning it +// first and filling it in afterwards latches a half-built state: the plugin's per-detector guard +// logs the throw once, every later tick sees a non-null io_ and skips the setup, and the detector +// then runs forever with no reader at all - indistinguishable from a healthy one at every call site. +TEST_F(ParamDriftIntegrationTest, AFailedLazyInitRetriesOnTheNextTick) { + ScriptedTransport * fake = nullptr; + int attempts = 0; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test( + [&fake, &attempts](rclcpp::Node *) -> std::unique_ptr { + if (++attempts == 1) { + throw std::runtime_error("transport unavailable"); + } + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json::object()); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_retry", "/pd_it_retry"}}); + + EXPECT_THROW(det->tick(ctx), std::runtime_error); // the plugin guard swallows this in production + for (int i = 0; i < 8; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(attempts, 2) << "the detector did not retry the transport after the first failure"; + ASSERT_NE(fake, nullptr) << "no transport was ever built, so the detector has no reader"; + EXPECT_GT(fake->calls(), 0) << "the reader never ran after the retry"; +} + +// The gateway must never read its own parameters: the transport spins a hidden node of its own, so +// reading the gateway would have it interrogate itself and can raise drift against the plugin. The +// skip is one line and nothing covered it. +TEST_F(ParamDriftIntegrationTest, TheGatewaysOwnNodeIsNeverRead) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json::object()); + auto ctx = make_ctx(DetectorMode::Raise); + + // Only the gateway itself: no read may happen. + set_apps({{"pd_it_gateway", gateway_->get_fully_qualified_name()}}); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_NE(fake, nullptr); + EXPECT_EQ(fake->calls(), 0) << "the detector read the gateway's own parameters"; + + // Positive control: zero above must mean "skipped the gateway", not "read nothing at all". Any + // mutation that empties the armed set would also give zero, so an ordinary app has to produce + // reads through the very same path. + set_apps({{"pd_it_other", "/pd_it_other"}}); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_GT(fake->calls(), 0) << "no app was ever read, so the zero above proves nothing"; +} + +// A config warning that never reaches the log is no warning at all. Six unit tests verify that the +// warnings vector gets populated and none of them proves it is ever surfaced, so deleting the +// logging call changed nothing. Here the detector is handed a misspelt key and the tick must reach +// the logger - asserted through the rcutils log handler rather than by reading stderr. +TEST_F(ParamDriftIntegrationTest, ConfigWarningsReachTheLogger) { + const LogCapture log; + + auto det = make_param_drift(); + ScriptedTransport * fake = nullptr; + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + const nlohmann::json typo{{"ignore_globs", nlohmann::json::array({"*_stamp"})}}; // misspelt on purpose + det->configure(typo); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_warn", "/pd_it_warn"}}); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + ASSERT_NE(fake, nullptr); + + EXPECT_EQ(log.count("ignore_globs"), 1) + << "expected the typo logged exactly once across ten ticks; got " << log.count("ignore_globs") + << " (0 means it never reached the log, more than 1 means the once-guard is gone and every " + "tick repeats it forever)"; + + // The once-guard has to be RE-ARMED by configure(). The plugin re-delivers configuration to a + // live detector, and without the reset an operator who fixes one key and mis-spells another is + // told nothing at all about the second one - the warning is latched for the rest of the run. + det->configure(typo); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + EXPECT_EQ(log.count("ignore_globs"), 2) + << "the same typo delivered by a second configure() was never logged again, so the " + "once-per-run latch is never released and re-configuration is silent"; +} + +// A pin naming a parameter no node declares is answered NOT_FOUND on every read, and NOT_FOUND is +// deliberately not a drift - a node is not at fault for a parameter it never had. The cost is that +// a misspelt pin is inert forever, and from outside inert is indistinguishable from a pin that is +// checked and passes every time. That is the silent failure this detector exists to rule out, so +// it has to say something. Exactly once, and only about the pin that is actually unmatched. +TEST_F(ParamDriftIntegrationTest, AnExpectPinNoAppDeclaresIsReportedOnce) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // Two pins: one the node has, one misspelt. The good pin is the control - it must stay silent, + // or the warning is just firing on every pin and proves nothing about the typo. + det->configure(nlohmann::json{{"baseline", false}, {"expect", {{"gain", 1.0}, {"gian", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_typo", "/pd_it_typo"}}); + det->tick(ctx); + ASSERT_NE(fake, nullptr); + fake->set_declared({"gain"}); + + for (int i = 0; i < 80 && log.count("no app declares 'gian'") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(log.count("no app declares 'gian'"), 1) << "the misspelt pin was never reported"; + + // Keep ticking: the report is once per pin, not once per tick. + for (int i = 0; i < 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(log.count("no app declares 'gian'"), 1) << "the once-guard is gone and every tick repeats the warning"; + EXPECT_EQ(log.count("no app declares 'gain'"), 0) << "the pin the node actually declares was reported as unmatched"; +} + +// The other side of the same rule, and the one that costs a real alarm if it is wrong. DDS +// discovery drops a node from a single poll routinely, with no restart involved - WarmupTracker +// documents exactly that and absorbs such a gap. If a one-sweep gap re-baselined, the node would +// come back and its CURRENT, still drifted value would become the new reference: the confirmed +// fault heals while the parameter is wrong, and can never fire again because the baseline now IS +// the wrong value. Nothing in the log would say so. +TEST_F(ParamDriftIntegrationTest, ChurnMustNotLaunderARaisedDrift) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + const std::vector> present{{"pd_it_churn", "/pd_it_churn"}}; + set_apps(present); + for (int i = 0; i < 6; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_NE(fake, nullptr); + + // A genuine drift, confirmed up before we disturb anything. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(9.0); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // One dropped poll. The node never restarted and its parameter is still 9.0. + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + set_apps({}); + det->tick(ctx); + std::this_thread::sleep_for(50ms); + set_apps(present); + for (int i = 0; i < 14; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "a single dropped discovery poll re-baselined the node on its drifted value, so the fault " + "healed while the parameter is still wrong and can never fire again"; + + // Positive control: fixing it for real still clears, so the zero above is not a dead detector. + fake->set_value(1.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)); +} + +// The withheld clear must not become a permanent latch. Its purpose is the case where nothing is +// known yet: the round-robin sweep has not reached this app. A node that never answers AT ALL - no +// parameter service, a wedged executor - is a permanent property, and there is no per-node opt-out +// here, so without a bound one such node keeps a fixed drift reported as CONFIRMED for the whole +// lifetime of the gateway. Past the bound the app is declared unmeasurable, logged, and stops +// blocking. +// +// The bound is in FAILED ATTEMPTS, not in elapsed ticks: this app has been read sixty-one times and +// answered none of them, which is evidence, where "sixty ticks went by" is not - the sweep may +// simply not have got here. That is the whole distinction, and it is why the escape hatch survives +// the rule that an app the sweep has never attempted blocks the clear indefinitely. The budget is +// set so the sixty-first attempt lands inside a second rather than at the default pacing. +TEST_F(ParamDriftIntegrationTest, APermanentlyUnreadableAppStopsBlockingTheClear) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + // The ghost is marked unreadable inside the factory, before the transport is ever handed to the + // reader. Marking it after the first tick is a race the reader can win, and if it does the ghost + // has a cached read for the rest of the run, never joins the unread set, and never blocks + // anything - the test would pass while proving nothing. + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_ghost2"); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 20}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + // The real app is served by the stand-in; the ghost has no node at all, so no read for it can + // ever succeed. Both are armed every tick. + set_apps({{"pd_it_real", "/pd_it_real"}, {"pd_it_ghost2", "/pd_it_ghost2"}}); + det->tick(ctx); // builds the transport and hands the reader both targets + ASSERT_NE(fake, nullptr); + // Wait for EVIDENCE that the real app's baseline exists rather than sleeping a fixed time. Only + // /pd_it_real reaches the stand-in's call counter (the ghost fails before it), so a second + // counted read means the first one has already been published to the cache, and the tick after + // it turns that into the baseline. A fixed window here is a race against the sweep's own pacing: + // the ghost's failed visit is charged too, so how long the real app waits for its turn is a + // function of the budget, and losing that race makes the drifted value the baseline instead - + // after which no drift exists at all and the rest of the test is unreachable. + ASSERT_TRUE(fake->wait_until_reading(2)) << "the reader never completed a read of /pd_it_real"; + det->tick(ctx); + + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(9.0); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "the drift never raised, so there is nothing whose clearing this test can observe"; + + // Fix it. The ghost is still silent and always will be. + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + fake->set_value(1.0); + bool cleared = false; + for (int i = 0; i < 120 && !cleared; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + cleared = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED) > passed_before; + } + EXPECT_TRUE(cleared) << "one permanently unreadable app blocked the clear forever, so a drift that " + "was fixed stays CONFIRMED for the lifetime of the gateway"; +} + +// Restarting a node is the documented way to accept a new configuration, so a node that was gone +// long enough to have restarted must come back to a fresh baseline rather than being compared +// against values from before the change. +// +// "Long enough" is deliberately the window WarmupTracker absorbs, not a single sweep: that header +// documents that DDS discovery drops a node from one poll routinely, with no restart involved, and +// re-capturing on such a gap would take the node's current - possibly already drifted - values as +// the new reference, healing a real fault that can then never fire again. +// +// Here the app is absent for three sweeps and returns with a different value. Nothing may be +// reported: that value is the new reference. +TEST_F(ParamDriftIntegrationTest, ARestartLengthAbsenceReBaselinesInsteadOfReportingDrift) { + set_apps({{"pd_it_target", "/pd_it_target"}}); + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_target")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear arrived, so the app was never read and there is no pre-restart baseline whose " + "being forgotten this test could observe"; + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for this test to mean anything"; + + // Absent for longer than the churn window, i.e. long enough to be a restart. + set_apps({}); + for (int i = 0; i < 4; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + + // It comes back carrying the operator's new value. + target_->set_parameter(rclcpp::Parameter("speed_limit", 5.0)); + set_apps({{"pd_it_target", "/pd_it_target"}}); + const auto failed_before_return = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "the returning app was never re-read, so the silence below is the silence of a detector " + "that measured nothing"; + for (int i = 0; i < 12; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), failed_before_return) + << "reported drift against the baseline from before the restart, so the README's recovery " + "path (restart the node to re-baseline) does not work"; + + // POSITIVE CONTROL. Every assertion above this line is an all-zeros one, and a detector that + // reads nothing satisfies them exactly as well as one whose forget() works. Moving off the value + // the node came back with must report - which is only possible if that value really was captured + // as the new reference. + const auto failed_before_drift = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + target_->set_parameter(rclcpp::Parameter("speed_limit", 0.25)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before_drift, *det, ctx)) + << "no drift off the re-captured baseline, so the silence above proves nothing at all"; +} + +// A clear says "nothing is drifting anywhere". The detector may only say that about apps it +// actually read. Here one app is real and healthy while a second app is in the snapshot with an +// FQN that answers nothing, so no read for it can ever succeed. Emitting PASSED in that state +// would report health that was never measured - and because the README tells operators to enable +// healing, those PASSED events are what would walk a genuine, still-present drift to HEALED. +// +// The healthy app is what makes this a real discriminator: without the guard, its clean reads +// leave drifted_ empty and the detector cheerfully clears every single tick. +TEST_F(ParamDriftIntegrationTest, NoClearWhileAnArmedAppWasNeverRead) { + set_apps({{"pd_it_target", "/pd_it_target"}, {"pd_it_ghost", "/pd_it_ghost"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_target")); // the real one is readable + for (int i = 0; i < 12; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), 0u) + << "cleared GRAPH_PARAM_DRIFT while /pd_it_ghost had never been read - that is health " + "the detector never measured"; + // And nothing was invented either: no drift exists, so no raise should have happened. + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); +} + +// An app that has been given up on must STAY given up on. The blocking set is recomputed from +// scratch every tick while the give-up is one-shot, so any bound that moves with the shape of the +// graph silently re-admits an app already logged as "no longer holding back the clear": the graph +// grows, the bound grows past the app's count, and every clear from then on is suppressed with +// nothing in the log to say why. A count of that app's OWN failed attempts cannot do that - it +// depends on nothing but the app, and it only ever grows while the app stays unread. +// +// The armed set is grown and then shrunk around a given-up ghost, and a clear is required to arrive +// on both sides. Forty extra apps at eight round trips per 10 ms tick period is a 105 ms rotation, +// so the grown graph is fully read inside a fraction of the polls below - while a bound scaled by +// the armed set would need 60 x 42 failures, which at one ghost retry per rotation is minutes away. +TEST_F(ParamDriftIntegrationTest, AGivenUpAppDoesNotReBlockWhenTheArmedSetGrowsOrShrinks) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_grow_ghost"); // before the reader can ever answer for it + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + const std::vector> small{{"pd_it_grow_real", "/pd_it_grow_real"}, + {"pd_it_grow_ghost", "/pd_it_grow_ghost"}}; + set_apps(small); + // The first clear IS the give-up: nothing drifts, so the only thing that can be holding the clear + // back is the ghost, and the only thing that can release it is having failed on it often enough. + bool gave_up = false; + for (int i = 0; i < 400 && !gave_up; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + gave_up = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED) > 0; + } + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(gave_up) << "the ghost never stopped blocking, so there is no give-up to re-block"; + + // GROW. Forty apps arrive at once; each of them legitimately blocks the clear until it is read, + // and the ghost must not join them again on the way past. + std::vector> grown = small; + for (int i = 0; i < 40; ++i) { + const std::string id = "pd_it_grow" + std::to_string(100 + i); + grown.emplace_back(id, "/" + id); + } + set_apps(grown); + auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + bool cleared_grown = false; + for (int i = 0; i < 150 && !cleared_grown; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + cleared_grown = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED) > passed_before; + } + EXPECT_TRUE(cleared_grown) << "no clear once the grown graph had been read: an app already given up " + "on was re-admitted to the blocking set because the graph got bigger, " + "and nothing in the log says so"; + + // SHRINK back. Every remaining app is either read or given up on, so the very next ticks clear. + set_apps(small); + passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + bool cleared_small = false; + for (int i = 0; i < 40 && !cleared_small; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + cleared_small = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED) > passed_before; + } + EXPECT_TRUE(cleared_small) << "the ghost started blocking again when the graph shrank back"; +} + +// The stale-read fence has to be keyed on the ARMING EPISODE, not on the app. A read runs with the +// lock released and can take seconds; inside that window the app's lifecycle gate can close and +// re-open, and tick() rewrites the target list every tick, so an identity check on the app id - +// or on the fqn, which is just as stable - lets a value sampled while the node was between +// episodes land in the cache. That value then becomes what the node is measured against: a drift +// that is not real and that no later read can heal, which is the exact failure the fence exists +// for. A token issued once per episode and never reused is what tells the two apart. +// +// Only reachable through the stand-in: a live read returns in milliseconds and nothing in a real +// graph holds one open across a de-arm and a re-arm. +TEST_F(ParamDriftIntegrationTest, AReadFromAnInterruptedArmingEpisodeIsNotPublished) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // A 500 ms tick period against eight round trips is a 62.5 ms slot, so a baseline visit owes + // 125 ms: after the held read is released there is a comfortable margin before the reader could + // come back round to the fenced app. + det->configure(nlohmann::json{{"tick_interval_ms", 500}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + const std::string kEpoch = "/pd_it_epoch"; + const std::string kKeep = "/pd_it_epochkeep"; + set_apps({{"pd_it_epoch", kEpoch}, {"pd_it_epochkeep", kKeep}}); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "both apps must be read and clean before the fence can be tested"; + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for the rest of this test to mean anything"; + + // Hold a read of the fenced app open and arrange for it to answer 9.0 - value_for() is sampled + // after the gate, so that is what the held call returns when it is finally released. Marking the + // app unreadable at the same time only affects LATER reads (the held one is already past that + // check), so nothing can overwrite whatever this one publishes, if it publishes. + fake->block_node(kEpoch); + ASSERT_TRUE(fake->wait_until_parked()) << "no read of the fenced app was ever held"; + fake->set_value(kEpoch, 9.0); + fake->set_unreadable(kEpoch); + + // De-arm and re-arm inside that one read window. One tick out and one tick back in: far short of + // the prune, so the baseline (1.0) is untouched, and the app id and the fqn are both unchanged - + // an identity check on either sees nothing at all happen here. + set_apps({{"pd_it_epochkeep", kKeep}}); + det->tick(ctx); + set_apps({{"pd_it_epoch", kEpoch}, {"pd_it_epochkeep", kKeep}}); + det->tick(ctx); + + const int calls_before_release = fake->calls(); + fake->release_node(kEpoch); + // A read of the OTHER app is the next counted one - an unreadable node fails before the counter - + // so this means the released read has finished and either published or been fenced out. + ASSERT_TRUE(fake->wait_until_reading(calls_before_release + 1)) + << "the reader never got past the released read, so nothing was published either way"; + + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "a value sampled while the app was between arming episodes was published as its reading, " + "so the node is now measured against something it held only while it was not a target"; + + // Positive control: the raise path is alive, so the zero above is the fence and not a dead + // detector. The other app has been armed continuously throughout and drifting it must report. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value(kKeep, 9.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "no drift was reported for an app that plainly drifted, so nothing above proves anything"; +} + +// Forgetting a vanished node's baseline must be reachable at EVERY accepted prune_grace. The two +// horizons are independent - the baseline goes after a fixed three missed sweeps, the finding after +// prune_grace of them - and dropping the absence counter is what makes any later cleanup +// unreachable. With prune_grace 0 or 1, both inside the documented range, the counter was dropped +// before it could ever reach the forget, so the forget never ran for any app in any run: the +// baselines map grew without bound under node-name churn and every returning node was measured +// against values from before it left. +// +// Each case runs the same story - baseline, vanish, return holding a DIFFERENT value - and the +// return must be silent, because a forgotten baseline makes that value the new reference. The +// endpoints of the range (0 and 3600), the value below the churn window (1), the detector's own +// fallback (2), what the plugin injects when an operator sets nothing (60), and two values outside +// the range, which must be rejected, must warn, and must leave the fallback behaviour intact. +TEST_F(ParamDriftIntegrationTest, BaselineForgettingIsReachableAtEveryAcceptedPruneGrace) { + const LogCapture log; + + const std::vector> cases{{"0", 0}, {"1", 1}, {"2", 2}, {"60", 60}, + {"3600", 3600}, {"4000", 4000}, {"-1", -1}}; + const int out_of_range = 2; // the last two cases; both must warn and both must keep the fallback + + int seq = 0; + for (const auto & [label, value] : cases) { + SCOPED_TRACE("prune_grace=" + label); + ++seq; + const std::string id = "pd_it_pg" + std::to_string(seq); + const std::string fqn = "/" + id; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}, {"prune_grace", value}}); + auto ctx = make_ctx(DetectorMode::Raise); + + // Baseline captured at the stand-in's default of 1.0. The clear proves the app was read. + set_apps({{id, fqn}}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the app was never read, so it has no baseline to forget"; + ASSERT_NE(fake, nullptr); + + // Absent for longer than either horizon can need: the churn window absorbs two missed sweeps + // and the forget runs on the third, while the shortest configurable grace drops the counter on + // the first. + set_apps({}); + for (int i = 0; i < 6; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + + // It returns holding a value it never held before. With the baseline forgotten that value IS + // the new reference; with it retained, 1.0 against 5.0 is a drift the node can never shake off. + fake->set_value(fqn, 5.0); + set_apps({{id, fqn}}); + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + for (int i = 0; i < 40; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), failed_before) + << "the returning node was reported as drifted against a baseline from before it left, so " + "the forget never ran at this prune_grace"; + + // Positive control: the baseline it came back with is a real one, so moving off it still + // reports. Without this the case above is satisfied by a detector that captured nothing. + fake->set_value(fqn, 9.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "no drift off the re-captured baseline, so the silence above proves nothing"; + } + + EXPECT_EQ(log.count("'prune_grace' must be an integer"), out_of_range) + << "an out-of-range prune_grace was accepted or rejected in silence; an operator has no other " + "way to find out the value they wrote is not the one in force"; +} + +// Every per-app map in this detector is keyed by node name, and node names are not a bounded set: +// discovery has no hidden-node filter, so each `ros2` CLI invocation appears once under a name +// nothing will ever use again. A map no prune path reaches therefore grows for the life of the +// gateway process, and a leak shows up in no fault, no log line and no description - which is why +// this asserts on a count rather than on a behaviour. +// +// The churn here is the real shape: a short-lived node that is present for a single sweep, is gate- +// denied for all of it (it has not been present long enough to arm), and then is gone. That path +// creates the frozen-hold counter and nothing on it ever created anything else, so the frozen-hold +// counter was the one map prune_vanished did not clean. +TEST_F(ParamDriftIntegrationTest, PerAppBookkeepingStaysBoundedUnderNodeNameChurn) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}, {"prune_grace", 2}}); + auto ctx = make_ctx(DetectorMode::Raise); + + IntrospectionInput snap; + ReliabilityGate gate(3, gateway_.get(), &node_mutex_); + ctx.gate = &gate; + ctx.snapshot = &snap; + + const auto seed = [&snap](const std::vector & ids) { + snap.apps.clear(); + for (const auto & id : ids) { + App a; + a.id = id; + a.bound_fqn = "/" + id; + snap.apps.push_back(a); + } + }; + + uint64_t tick_no = 0; + const std::string stable = "pd_it_churn_stable"; + for (int i = 0; i < 200; ++i) { + // One long-lived app plus one that exists for exactly this sweep, under a name never reused. + seed({stable, "ros2_cli_" + std::to_string(i)}); + gate.update(snap, ++tick_no); + det->tick(ctx); + std::this_thread::sleep_for(2ms); + } + ASSERT_NE(fake, nullptr); + + // Let the last few transients age past the prune horizon. + seed({stable}); + for (int i = 0; i < 10; ++i) { + gate.update(snap, ++tick_no); + det->tick(ctx); + std::this_thread::sleep_for(2ms); + } + + const std::size_t tracked = det->tracked_count_for_test(); + EXPECT_GE(tracked, 1u) << "the detector is tracking nothing at all, so the bound below is vacuous"; + EXPECT_LE(tracked, 4u) << "still holding per-app bookkeeping for " << tracked + << " identities after 200 one-sweep nodes came and went: a map keyed by node " + "name is not being pruned, and it will grow for the life of the process"; +} + +// A single app's share of the aggregated description is capped, and that cap is what makes going +// second worth anything at all. Without it, one node drifting on a dozen parameters spends the +// whole 480-character budget by itself and every other affected app is cut - whatever the ordering +// says, because there is no room left to order anything into. +// +// Three apps, each drifting on twelve long-valued parameters. Uncapped, the first alone renders to +// roughly 1900 characters and the other two never appear. Capped at 150 each, all three fit inside +// the budget with room to spare, so naming all three is exactly the per-app cap doing its job. +// +// The stand-in is what makes the sizing exact: a live node's parameter set is not fully under the +// test's control, and this assertion is about character counts. +TEST_F(ParamDriftIntegrationTest, OneAppsShareOfTheDescriptionIsCapped) { + const std::string clean_value(60, 'a'); + const std::string drifted_value(60, 'b'); + const std::vector ids{"pd_it_cap1", "pd_it_cap2", "pd_it_cap3"}; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake, &ids, &clean_value](rclcpp::Node *) { + auto t = std::make_unique(); + // Installed in the FACTORY: a drifted first read would simply become the baseline. + for (const auto & id : ids) { + std::map clean; + for (int p = 0; p < 12; ++p) { + clean.emplace("p" + std::to_string(p), clean_value); + } + t->set_params("/" + id, clean); + } + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 100}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + apps.reserve(ids.size()); + for (const auto & id : ids) { + apps.emplace_back(id, "/" + id); + } + set_apps(apps); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) << "not every app was read, so nothing has a baseline yet"; + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for the cap assertion below to mean anything"; + + for (const auto & id : ids) { + std::map drifted; + for (int p = 0; p < 12; ++p) { + drifted.emplace("p" + std::to_string(p), drifted_value); + } + fake->set_params("/" + id, drifted); + } + + ASSERT_TRUE(poll_until( + [this, &ids]() { + return latest_failed_desc_contains("graph_watchdog", ids); + }, + *det, ctx)) + << "the description names " << latest_failed_desc("graph_watchdog").size() + << " characters' worth of one or two apps and not all three: a single app is spending more " + "than its share of the budget and the others are being cut - " + << latest_failed_desc("graph_watchdog"); + + // Belt and braces on the sizing: the whole point is that three capped apps FIT, so the aggregate + // must be under its own cap here. If it were at the cap the three names above could be a + // coincidence of truncation rather than the per-app budget working. + EXPECT_LE(latest_failed_desc("graph_watchdog").size(), AggregatedFault::kMaxDescriptionChars); +} + +// The description is capped, so ORDER decides what an operator actually reads. Four apps that sort +// early and drift on many parameters fill the cap between them; a node that starts drifting +// afterwards sorts last and would never appear, even though it is the one thing that just changed. +// +// Four noisy apps at the per-app cap of 150 characters overrun the 480-character budget, which is +// what makes this a real trade rather than a hypothetical one - and the assertion below refuses to +// run unless the description really is at its cap. `pd_it_zzz` sorts after all of them, so in plain +// map order it is precisely what falls off the end; ordering newly affected apps first puts it at +// the FRONT of the report in which it first appears, which is where it survives. +TEST_F(ParamDriftIntegrationTest, ANewlyDriftedAppSurvivesTheDescriptionCap) { + const std::string clean_value(60, 'a'); + const std::string drifted_value(60, 'b'); + const std::string kLate = "pd_it_zzz"; // sorts after every pd_it_ordN + std::vector noisy_ids; + for (int n = 1; n <= 4; ++n) { + noisy_ids.push_back("pd_it_ord" + std::to_string(n)); + } + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake, &noisy_ids, &kLate, &clean_value](rclcpp::Node *) { + auto t = std::make_unique(); + for (const auto & id : noisy_ids) { + std::map clean; + for (int p = 0; p < 12; ++p) { + clean.emplace("p" + std::to_string(p), clean_value); + } + t->set_params("/" + id, clean); + } + t->set_params("/" + kLate, {{"gain", 1.0}}); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 100}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + apps.reserve(noisy_ids.size() + 1); + for (const auto & id : noisy_ids) { + apps.emplace_back(id, "/" + id); + } + apps.emplace_back(kLate, "/" + kLate); + set_apps(apps); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) << "not every app was read, so nothing has a baseline yet"; + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent, otherwise the ordering assertions below prove nothing"; + + // The four early-sorting apps drift first and fill the description between them. + for (const auto & id : noisy_ids) { + std::map drifted; + for (int p = 0; p < 12; ++p) { + drifted.emplace("p" + std::to_string(p), drifted_value); + } + fake->set_params("/" + id, drifted); + } + ASSERT_TRUE(poll_until( + [this]() { + return latest_failed_desc("graph_watchdog").size() >= AggregatedFault::kMaxDescriptionChars; + }, + *det, ctx)) + << "the description never filled up, so nothing was competing for room and the ordering could " + "not have mattered either way: " + << latest_failed_desc("graph_watchdog"); + ASSERT_EQ(latest_failed_desc("graph_watchdog").find(kLate), std::string::npos) + << "the app that has not drifted yet is already in the description, so its appearance below " + "would say nothing: " + << latest_failed_desc("graph_watchdog"); + + // Now the late app drifts. The report in which it first appears must lead with it: that is the + // whole of the newly-affected-first rule, and it is the only thing that can keep it in a + // description four other apps have already overrun. Every poll below reads the CURRENT record. + fake->set_params("/" + kLate, {{"gain", 9.0}}); + EXPECT_TRUE(poll_until( + [this, &kLate]() { + return latest_failed_desc("graph_watchdog").rfind(kLate + ":", 0) == 0; + }, + *det, ctx)) + << "the app that just started drifting was cut from the description to make room for apps " + "that merely sort earlier and had already been reported: " + << latest_failed_desc("graph_watchdog"); +} + +// The other half of the ordering, and the one that decides what an operator reads when the +// description is full: a violation of a pin the operator WROTE outranks drift the detector +// captured by itself. The two are not equally interesting. A self-captured entry says a value moved +// and the detector has no idea whether that matters; an `expect` entry says a value an operator +// declared is wrong. Cutting the declared one to make room for four apps' worth of the other is the +// wrong trade, and the cap makes it a real trade rather than a hypothetical one. +// +// Nothing exercised this. Every `expect` configuration in this file also sets `baseline: false`, so +// a pinned violation and self-captured drift never coexisted in a single run and the pinned buckets +// of emit_order() were never reached with anything in them. With `baseline: true` the pins are +// checked against the same list_parameters() the capture uses, so one read produces both kinds - +// which is what makes the combination reachable at all. +// +// The setup is sized so the ordering is the only thing that can decide the outcome. Four noisy apps +// drift on six parameters each, which overruns the detector's per-app share of the description, so +// each contributes exactly that share - and four of them are needed to overflow the cap, not three: +// three capped entries and their separators are 150 + 2 + 150 + 2 + 150 = 454 characters against a +// budget of 480, and it is the fourth that pushes it over. The pinned app sorts LAST of the five, +// so in plain map order it is precisely what falls off the end. And the +// assertion is taken only after every app has been reported at least once, so the separate "newly +// affected first" rule (ANewlyDriftedAppSurvivesTheDescriptionCap) cannot be what keeps it in. +TEST_F(ParamDriftIntegrationTest, AnOperatorPinnedViolationOutranksSelfCapturedDrift) { + const std::string kPin = "pinned_gain"; + const std::string kPinnedApp = "pd_ord_zzz"; // sorts after every pd_ord_noisyN + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake, &kPin](rclcpp::Node *) { + auto t = std::make_unique(); + // Installed in the FACTORY so the very first read of every noisy node is the clean one. A + // drifted first read would simply become that node's baseline and nothing would ever be + // reported, leaving the pinned app alone in a description with room to spare. + for (int n = 1; n <= 4; ++n) { + std::map clean; + for (int p = 0; p < 6; ++p) { + clean.emplace("p" + std::to_string(p), 1.0); + } + t->set_params("/pd_ord_noisy" + std::to_string(n), clean); + } + // The pinned app violates its pin from the first read and can carry nothing else: a pinned + // parameter is excluded from the capture set, so its whole contribution is the one entry an + // operator declared. + t->set_params("/pd_ord_zzz", {{kPin, 9.0}}); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // baseline stays true - that is the whole point - and the budget only keeps the sweep brisk. + det->configure(nlohmann::json{{"tick_interval_ms", 100}, {"max_reads_per_tick", 8}, {"expect", {{kPin, 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + for (int n = 1; n <= 4; ++n) { + const std::string id = "pd_ord_noisy" + std::to_string(n); + apps.emplace_back(id, "/" + id); + } + apps.emplace_back(kPinnedApp, "/" + kPinnedApp); + set_apps(apps); + + det->tick(ctx); // builds the transport and hands the reader all five targets + ASSERT_NE(fake, nullptr); + // The sixth read STARTING means the first five have completed and been published - the reader is + // a single thread round-robinning five targets - so the tick after it captures a clean baseline + // for every noisy app. + ASSERT_TRUE(fake->wait_until_reading(6)) << "the reader never got round all five apps"; + det->tick(ctx); + + // Now every noisy app drifts on all six of its parameters. + for (int n = 1; n <= 4; ++n) { + std::map drifted; + for (int p = 0; p < 6; ++p) { + drifted.emplace("p" + std::to_string(p), 9.0); + } + fake->set_params("/pd_ord_noisy" + std::to_string(n), drifted); + } + ASSERT_TRUE(poll_until( + [this]() { + return latest_failed_desc("graph_watchdog").size() >= AggregatedFault::kMaxDescriptionChars; + }, + *det, ctx)) + << "the description never filled up, so nothing was competing for room and the ordering " + "could not have mattered either way"; + + // Keep ticking well past that point: several more full rotations, so every app has been reported + // at least once and none of them is "newly affected" any more. + for (int i = 0; i < 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + + const std::string latest = latest_failed_desc("graph_watchdog"); + ASSERT_GE(latest.size(), AggregatedFault::kMaxDescriptionChars) + << "the description is no longer at its cap, so the assertions below prove nothing: " << latest; + ASSERT_NE(latest.find("pd_ord_noisy1"), std::string::npos) + << "no self-captured drift is in the description at all, so the pinned entry had the cap to " + "itself and its presence says nothing about priority: " + << latest; + EXPECT_NE(latest.find(kPinnedApp + ":\"" + kPin + "\" expected="), std::string::npos) + << "the violation of a pin an operator WROTE was cut from the description an operator reads, " + "to make room for self-captured drift on apps that merely sort earlier: " + << latest; + EXPECT_EQ(latest.rfind(kPinnedApp + ":", 0), std::size_t{0}) + << "the pinned violation is in the description but not at the front of it, so it survived on " + "where its app happens to sort rather than on being pinned: " + << latest; +} + +// Aggregation: two nodes drift -> ONE GRAPH_PARAM_DRIFT (source "graph_watchdog") whose +// description names BOTH apps; revert one -> the aggregate stays raised (the other still +// drifts, no clear); revert both -> the aggregate clears. +TEST_F(ParamDriftIntegrationTest, MultiNodeAggregationOneFaultNamesBothThenClears) { + auto a1 = std::make_shared("pd_it_agg1"); + a1->declare_parameter("alpha", 1.0); + auto a2 = std::make_shared("pd_it_agg2"); + a2->declare_parameter("beta", 2.0); + exec_.add_node(a1); + exec_.add_node(a2); + const auto nodes_guard = on_scope_exit([this, a1, a2] { + exec_.remove_node(a1); + exec_.remove_node(a2); + }); + set_apps({{"pd_it_agg1", "/pd_it_agg1"}, {"pd_it_agg2", "/pd_it_agg2"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json::object()); // baseline mode + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_agg1")); + ASSERT_TRUE(wait_param_service("/pd_it_agg2")); + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) + << "no clear arrived, so at least one of the two apps was never read and has no baseline"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u); + + // Drift both -> a single aggregated FAILED naming both apps. Asserted on the CURRENT record, not + // on the history: what an operator reads is the description the store last overwrote. + a1->set_parameter(rclcpp::Parameter("alpha", 0.1)); + a2->set_parameter(rclcpp::Parameter("beta", 0.2)); + EXPECT_TRUE(poll_until( + [this]() { + return latest_failed_desc_contains("graph_watchdog", {"pd_it_agg1", "pd_it_agg2"}); + }, + *det, ctx)) + << "one aggregated fault naming both apps never appeared: " << latest_failed_desc("graph_watchdog"); + + // Revert ONE: the aggregate must STAY raised, because agg2 still drifts. + // + // The snapshot for that is taken only after the revert has SETTLED - long enough for agg1's + // clean re-read to have landed and dropped it from the finding set. Counting raises from before + // that point is the hole in "some raise exists": every raise emitted while agg1 was still in the + // set satisfies it, so a detector that goes quiet the moment the set shrinks passes. + a1->set_parameter(rclcpp::Parameter("alpha", 1.0)); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + const auto failed_settled = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_settled, *det, ctx)) + << "no raise arrived once the reverted node had dropped out of the finding set, so the " + "aggregate went quiet while pd_it_agg2 is still drifting: " + << latest_failed_desc("graph_watchdog"); + EXPECT_TRUE(latest_failed_desc_contains("graph_watchdog", {"pd_it_agg2"})) + << "the current description no longer names the app that is still drifting: " + << latest_failed_desc("graph_watchdog"); + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "reverting one node cleared the aggregate while another still drifts"; + + // Revert BOTH -> the aggregate clears. + const auto passed_before2 = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + a2->set_parameter(rclcpp::Parameter("beta", 2.0)); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before2, *det, ctx)); +} + +// Guards the atomic expect-read fix. Config: baseline:false, 3 expect pins, max_reads_per_tick:6, +// two expect nodes each costing 3 pins x 3 round trips = 9. One node violates its LAST (sorted) pin +// z_bad from the start. Because expect is a std::map (keys sorted a_ok, b_ok, z_bad), a budget that +// truncated a visit at 6 would stop after b_ok, drop the drift from `observed`, and spuriously +// CLEAR a real, unfixed fault. The atomic read fetches the whole expect set for a visited node +// regardless of budget, so the drift stays raised and never flaps. Assert the aggregated FAILED +// stays raised with NO interleaved PASSED clear. +// +// The 250 ms tick is what keeps the assertion meaningful and the test honest at the same time. A +// visit owes 9 slots of read_gap(250, 6) = 41.7 ms, i.e. 375 ms, so a two-node rotation is 750 ms: +// the absence loop below spans more than two full rotations, and a bud1 visit that truncated its +// expect set has several chances to clear inside it. It also keeps the retry cadence short enough +// that a first read lost to discovery still recovers well inside poll_for_new's 13 s bound, which +// was sized to outlast the transport's 10 s negative cache. +TEST_F(ParamDriftIntegrationTest, ExpectBudgetTruncationDoesNotFlap) { + auto bud1 = std::make_shared("pd_it_bud1"); + bud1->declare_parameter("a_ok", 1.0); + bud1->declare_parameter("b_ok", 2.0); + bud1->declare_parameter("z_bad", 9.0); // violates the LAST sorted pin from the start + auto bud2 = std::make_shared("pd_it_bud2"); + bud2->declare_parameter("a_ok", 1.0); + bud2->declare_parameter("b_ok", 2.0); + bud2->declare_parameter("z_bad", 3.0); // all pins satisfied + exec_.add_node(bud1); + exec_.add_node(bud2); + const auto nodes_guard = on_scope_exit([this, bud1, bud2] { + exec_.remove_node(bud1); + exec_.remove_node(bud2); + }); + set_apps({{"pd_it_bud1", "/pd_it_bud1"}, {"pd_it_bud2", "/pd_it_bud2"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json{{"baseline", false}, + {"tick_interval_ms", 250}, + {"max_reads_per_tick", 6}, + {"expect", {{"a_ok", 1.0}, {"b_ok", 2.0}, {"z_bad", 3.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/pd_it_bud1")); + ASSERT_TRUE(wait_param_service("/pd_it_bud2")); + + // Establish the raised state (tolerates initial discovery misses). Once bud1's z_bad drift is + // observed it stays in drifted_ (a failed read never erases it), so the clear count freezes. + const auto failed0 = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed0, *det, ctx)); + const auto passed_after_raise = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + const auto failed_after_raise = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + + // Tick across several full round-robin rotations. A visit owes 375 ms and there are two nodes, + // so a rotation is 750 ms and this 2 s loop covers more than two of them - bud1 is re-read + // several times inside it. The clear count must NOT advance: z_bad is caught atomically on every + // bud1 visit. + for (int i = 0; i < 40; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_after_raise) + << "spurious clear at rotation tick " << i << " - expect set truncated by a per-call budget"; + } + EXPECT_GT(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), failed_after_raise); +} + +// The other side of the same rule, in the OTHER read mode. A visit in `expect` mode makes one +// get_parameter per pin and any of them can come back with something that is not NOT_FOUND - a +// wedged executor, parameter services torn down mid-sweep. Publishing what was collected up to that +// point would be reporting a partial view as a complete one: evaluate() finds nothing wrong in a +// view that is simply missing the parameter that is wrong, and the tick clears a fault that is +// still true. NOT_FOUND is the one failure that is deliberately carried on with, because a node is +// not at fault for a parameter it never declared. +// +// No live node can be asked to answer a get with SERVICE_UNAVAILABLE on demand, which is why this +// needs the stand-in. +TEST_F(ParamDriftIntegrationTest, AFailedExpectReadDoesNotClearARaisedDrift) { + const std::string kFqn = "/pd_it_expfail"; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 20}, {"max_reads_per_tick", 8}, {"expect", {{"gain", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_expfail", kFqn}}); + + // The stand-in answers 9.0, so the pin is violated from the first read. + det->tick(ctx); + ASSERT_NE(fake, nullptr); + fake->set_value(kFqn, 9.0); + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "the pinned violation never raised, so there is no fault whose clearing this test can watch"; + + // Every read of the pin now fails with SERVICE_UNAVAILABLE. The parameter is still 9.0. + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + fake->set_unreadable(kFqn); + for (int i = 0; i < 40; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "a failed expect read was published as a complete view of the node, and the pinned " + "violation it does not contain was cleared while the parameter is still wrong"; + + // Positive control: a read that succeeds AND satisfies the pin does clear it, so the silence + // above is the guard and not a detector that has stopped emitting. + fake->set_value(kFqn, 1.0); + fake->set_readable(kFqn); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the fault never cleared even once the pin was satisfied on a readable node"; +} + +// tick_interval_ms is injected by the plugin and is half of what paces the reader: read_gap divides +// the TICK PERIOD by the budget, so a detector that took the budget and ignored the period would +// pace every deployment at its own default no matter what an operator configured - and the coverage +// latency the README tells them to compute would be wrong by whatever factor they had changed the +// tick by. Nothing asserted on the key directly. +// +// Two runs differing ONLY in tick_interval_ms. At a budget of eight, one slot is period/8 and a +// baseline visit owes two of them, so sixteen visits after the first owe 4 x period: ~1.6 s at a +// 400 ms period and ~0.4 s at a 100 ms one. Both bands exclude the ignored-key behaviour, which +// paces both runs at the 1000 ms default and takes ~4 s. +TEST_F(ParamDriftIntegrationTest, TheInjectedTickIntervalPacesTheReader) { + const auto time_seventeen_visits = [this](int tick_interval_ms) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + EXPECT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })); + det->configure(nlohmann::json{{"tick_interval_ms", tick_interval_ms}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_pace", "/pd_it_pace"}}); + + // One tick arms the reader; it round-robins its single target on its own from there, so the + // elapsed time is pure pacing rather than the tick loop's cadence. + const auto started = std::chrono::steady_clock::now(); + det->tick(ctx); + EXPECT_NE(fake, nullptr); + EXPECT_TRUE(fake != nullptr && fake->wait_until_reading(17)) + << "the reader never issued seventeen visits at a " << tick_interval_ms << " ms tick period"; + return std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count(); + }; + + const auto slow_ms = time_seventeen_visits(400); + EXPECT_GT(slow_ms, 1200) << "seventeen visits at a 400 ms tick period took " << slow_ms + << " ms: the reader is pacing faster than the configured period allows"; + EXPECT_LT(slow_ms, 2400) << "seventeen visits at a 400 ms tick period took " << slow_ms + << " ms, which is the default 1000 ms period - the injected value was ignored"; + + const auto fast_ms = time_seventeen_visits(100); + EXPECT_LT(fast_ms, 800) << "seventeen visits at a 100 ms tick period took " << fast_ms + << " ms: shortening the tick period did not shorten the reader's slot, so " + "the injected value is not reaching read_gap"; + EXPECT_GT(fast_ms, 200) << "seventeen visits at a 100 ms tick period took " << fast_ms + << " ms: the reader is not honouring the budget at all"; +} + +// ================================================================================================ +// Evidence for a clear, continued: the two states in which the detector knows NOTHING about an app +// and previously said "nothing is drifting anywhere" anyway. +// ================================================================================================ + +// A gate-denied app never reaches the armed set, so it never reaches the unread set and never joins +// the blocking one either. With nothing armed at all - which is the state of EVERY app for the +// first `warmup_cycles` ticks of every gateway or plugin start - the finding set is empty and the +// blocking set is empty, and that pair is exactly what emit_aggregated reads as "nothing is +// drifting anywhere". clear_fault carries no gate of its own (by design: a fault must always be +// clearable), so the clear goes out. With `healing_enabled`, which this package's README tells +// operators to switch on, those clears walk a persisted CONFIRMED GRAPH_PARAM_DRIFT toward HEALED +// before the detector has issued a single parameter round trip. +// +// The evidence rule does not care WHY an app has no usable read. "The round-robin has not reached +// it" and "the gate has not armed it" are the same state: nothing is known about the app, so +// nothing may be said about it. +TEST_F(ParamDriftIntegrationTest, NoClearWhileTheGateHasNotArmedAnyApp) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}}); + + IntrospectionInput snap; + { + App a; + a.id = "pd_it_warmup"; + a.bound_fqn = "/pd_it_warmup"; + snap.apps.push_back(a); + } + // A REAL gate, fed exactly one snapshot. warmup_cycles is the shipped default of 5 and one tick + // has elapsed, so the app is KNOWN and NOT armed - the state every app is in immediately after a + // restart, reached here without having to drive five seconds of wall clock. + ReliabilityGate gate(5, gateway_.get(), &node_mutex_); + gate.update(snap, 1); + ASSERT_FALSE(reliability_allows(&gate, "pd_it_warmup")); + + auto ctx = make_ctx(DetectorMode::Raise); + ctx.gate = &gate; + ctx.snapshot = &snap; + + for (int i = 0; i < 40; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + std::this_thread::sleep_for(300ms); // reports are service round trips; let the in-flight ones land + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), 0u) + << "cleared GRAPH_PARAM_DRIFT while the only app in the graph was still warming up and had " + "never been read once - health the detector never measured, emitted on every tick of " + "every restart"; + + // Positive control: the guard releases the moment the app arms and is actually read. Without it + // the zero above is satisfied just as well by a detector that emits nothing at all. + gate.update(snap, 20); + ASSERT_TRUE(reliability_allows(&gate, "pd_it_warmup")); + ASSERT_NE(fake, nullptr); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "no clear even once the app had armed and been read, so the guard never releases"; +} + +// The other state: an app that answered ONCE and then stopped for good. Its cached read keeps it in +// the measured set forever - it never joins the unread set, so it never blocks a clear and neither +// unread warning can ever fire about it. The detector then reports "nothing is drifting anywhere" +// off a snapshot of a moment that has passed, indefinitely, and says nothing at all about the fact. +// +// A cached read is evidence about the instant it was taken, not a standing licence. The moment a +// read of an app FAILS, what is cached about it stops being a measurement of the app's current +// state, and the detector is back to knowing nothing. +TEST_F(ParamDriftIntegrationTest, AnAppThatAnsweredOnceAndThenStoppedIsNoLongerCountedAsMeasured) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // The default tick period against the default budget: one slot is 125 ms and a baseline visit, + // successful or failed, is charged two of them - so a failed attempt lands every 250 ms and the + // tenth arrives inside three seconds, well short of the sixty-first that ends the blocking. + det->configure(nlohmann::json{{"tick_interval_ms", 1000}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_onceonly", "/pd_it_onceonly"}}); + + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "the app was never read, so it was never 'measured' and there is nothing here to go stale"; + ASSERT_NE(fake, nullptr); + + // And now it stops answering, permanently. Nothing about its parameters is known from here on. + // + // The baseline for the clear count is taken once a read has actually FAILED and the detector has + // seen it, not the instant the stand-in is reconfigured: a clear emitted while the last completed + // read was still a success is a clear made on evidence that was current, and this test is about + // what happens afterwards. + fake->set_unreadable("/pd_it_onceonly"); + ASSERT_TRUE(fake->wait_until_failed(1)) << "no read of the silenced app ever failed"; + det->tick(ctx); + std::this_thread::sleep_for(200ms); // let a report issued before that tick land + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < 90 && log.count("consecutive parameter reads of 'pd_it_onceonly'") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + EXPECT_EQ(log.count("consecutive parameter reads of 'pd_it_onceonly'"), 1) + << "ten consecutive reads of the app failed and nothing was said about it, because a cached " + "read from before the node went silent keeps it out of the unread set entirely"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), passed_before) + << "kept clearing GRAPH_PARAM_DRIFT off a cached read taken before the node stopped " + "answering, so a drift introduced since then is reported as health"; + + // Positive control: the app is not written off, it is simply unmeasured. Answering again puts it + // straight back into the measured set and the clear flows. + fake->set_readable("/pd_it_onceonly"); + const auto passed_again = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_again, *det, ctx)) + << "the app answered again and the clear never came back, so the guard latches instead of " + "tracking what is currently known"; +} + +// Counting the give-up in FAILED ATTEMPTS rather than elapsed ticks is right - a tick horizon +// expires on exactly the large graphs where the rotation is slowest - but attempts alone are not a +// duration. The reader paces itself at `max_reads_per_tick` round trips per tick period, so an +// attempt costs read_gap x its charge in wall clock, and the whole horizon scales with the load +// knob. At the accepted ceiling of 100000 the gap floors at one microsecond, a failed baseline +// visit owes two of them, and sixty-one attempts are spent in a couple of milliseconds: the very +// first ticks of a run write off any node whose parameter service DDS has not finished discovering +// yet. The README's own remedy for a slow sweep - raise max_reads_per_tick - is what collapses it. +// +// So the horizon needs a floor that the budget cannot move. Ticks are that floor: they are the one +// quantity the load knob does not scale. Used as a CONJUNCTION with the attempt count it can only +// ever delay the give-up, never bring it forward, so it does not reintroduce the failure mode that +// counting attempts was adopted to fix. +TEST_F(ParamDriftIntegrationTest, GivingUpOnAnAppNeedsElapsedTicksAsWellAsFailedAttempts) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_slowstart"); // before the reader can ever answer for it + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 1000}, {"max_reads_per_tick", 100000}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_slowstart", "/pd_it_slowstart"}}); + + det->tick(ctx); // builds the transport and hands the reader its single target + ASSERT_NE(fake, nullptr); + // Far more failed attempts than the horizon counts, inside a handful of ticks. This is the whole + // point: at this budget the ATTEMPT count is exhausted before the run has properly begun. + ASSERT_TRUE(fake->wait_until_failed(300)) << "the reader never got anywhere near the attempt horizon at " + "the ceiling budget, so this test is not measuring what it claims"; + for (int i = 0; i < 4; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(log.count("giving up on"), 0) + << "wrote the app off as unmeasurable within five ticks of the run starting, purely because " + "the load knob was raised: a node that is merely slow to advertise its parameter services " + "is declared unmeasurable before DDS has finished discovering it"; + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED), 0u) + << "cleared GRAPH_PARAM_DRIFT on the strength of that write-off"; + + // Positive control: the give-up is floored, not removed. Keep ticking and it arrives - otherwise + // one node with no parameter service holds the fault CONFIRMED for the life of the gateway. + for (int i = 0; i < 300 && log.count("giving up on") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(2ms); + } + EXPECT_EQ(log.count("giving up on"), 1) << "the app never stopped blocking the clear, so the floor became a latch"; +} + +// An app id whose BINDING moves to a different node is a different node, whatever the id says. The +// reader already treats it that way - a target's arming token is keyed on (app_id, fqn), so a read +// in flight from the old binding can never publish into the new one - but nothing resets the state +// the old binding left behind: the captured baseline, the cached read, the failure count. The new +// node's first reading is then compared against values a DIFFERENT node happened to hold, which is +// a drift no later read can heal, because the reference itself belongs to somewhere else. +TEST_F(ParamDriftIntegrationTest, AnFqnReBindStartsFromAFreshBaseline) { + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_value("/pd_it_bind_a", 1.0); + t->set_value("/pd_it_bind_b", 5.0); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 20}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + + set_apps({{"pd_it_bind", "/pd_it_bind_a"}}); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "the app was never read under its first binding, so there is no stale baseline to carry over"; + ASSERT_NE(fake, nullptr); + + // The same app id, now bound to a DIFFERENT node holding a different value. + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + set_apps({{"pd_it_bind", "/pd_it_bind_b"}}); + for (int i = 0; i < 30; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + EXPECT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), failed_before) + << "reported the new node's value as drift against a baseline captured from a DIFFERENT node: " + << latest_failed_desc("graph_watchdog"); + + // Positive control: the new node's value really was captured as the reference, so moving off it + // still reports. Without this the silence above is satisfied by a detector that captured nothing. + const auto failed_before_drift = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + fake->set_value("/pd_it_bind_b", 9.0); + EXPECT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before_drift, *det, ctx)) + << "no drift off the re-captured baseline, so the silence above proves nothing at all"; +} + +// The unmatched-pin warning waits for the sweep to cover the graph, which raises the question of +// whether it can still fire on a graph big enough to take many ticks to cover, and on one where +// short-lived nodes keep arriving. Both are answered here: sixty apps, and a run of one-sweep +// transients on top of them. The warning has to arrive in both. +// +// Sixty apps at eight round trips per 10 ms tick period is a 1.25 ms slot; an undeclared pin is +// charged two, so a rotation is 150 ms and the loop below spans many of them. The transients are +// the shape a real graph churns in - one `ros2` CLI invocation, present for a sweep under a name +// nothing will use again - and each of them legitimately holds the warning back while it is there. +// What must not happen is that they hold it back for good. +TEST_F(ParamDriftIntegrationTest, AnUnmatchedPinIsStillReportedOnALargeAndChurningGraph) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_declared(std::set{"gain"}); // no node declares the misspelt pin + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 10}, {"max_reads_per_tick", 8}, {"expect", {{"gian", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + for (int i = 0; i < 60; ++i) { + const std::string id = "pd_it_wide2_" + std::to_string(100 + i); // uniform width -> sorted == creation order + apps.emplace_back(id, "/" + id); + } + + // Churn phase: every tick one short-lived app arrives under a name never reused, on top of the + // sixty that stay. Nothing may be reported while such an app has not been looked at. + for (int i = 0; i < 60; ++i) { + auto churned = apps; + churned.emplace_back("pd_it_cli_" + std::to_string(i), "/pd_it_cli_" + std::to_string(i)); + set_apps(churned); + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + ASSERT_NE(fake, nullptr); + + // The churn stops, the sixty stay. The sweep can now finish, and the misspelt pin must be named. + set_apps(apps); + for (int i = 0; i < 300 && log.count("no app declares 'gian'") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + EXPECT_EQ(log.count("no app declares 'gian'"), 1) + << "the misspelt pin was never named on a sixty-app graph that had just been churning, so the " + "warning is dead on exactly the graphs a silent pin is easiest to miss on"; + EXPECT_EQ(log.count("no app declares 'gain'"), 0) << "the pin every node declares was reported as unmatched"; +} + +// ================================================================================================ +// Pins for behaviour that a mutation of the detector's own constants used to leave green. +// ================================================================================================ + +// The per-read timeout is a BOUND, and a bound is only real if a read that overruns it is +// abandoned. Nothing here could see that: tick() does no IO, so its duration says nothing, and the +// unresponsive-node test only requires the sweep to get PAST a node that never answers at all - +// which it does whether the bound is half a second or half an hour. +// +// The discriminator is a node that answers LATE rather than never. Its parameter service takes +// 1.5 s to reply, which is past the 0.5 s per-read bound and inside anything meaningfully larger. +// At the real bound the read is abandoned and the node is never read, so it can never appear in a +// description; raise the bound and the same reply lands and it does. The counter on the service +// handler is what keeps that absence honest: it proves the read was actually issued and the node +// actually served it late, rather than the transport never having found the service at all. +TEST_F(ParamDriftIntegrationTest, AReadThatOverrunsThePerReadBoundIsAbandoned) { + const std::string kSlow = "pd_it_to_slow"; // sorts after pd_it_to_ok: read second in the rotation + const std::string kOk = "pd_it_to_ok"; + std::atomic slow_calls{0}; + + auto ok = std::make_shared(kOk); + ok->declare_parameter("speed_limit", 3.5); // violates the pin below from the first read + exec_.add_node(ok); + const auto ok_guard = on_scope_exit([this, ok] { + exec_.remove_node(ok); + }); + + // The late node. list_parameters is the round trip an `expect` read makes first, so holding it is + // what makes the whole read overrun; get_parameters exists so discovery finds a complete node. + auto slow = std::make_shared(kSlow, rclcpp::NodeOptions().start_parameter_services(false)); + auto slow_list = slow->create_service( + "/" + kSlow + "/list_parameters", service_callback( + [&slow_calls](const rcl_interfaces::srv::ListParameters::Request & /*req*/, + rcl_interfaces::srv::ListParameters::Response & resp) { + ++slow_calls; + std::this_thread::sleep_for(1500ms); + resp.result.names = {"speed_limit"}; + })); + auto slow_get = slow->create_service( + "/" + kSlow + "/get_parameters", + service_callback([](const rcl_interfaces::srv::GetParameters::Request & req, + rcl_interfaces::srv::GetParameters::Response & resp) { + for (std::size_t i = 0; i < req.names.size(); ++i) { + rcl_interfaces::msg::ParameterValue v; + v.type = rcl_interfaces::msg::ParameterType::PARAMETER_DOUBLE; + v.double_value = 7.5; // would violate the pin, if it were ever read + resp.values.push_back(v); + } + })); + const auto slow_idle = add_idle_parameter_services(slow, "/" + kSlow); + // Its own executor: the 1.5 s sleep owns whatever thread serves it, and blocking the fixture's + // executor would stall the healthy node and the fake fault service with it. + rclcpp::executors::SingleThreadedExecutor slow_exec; + slow_exec.add_node(slow); + std::thread slow_spin([&slow_exec] { + slow_exec.spin(); + }); + const auto slow_guard = on_scope_exit([&slow_exec, &slow_spin] { + slow_exec.cancel(); + if (slow_spin.joinable()) { + slow_spin.join(); + } + }); + + set_apps({{kOk, "/" + kOk}, {kSlow, "/" + kSlow}}); + auto det = make_param_drift(); + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 100}, {"max_reads_per_tick", 8}, {"expect", {{"speed_limit", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + + ASSERT_TRUE(wait_param_service("/" + kOk)); + ASSERT_TRUE(wait_param_service("/" + kSlow)); + + // Positive control FIRST: the healthy node's pinned violation is reported, which proves the + // transport, the reader and the aggregate are all alive on this graph. + ASSERT_TRUE(poll_until( + [this, &kOk]() { + return latest_failed_desc_contains("graph_watchdog", {kOk, "got=3.5"}); + }, + *det, ctx)) + << "the healthy node's pinned violation never reported, so the absence below proves nothing: " + << latest_failed_desc("graph_watchdog"); + + // And the late node WAS reached: it served the request, it just served it too late. + for (int i = 0; i < 200 && slow_calls.load() == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + ASSERT_GT(slow_calls.load(), 0) << "the late node's parameter service was never called at all, so nothing " + "here exercised the per-read bound"; + + // Give a read that was NOT abandoned every chance to land: the reply is 1.5 s behind the request. + for (int i = 0; i < 40; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(100ms); + } + EXPECT_EQ(latest_failed_desc("graph_watchdog").find(kSlow), std::string::npos) + << "a node whose parameter service took 1.5 s to answer was read anyway, so the per-read " + "bound is no longer half a second and one slow node can hold the sweep for as long as it " + "likes: " + << latest_failed_desc("graph_watchdog"); +} + +// The unread warning is worth something only if it means what it says. Firing it at the first +// failed read makes it a bringup notice: one dropped reply on a busy graph is not "this node does +// not answer", and an operator who is told it every time stops reading the log. Both sides of the +// threshold are pinned here, off the ATTEMPT counter rather than off wall clock, because the +// attempt count is the thing under test and pacing it by time would make the assertion a function +// of the budget instead. +TEST_F(ParamDriftIntegrationTest, TheUnreadWarningWaitsForTenFailedAttemptsNotOne) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_tenth"); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // One slot is 125 ms and a failed baseline visit is charged two, so attempts arrive every 250 ms: + // slow enough that the assertion below cannot race the attempt it is asserting about. + det->configure(nlohmann::json{{"tick_interval_ms", 1000}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_tenth", "/pd_it_tenth"}}); + + det->tick(ctx); + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_failed(5)) << "the reader never made five attempts against the silent app"; + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count("consecutive parameter reads of"), 0) + << "named the app after a handful of failed reads: a single dropped reply is not evidence " + "that a node has stopped answering, and a warning that fires on one is noise"; + + for (int i = 0; i < 200 && log.count("consecutive parameter reads of 'pd_it_tenth'") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_EQ(log.count("consecutive parameter reads of 'pd_it_tenth'"), 1) + << "the app that never answers was never named, so there is no threshold here to pin"; + EXPECT_GE(fake->failures(), 10) << "the warning fired after only " << fake->failures() + << " failed attempts, short of the ten it documents"; +} + +// A per-app detail that overruns its share is cut from the MIDDLE, keeping both ends. The head +// names the app and its first drifted parameter; the tail is what stops the entry ending mid-token +// and is the only place the last of a long run of drifted parameters can survive. Head-only +// truncation renders the whole tail unreachable and leaves every existing assertion green, because +// they all look at the front of the entry. +TEST_F(ParamDriftIntegrationTest, AnOverlongPerAppDetailKeepsItsTailAsWellAsItsHead) { + const std::string kApp = "pd_it_tail"; + // Six drifted parameters render to about 230 characters, comfortably past the 150 an app may + // spend, and the last of them carries a name nothing else in the text can produce. + const std::vector names{"p1_aaaa", "p2_bbbb", "p3_cccc", "p4_dddd", "p5_eeee", "zz_lastone"}; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake, &kApp, &names](rclcpp::Node *) { + auto t = std::make_unique(); + std::map clean; + for (const auto & n : names) { + clean.emplace(n, 1.0); + } + t->set_params("/" + kApp, clean); // installed in the factory: the first read must be the clean one + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 100}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{kApp, "/" + kApp}}); + + ASSERT_TRUE(wait_for_baseline_evidence(*det, ctx)) << "the app was never read, so it has no baseline to drift from"; + ASSERT_NE(fake, nullptr); + ASSERT_EQ(count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED), 0u) + << "baseline capture must be silent for the assertions below to mean anything"; + + std::map drifted; + for (const auto & n : names) { + drifted.emplace(n, 9.0); + } + fake->set_params("/" + kApp, drifted); + ASSERT_TRUE(poll_until( + [this]() { + return latest_failed_desc("graph_watchdog").find("...") != std::string::npos; + }, + *det, ctx)) + << "the entry never overran its share, so there was nothing to trim and this test would pass " + "under any trimming rule at all: " + << latest_failed_desc("graph_watchdog"); + + const std::string desc = latest_failed_desc("graph_watchdog"); + EXPECT_EQ(desc.rfind(kApp + ":\"p1_aaaa\"", 0), std::size_t{0}) + << "the head no longer names the app and its first drifted parameter: " << desc; + EXPECT_NE(desc.find("zz_lastone"), std::string::npos) + << "the trimmed entry keeps only its head, so the last of a long run of drifted parameters is " + "unreachable however many of them there are: " + << desc; + EXPECT_EQ(desc.find("p3_cccc"), std::string::npos) + << "nothing was actually cut, so 'both ends' is not what is being tested here: " << desc; +} + +// A graph on which every armed app has been GIVEN UP on is swept in name only. Nothing answered, so +// nothing is known about which parameters exist anywhere, and a pin cannot be called unmatched on +// the strength of it - that is a warning telling an operator to go and fix a name which may well be +// correct, produced by a detector that read nothing. +TEST_F(ParamDriftIntegrationTest, AnUnmatchedPinIsNotReportedWhenEveryAppWasGivenUpOn) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_allmute"); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{ + {"baseline", false}, {"tick_interval_ms", 10}, {"max_reads_per_tick", 8}, {"expect", {{"gian", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_allmute", "/pd_it_allmute"}}); + + // Tick well past the point the only app is given up on: from there the sweep is "complete" in the + // sense the clear uses, and nothing but the answered-something guard stands between the pin and a + // warning about it. + for (int i = 0; i < 400 && log.count("giving up on") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_NE(fake, nullptr); + ASSERT_EQ(log.count("giving up on"), 1) << "the app was never given up on, so the swept-in-name-only state this " + "test is about was never reached"; + for (int i = 0; i < 60; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + EXPECT_EQ(log.count("no app declares 'gian'"), 0) + << "called the pin unmatched on a graph where not one app ever answered a read, which says " + "nothing whatever about which parameters the graph declares"; + + // Positive control: once an app does answer and still does not declare the pin, the warning is + // due and must arrive. Otherwise the zero above is a detector that never reports anything. + fake->set_declared(std::set{"gain"}); + fake->set_readable("/pd_it_allmute"); + for (int i = 0; i < 300 && log.count("no app declares 'gian'") == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + EXPECT_EQ(log.count("no app declares 'gian'"), 1) + << "the pin was never reported even once an app had answered and did not declare it"; +} + +// The unread warnings are one-shot per RUN OF FAILURES, not once per process. An app that goes +// quiet, comes back, and goes quiet again has failed twice, and the second time is worth saying: +// the operator who fixed it needs to know it has broken again. What makes that possible is erasing +// the latch when the app leaves the unread set - and leaving the latch behind is invisible, because +// silence is what a working latch looks like too. +TEST_F(ParamDriftIntegrationTest, TheUnreadWarningFiresAgainAfterTheAppRecoversAndFailsOnceMore) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_unreadable("/pd_it_relapse"); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 20}, {"max_reads_per_tick", 8}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{"pd_it_relapse", "/pd_it_relapse"}}); + + const std::string needle = "consecutive parameter reads of 'pd_it_relapse'"; + for (int i = 0; i < 400 && log.count(needle) == 0; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_NE(fake, nullptr); + ASSERT_EQ(log.count(needle), 1) << "the app that never answered was never named the first time"; + + // It recovers. The reader reads it, the failure count is dropped, and so must the latch be. + fake->set_readable("/pd_it_relapse"); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the recovered app was never read again, so nothing could have reset its latch either way"; + + // And it breaks again. + fake->set_unreadable("/pd_it_relapse"); + for (int i = 0; i < 400 && log.count(needle) < 2; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + EXPECT_EQ(log.count(needle), 2) + << "the app broke a second time and nothing was said, because the one-shot latch from the " + "first run of failures was never erased - so from the first failure onward this app is " + "silent for the life of the gateway"; +} + +// The description's ordering has four buckets and the cross combination is what decides a real +// report: a pinned violation that has already been reported against self-captured drift that is +// brand new. Priority runs pinned-before-self-captured FIRST and newly-affected-before-reported +// only WITHIN a bucket, so the pinned entry leads even though the other one is the thing that just +// changed - an operator who declared a value cares that it is still wrong more than about a value +// the detector itself decided to watch. +// +// The two rules were only ever exercised one at a time, so swapping the middle two buckets left the +// whole suite green. The assertion is taken on the FIRST report the new app appears in, because +// that is the only report in which the two orderings differ at all. +TEST_F(ParamDriftIntegrationTest, AReportedPinnedViolationStillOutranksBrandNewSelfCapturedDrift) { + const std::string kPin = "pinned_gain"; + const std::string kPinned = "pd_ord_x_pinned"; // sorts AFTER the self-captured app + const std::string kSelf = "pd_ord_a_self"; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake, &kPin, &kPinned, &kSelf](rclcpp::Node *) { + auto t = std::make_unique(); + t->set_params("/" + kPinned, {{kPin, 9.0}}); // violates its pin from the very first read + t->set_params("/" + kSelf, {{"gain", 1.0}}); // clean, so its first read is a silent capture + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 50}, {"max_reads_per_tick", 8}, {"expect", {{kPin, 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + set_apps({{kSelf, "/" + kSelf}, {kPinned, "/" + kPinned}}); + + // The pinned violation is reported on its own for a while: by the time the other app drifts it is + // firmly in the already-reported bucket, which is the whole point of the combination. + ASSERT_TRUE(poll_until( + [this, &kPinned]() { + return latest_failed_desc_contains("graph_watchdog", {kPinned}); + }, + *det, ctx)) + << "the pinned violation never reported, so there is no reported-pinned entry to rank"; + ASSERT_NE(fake, nullptr); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(50ms); + } + ASSERT_EQ(latest_failed_desc("graph_watchdog").find(kSelf), std::string::npos) + << "the self-capturing app is in the description already, so its arrival below says nothing: " + << latest_failed_desc("graph_watchdog"); + + // Now the self-capturing app drifts. It is brand new to the report and it sorts first, so plain + // map order and newly-affected-first both put it ahead; only pinned-first keeps the pin in front. + fake->set_params("/" + kSelf, {{"gain", 9.0}}); + ASSERT_TRUE(poll_until( + [this, &kSelf]() { + return !first_failed_desc_containing("graph_watchdog", kSelf).empty(); + }, + *det, ctx)) + << "the self-captured drift was never reported at all"; + const std::string first = first_failed_desc_containing("graph_watchdog", kSelf); + EXPECT_EQ(first.rfind(kPinned + ":", 0), std::size_t{0}) + << "brand-new self-captured drift was put ahead of a violation of a pin an operator wrote, so " + "on a full description the declared value is the one that gets cut: " + << first; +} + +// The captured baseline is dropped on the THIRD consecutive missed sweep, and both ends of that +// matter. Dropping it sooner re-captures a node on its already-drifted value after an ordinary +// dropped discovery poll - a confirmed fault heals while the parameter is still wrong, and can +// never fire again because the new baseline IS the wrong value. Dropping it later means a restart, +// which is the documented way to accept a new configuration, keeps reporting the operator's own +// change as drift. +// +// Existing coverage only pinned the short side. `prune_grace` is set well above the horizon here so +// the FINDING prune cannot reach it: the baseline forget is the only thing that can be under test. +TEST_F(ParamDriftIntegrationTest, TheBaselineForgetHorizonIsExactlyThreeMissedSweeps) { + struct Case { + int absences; + bool expect_drift; + const char * why; + }; + const std::vector cases{ + {2, true, + "two missed sweeps re-baselined the node, so one dropped discovery poll away from that " + "launders a real drift into the new reference"}, + {3, false, + "three missed sweeps did NOT re-baseline the node, so a restart - the documented way to " + "apply a new configuration - reports the operator's own change as drift"}, + }; + + int seq = 0; + for (const auto & [absences, expect_drift, why] : cases) { + SCOPED_TRACE("absences=" + std::to_string(absences)); + ++seq; + const std::string id = "pd_it_forget" + std::to_string(seq); + const std::string fqn = "/" + id; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + // prune_grace far above the absence counts below, so `dropping` is never true and the finding + // prune cannot stand in for the baseline forget. + det->configure(nlohmann::json{{"tick_interval_ms", 10}, {"max_reads_per_tick", 8}, {"prune_grace", 60}}); + auto ctx = make_ctx(DetectorMode::Raise); + + set_apps({{id, fqn}}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the app was never read, so it has no baseline whose fate this case could observe"; + ASSERT_NE(fake, nullptr); + + // Exactly `absences` sweeps without it. prune_vanished counts one missed sweep per tick. + set_apps({}); + for (int i = 0; i < absences; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + + // It comes back holding a value it never held while it was present. + fake->set_value(fqn, 5.0); + set_apps({{id, fqn}}); + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + bool drifted = false; + for (int i = 0; i < 60 && !drifted; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + drifted = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED) > failed_before; + } + EXPECT_EQ(drifted, expect_drift) << why << ": " << latest_failed_desc("graph_watchdog"); + } +} + +// Withholding a clear is correct and, from outside the process, completely invisible. No fault is +// raised, no fault is cleared, and a GRAPH_PARAM_DRIFT already in the store simply never heals - +// which is exactly what a detector that is working and finding nothing looks like. Worse, the apps +// doing the holding are usually ones the per-app unread warning may NOT name: nothing has been +// attempted against them, because they are waiting their turn or the gate has not armed them, and +// naming those would be a bringup warning about healthy nodes. So the aggregate has to speak for +// itself: once per episode, how long it has been withheld and how many apps are behind it. +// +// 160 apps at one round trip per 200 ms tick period is a rotation of a minute, so the tail of the +// graph stays uncontacted for the whole run. No sleeps: the horizon is counted in TICKS. +TEST_F(ParamDriftIntegrationTest, AWithheldClearSaysSoInTheLog) { + const LogCapture log; + + ScriptedTransport * fake = nullptr; + auto det = make_param_drift(); + ASSERT_TRUE(det->set_parameter_transport_factory_for_test([&fake](rclcpp::Node *) { + auto t = std::make_unique(); + fake = t.get(); + return t; + })) << "the transport factory hook did not reach param_drift"; + det->configure(nlohmann::json{{"tick_interval_ms", 200}, {"max_reads_per_tick", 1}}); + auto ctx = make_ctx(DetectorMode::Raise); + + std::vector> apps; + for (int i = 0; i < 160; ++i) { + const std::string id = "pd_it_held" + std::to_string(1000 + i); + apps.emplace_back(id, "/" + id); + } + set_apps(apps); + det->tick(ctx); + ASSERT_NE(fake, nullptr); + ASSERT_TRUE(fake->wait_until_reading(1)) << "the reader never started, so nothing here is under test"; + + const std::string needle = "is being withheld from clearing"; + for (int i = 0; i < 30; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 0) << "announced a withheld clear after thirty ticks: an ordinary rotation on a " + "large graph takes longer than that, so this fires during normal bringup"; + + for (int i = 0; i < 200; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 1) + << "the clear has been withheld for over two hundred ticks and nothing in the log says so, " + "so an operator watching GRAPH_PARAM_DRIFT refuse to heal has no way to tell a graph the " + "sweep cannot cover from a detector that is working normally (got " + << log.count(needle) << " lines)"; + + // The episode ends when the graph is actually covered, and the warning must re-arm with it: a + // one-shot-per-process latch would say nothing at all the next time coverage breaks down. + set_apps({apps.front()}); + const auto passed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_PASSED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "no clear even once the graph was one already-read app, so the episode never ended"; + + set_apps(apps); + for (int i = 0; i < 300; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 2) + << "coverage broke down a second time and nothing was said, so the warning is one-shot per " + "process rather than per episode"; +} + +// An `expect` pin on a parameter the node DECLARED and never SET. +// +// Not a corner case and not the NOT_FOUND carve-out. rclcpp lets a node declare a dynamically typed +// parameter with no value, and its parameter service answers that name with a PARAMETER_NOT_SET +// VALUE rather than an absent one - so the transport gets a parameter back and returns SUCCESS +// carrying null. The pin therefore DOES fire, which is right: a pin says the value must be 1.0, and +// a node holding no value at all is not holding 1.0. Nothing asserted that in either direction. +// +// What the fault SAYS has to survive the round trip too. A NaN double dumps as `null` as well, so +// one description said the same thing about "the node never set this parameter" and "the value is +// NaN" - two states with different remedies, told apart by nothing. Both are pinned here, on the +// same node and in the same description, so the text has to distinguish them or this fails. +// +// Deliberately a LIVE node and the real transport, not the scripted stand-in. The premise is a +// reply shape rclcpp produces; a stand-in scripted to produce it would restate the premise instead +// of testing it. +TEST_F(ParamDriftIntegrationTest, AnExpectPinOnADeclaredButUnsetParameterRaisesAndReadsAsUnset) { + auto node = std::make_shared("pd_it_unset"); + rcl_interfaces::msg::ParameterDescriptor dynamic_typing; + dynamic_typing.dynamic_typing = true; // a statically typed declaration demands a value or an override + node->declare_parameter("threshold", rclcpp::ParameterValue{}, dynamic_typing); // declared, never set + node->declare_parameter("headroom", std::numeric_limits::quiet_NaN()); // a value, and it is NaN + exec_.add_node(node); + const auto node_guard = on_scope_exit([this, node] { + exec_.remove_node(node); + }); + set_apps({{"pd_it_unset", "/pd_it_unset"}}); + + auto det = make_param_drift(); + det->configure(nlohmann::json{{"baseline", false}, {"expect", {{"threshold", 1.0}, {"headroom", 1.0}}}}); + auto ctx = make_ctx(DetectorMode::Raise); + ASSERT_TRUE(wait_param_service("/pd_it_unset")); // service up before a read-driven raise is asserted + + const auto failed_before = count_faults("graph_watchdog", ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(poll_for_new("graph_watchdog", ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "an expect pin on a parameter the node declared and never set raised nothing at all, so the " + "operator is told the pin is satisfied by a node holding no value for it"; + + const std::string desc = latest_failed_desc("graph_watchdog"); + ASSERT_NE(desc.find("threshold"), std::string::npos) << "the unset parameter is not named: " << desc; + ASSERT_NE(desc.find("headroom"), std::string::npos) << "the NaN parameter is not named: " << desc; + + // Each entry's own `got=` field, cut at the entry separator so the two cannot be confused. + const auto got_of = [&desc](const std::string & param) { + const auto name_at = desc.find('"' + param + '"'); + if (name_at == std::string::npos) { + return std::string{}; + } + const auto got_at = desc.find(" got=", name_at); + if (got_at == std::string::npos) { + return std::string{}; + } + const auto end = desc.find(';', got_at); + return desc.substr(got_at + 5, end == std::string::npos ? std::string::npos : end - got_at - 5); + }; + const std::string unset_got = got_of("threshold"); + const std::string nan_got = got_of("headroom"); + ASSERT_FALSE(unset_got.empty()) << "no got= field for the unset parameter in: " << desc; + ASSERT_FALSE(nan_got.empty()) << "no got= field for the NaN parameter in: " << desc; + + EXPECT_EQ(unset_got, "") << "a parameter the node declared and never set reads as '" << unset_got + << "' in the fault an operator sees: " << desc; + EXPECT_NE(unset_got, nan_got) << "the unset parameter and the NaN one both read as '" << unset_got + << "', so the description cannot tell an operator which fault this is: " << desc; +} diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_policy.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_policy.cpp new file mode 100644 index 00000000..0d2af2d4 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_param_drift_policy.cpp @@ -0,0 +1,648 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ros2_medkit_graph_watchdog/param_drift_policy.hpp" + +using nlohmann::json; +using ros2_medkit_graph_watchdog::glob_match; +using ros2_medkit_graph_watchdog::is_unset_param; +using ros2_medkit_graph_watchdog::kUnsetParameterRendering; +using ros2_medkit_graph_watchdog::param_entry_value; +using ros2_medkit_graph_watchdog::param_values_equal; +using ros2_medkit_graph_watchdog::ParamDriftConfig; +using ros2_medkit_graph_watchdog::ParamDriftPolicy; +using ros2_medkit_graph_watchdog::read_gap; +using ros2_medkit_graph_watchdog::unset_param_value; +using Params = std::map; + +namespace { +/// The `got=` field of one drifted descriptor - the text an operator reads for the value the node +/// actually holds. Empty when the descriptor has no such field, which fails the assertion rather +/// than quietly comparing two empty strings. +std::string rendered_got(const std::string & descriptor) { + const auto at = descriptor.find(" got="); + return at == std::string::npos ? std::string{} : descriptor.substr(at + 5); +} +} // namespace + +TEST(GlobMatch, StarWildcard) { + EXPECT_TRUE(glob_match("*_stamp", "header_stamp")); + EXPECT_TRUE(glob_match("qos_overrides.*", "qos_overrides./scan.reliability")); + EXPECT_TRUE(glob_match("*", "anything")); + EXPECT_TRUE(glob_match("exact", "exact")); + EXPECT_FALSE(glob_match("*_stamp", "stamp_header")); + EXPECT_FALSE(glob_match("exact", "exacter")); +} + +TEST(ParamValuesEqual, NumericNormalization) { + EXPECT_TRUE(param_values_equal(json(20), json(20.0))); // int vs double, same value + EXPECT_TRUE(param_values_equal(json(1.5), json(1.5))); + EXPECT_FALSE(param_values_equal(json(20.0), json(5.0))); + EXPECT_TRUE(param_values_equal(json(false), json(false))); + EXPECT_FALSE(param_values_equal(json(false), json(true))); + EXPECT_TRUE(param_values_equal(json("a"), json("a"))); + EXPECT_FALSE(param_values_equal(json("a"), json("b"))); +} + +// A NaN-valued param (common "unset/disabled" sentinel) must NOT read as drifted against +// itself - IEEE NaN != NaN would make it a permanent false positive that never clears. +TEST(ParamValuesEqual, NaNParamIsStable) { + const double nan = std::numeric_limits::quiet_NaN(); + EXPECT_TRUE(param_values_equal(json(nan), json(nan))); + EXPECT_FALSE(param_values_equal(json(nan), json(1.0))); + // Signed zero is not a drift. Written as an INTEGER parameter against a pin carrying a decimal + // point, because that is the pairing our own comparison has to decide: two doubles of the same + // kind never reach int_equals_float_exactly at all and only exercise nlohmann's operator==. + // -0.0 is the value that makes the integrality check subtle - it is neither greater nor less + // than zero, so a naive `frac != 0.0` test would have to get the sign of zero right. + EXPECT_TRUE(param_values_equal(json(0), json(-0.0))); + EXPECT_TRUE(param_values_equal(json(-0.0), json(0))); // and the mirrored branch +} + +// int64 values differing beyond 2^53 must be caught: a `double` coercion collapses them into a +// MISSED drift on 64-bit IDs, counters and epoch-ns params. Inside an ARRAY, because that is a +// path of our own - the recursion in param_values_equal - and because a 1-D array of large +// integers against a pin written with decimal points is exactly how the two get mixed. Two plain +// integers of the same kind would fall straight through to nlohmann's operator== and test nothing +// this package owns. +TEST(ParamValuesEqual, LargeInt64ExactCompareInsideAnArray) { + EXPECT_TRUE(param_values_equal(json::array({9007199254740992LL}), json::array({9007199254740992.0}))); + EXPECT_FALSE(param_values_equal(json::array({9007199254740993LL}), json::array({9007199254740992.0}))); + // A late element is the one that gets lost when the recursion stops early. + EXPECT_FALSE( + param_values_equal(json::array({1LL, 2LL, 9007199254740993LL}), json::array({1.0, 2.0, 9007199254740992.0}))); +} + +TEST(ParamValuesEqual, NaNInsideArray) { + const double nan = std::numeric_limits::quiet_NaN(); + EXPECT_TRUE(param_values_equal(json::array({nan, 1.0}), json::array({nan, 1.0}))); // never-changed NaN[] not drift + EXPECT_FALSE(param_values_equal(json::array({nan, 1.0}), json::array({nan, 2.0}))); + EXPECT_FALSE(param_values_equal(json::array({1.0}), json::array({1.0, 2.0}))); // length differs +} + +TEST(ParamDriftConfig, FlattensNestedExpectToDottedParamNames) { + // The gateway un-nests dotted config keys, so a pin on a dotted param name + // (expect.qos_overrides./scan.reliability) arrives NESTED. from_json must flatten it back + // to the full dotted param name - the key the observed param map actually uses. `baseline`/ + // `ignore` are single-level keys and survive extract_detector_config's nested output as-is. + json detector_cfg = { + {"baseline", false}, + {"expect", {{"use_sim_time", false}, {"qos_overrides", {{"/scan", {{"reliability", "best_effort"}}}}}}}, + {"ignore", json::array({"*_stamp"})}}; + const auto cfg = ParamDriftConfig::from_json(detector_cfg); + EXPECT_FALSE(cfg.baseline_capture); + ASSERT_EQ(cfg.expect.count("use_sim_time"), 1u); + EXPECT_EQ(cfg.expect.at("use_sim_time"), json(false)); + ASSERT_EQ(cfg.expect.count("qos_overrides./scan.reliability"), 1u); // recovered from nesting + EXPECT_EQ(cfg.expect.at("qos_overrides./scan.reliability"), json("best_effort")); + ASSERT_EQ(cfg.ignore_globs.size(), 1u); + EXPECT_EQ(cfg.ignore_globs[0], "*_stamp"); +} + +TEST(ParamDriftConfig, MalformedExpectWarnsAndKeepsEmpty) { + // A present-but-non-object expect (operator error) must warn, like the other fields. + const auto cfg = ParamDriftConfig::from_json({{"expect", "use_sim_time"}}); // scalar, not object + EXPECT_TRUE(cfg.expect.empty()); + EXPECT_FALSE(cfg.warnings.empty()); +} + +TEST(ParamDriftConfig, MisspeltKeyWarnsInsteadOfBeingDropped) { + // A wrong TYPE on a known key already warned; a wrong NAME did not, though it fails + // the same way: `ignore_globs` (the struct's field name) leaves the globs empty and + // param_drift keeps flagging exactly the params the operator meant to suppress. + const auto cfg = ParamDriftConfig::from_json({{"ignore_globs", json::array({"*_stamp"})}}); + EXPECT_TRUE(cfg.ignore_globs.empty()); + ASSERT_EQ(cfg.warnings.size(), 1u); + EXPECT_NE(cfg.warnings[0].find("ignore_globs"), std::string::npos); +} + +TEST(ParamDriftConfig, EveryKnownKeyIsAcceptedWithoutWarning) { + // Guards the known-key set against drifting away from the fields from_json reads: + // a key dropped from the set here would start warning on a correct config. + // baseline stays ENABLED here: `ignore` filters only the self-captured set, so pairing it with + // `baseline: false` is a dead combination that now warns on purpose (see the test below). + const auto cfg = ParamDriftConfig::from_json({{"baseline", true}, + {"ignore", json::array({"*_stamp"})}, + {"max_reads_per_tick", 4}, + {"expect", {{"use_sim_time", false}}}}); + EXPECT_TRUE(cfg.warnings.empty()) << (cfg.warnings.empty() ? "" : cfg.warnings.front()); +} + +TEST(ParamDriftConfig, DefaultsWhenEmpty) { + const auto cfg = ParamDriftConfig::from_json(json::object()); + EXPECT_TRUE(cfg.baseline_capture); // default true + EXPECT_TRUE(cfg.expect.empty()); + EXPECT_TRUE(cfg.ignore_globs.empty()); + EXPECT_TRUE(cfg.warnings.empty()); +} + +// Malformed config must not be dropped silently - it must warn (a mistyped `ignore` +// would otherwise re-enable drift on suppressed params with no trail). +TEST(ParamDriftConfig, MalformedIgnoreWarnsAndKeepsEmpty) { + const auto scalar = ParamDriftConfig::from_json({{"ignore", "*_stamp"}}); // scalar, not array + EXPECT_TRUE(scalar.ignore_globs.empty()); + EXPECT_FALSE(scalar.warnings.empty()); + + const auto wrong_elem = ParamDriftConfig::from_json({{"ignore", json::array({1, 2})}}); // int array + EXPECT_TRUE(wrong_elem.ignore_globs.empty()); + EXPECT_FALSE(wrong_elem.warnings.empty()); +} + +TEST(ParamDriftConfig, MalformedBaselineWarnsAndKeepsDefault) { + const auto cfg = ParamDriftConfig::from_json({{"baseline", "false"}}); // quoted string, not bool + EXPECT_TRUE(cfg.baseline_capture); // explicit override lost -> kept default true + EXPECT_FALSE(cfg.warnings.empty()); +} + +TEST(ParamDriftConfig, MaxReadsPerTick) { + EXPECT_EQ(ParamDriftConfig::from_json(json::object()).max_reads_per_tick, 8); // default + EXPECT_EQ(ParamDriftConfig::from_json({{"max_reads_per_tick", 20}}).max_reads_per_tick, 20); + const auto bad = ParamDriftConfig::from_json({{"max_reads_per_tick", 0}}); // non-positive + EXPECT_EQ(bad.max_reads_per_tick, 8); // kept default + EXPECT_FALSE(bad.warnings.empty()); +} + +TEST(ParamDriftPolicy, SelfCaptureFirstSeenNoDrift) { + ParamDriftPolicy p{ParamDriftConfig{}}; // baseline true, no expect + const auto out = + p.evaluate("/n", Params{{"speed", json(1.0)}, {"use_sim_time", json(false)}}, /*capture_allowed=*/true); + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, RuntimeChangeDriftsThenClears) { + ParamDriftPolicy p{ParamDriftConfig{}}; + p.evaluate("/n", Params{{"speed", json(1.0)}}, true); // capture + auto out = p.evaluate("/n", Params{{"speed", json(0.2)}}, true); // changed + ASSERT_EQ(out.size(), 1u); + EXPECT_NE(out[0].find("speed"), std::string::npos); + out = p.evaluate("/n", Params{{"speed", json(1.0)}}, true); // restored + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, NewParamAfterCaptureIsSilent) { + ParamDriftPolicy p{ParamDriftConfig{}}; + p.evaluate("/n", Params{{"a", json(1)}}, true); + const auto out = p.evaluate("/n", Params{{"a", json(1)}, {"b", json(2)}}, true); // b is new + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, IgnoredParamNeverDrifts) { + ParamDriftConfig cfg; + cfg.ignore_globs = {"*_stamp"}; + ParamDriftPolicy p{cfg}; + p.evaluate("/n", Params{{"header_stamp", json(1)}}, true); + const auto out = p.evaluate("/n", Params{{"header_stamp", json(999)}}, true); + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, ExpectRuleFiresRegardlessOfBaseline) { + ParamDriftConfig cfg; + cfg.baseline_capture = false; // no self-capture + cfg.expect = {{"use_sim_time", json(false)}}; + ParamDriftPolicy p{cfg}; + // First observation already violates - no prior "capture" needed. + auto out = p.evaluate("/n", Params{{"use_sim_time", json(true)}}, true); + ASSERT_EQ(out.size(), 1u); + EXPECT_NE(out[0].find("use_sim_time"), std::string::npos); + out = p.evaluate("/n", Params{{"use_sim_time", json(false)}}, true); + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, ExpectAbsentParamIsNotDrift) { + ParamDriftConfig cfg; + cfg.baseline_capture = false; + cfg.expect = {{"use_sim_time", json(false)}}; + ParamDriftPolicy p{cfg}; + const auto out = p.evaluate("/n", Params{{"other", json(1)}}, true); // node lacks use_sim_time + EXPECT_TRUE(out.empty()); +} + +TEST(ParamDriftPolicy, BaselineFalseIgnoresUnlistedParams) { + ParamDriftConfig cfg; + cfg.baseline_capture = false; + cfg.expect = {{"use_sim_time", json(false)}}; + ParamDriftPolicy p{cfg}; + p.evaluate("/n", Params{{"use_sim_time", json(false)}, {"speed", json(1.0)}}, true); + const auto out = p.evaluate("/n", Params{{"use_sim_time", json(false)}, {"speed", json(9.0)}}, true); + EXPECT_TRUE(out.empty()); // speed changed but unlisted + no baseline +} + +TEST(ParamDriftPolicy, ExpectPlusBaselineBothProduceDescriptors) { + ParamDriftConfig cfg; + cfg.expect = {{"use_sim_time", json(false)}}; // baseline_capture stays true + ParamDriftPolicy p{cfg}; + p.evaluate("/n", Params{{"use_sim_time", json(false)}, {"speed", json(1.0)}}, true); // capture + const auto out = p.evaluate("/n", Params{{"use_sim_time", json(true)}, {"speed", json(0.2)}}, true); + ASSERT_EQ(out.size(), 2u); // one expect drift + one baseline drift + const std::string joined = out[0] + out[1]; + EXPECT_NE(joined.find("use_sim_time"), std::string::npos); + EXPECT_NE(joined.find("speed"), std::string::npos); +} + +TEST(ParamDriftPolicy, CaptureGatedByCaptureAllowed) { + ParamDriftPolicy p{ParamDriftConfig{}}; + EXPECT_TRUE( + p.evaluate("/n", {{"x", json(1.0)}}, /*capture_allowed=*/false).empty()); // not armed: no capture, no drift + EXPECT_TRUE(p.evaluate("/n", {{"x", json(9.0)}}, /*capture_allowed=*/true) + .empty()); // first armed sweep: captures 9.0, no drift + const auto d = p.evaluate("/n", {{"x", json(1.0)}}, true); // now 1.0 != 9.0 -> drift + ASSERT_EQ(d.size(), 1u); + EXPECT_NE(d[0].find("/n"), std::string::npos); + EXPECT_NE(d[0].find('x'), std::string::npos); +} + +TEST(ParamDriftPolicy, ExpectFiresEvenWhenCaptureNotAllowed) { + ParamDriftConfig cfg; + cfg.baseline_capture = false; + cfg.expect = {{"use_sim_time", json(false)}}; + ParamDriftPolicy p{cfg}; + const auto d = p.evaluate("/n", {{"use_sim_time", json(true)}}, /*capture_allowed=*/false); + ASSERT_EQ(d.size(), 1u); // expect is absolute; independent of capture gating +} + +TEST(ParamDriftPolicy, ForgetDropsBaseline) { + ParamDriftPolicy p{ParamDriftConfig{}}; + p.evaluate("/n", {{"x", json(1.0)}}, true); + p.forget("/n"); + EXPECT_TRUE(p.evaluate("/n", {{"x", json(0.2)}}, true).empty()); // re-captures 0.2, not a drift +} + +// read_gap is the only thing enforcing max_reads_per_tick: the reader takes one target per +// pass and waits out the rest of its slot, so a zero gap is no budget at all. +TEST(ParamDriftPolicy, ReadGapSpreadsTheBudgetOverTheTickPeriod) { + EXPECT_EQ(read_gap(1000, 8), std::chrono::microseconds(125000)); + EXPECT_EQ(read_gap(200, 4), std::chrono::microseconds(50000)); +} + +// A budget larger than the tick period in milliseconds truncates to zero under integer +// millisecond division, which removes the bound instead of tightening it. +TEST(ParamDriftPolicy, ReadGapStaysNonZeroWhenTheBudgetExceedsTheTickInMilliseconds) { + EXPECT_GT(read_gap(1000, 2000).count(), 0); + EXPECT_EQ(read_gap(1000, 2000), std::chrono::microseconds(500)); +} + +// A non-positive budget must not divide by zero; it falls back to one read per period. +TEST(ParamDriftPolicy, ReadGapFallsBackToOneReadPerPeriod) { + EXPECT_EQ(read_gap(1000, 0), std::chrono::microseconds(1000000)); + EXPECT_EQ(read_gap(1000, -5), std::chrono::microseconds(1000000)); +} + +// A fault description is cut at a byte offset, so a rendered value must never contain a +// multi-byte sequence a cut could split, and must never make the renderer throw. +TEST(ParamDriftPolicy, RenderedValuesAreAsciiOnly) { + ParamDriftPolicy p(ParamDriftConfig{}); + p.evaluate("/n", Params{{"path", json("/maps/cafe/map.yaml")}}, true); + const auto out = p.evaluate("/n", Params{{"path", json("/maps/caf\xc3\xa9/map.yaml")}}, true); + ASSERT_EQ(out.size(), 1u); + for (const char c : out[0]) { + EXPECT_LT(static_cast(c), 0x80u) + << "non-ASCII byte in a description that is later cut at a byte offset"; + } +} + +// A parameter whose bytes are not valid UTF-8 must not throw out of evaluate(): the plugin's +// per-detector guard would swallow it and the detector would then neither raise nor clear for as +// long as that value exists, so a confirmed fault could never heal. +TEST(ParamDriftPolicy, InvalidUtf8InAValueDoesNotThrow) { + ParamDriftPolicy p(ParamDriftConfig{}); + // No explicit length: the literal is 10 bytes and hand-counting it got that wrong once, which + // read past the end of the literal and only showed up under AddressSanitizer. + const std::string bad("bad\xff\xfevalue"); + p.evaluate("/n", Params{{"s", json("ok")}}, true); + EXPECT_NO_THROW({ + const auto out = p.evaluate("/n", Params{{"s", json(bad)}}, true); + EXPECT_EQ(out.size(), 1u); + }); +} + +// One array-valued parameter must not be able to consume the whole description budget. +// +// The 400-element array is the `got=` field, so that is what has to be measured. Measuring the text +// between `baseline=` and ` got=` measures the four-element BASELINE, which is a dozen characters +// long whatever elide() does - disabling elision entirely left that form of this test green. +// +// The budget is asserted against a LITERAL and not against the constant itself. Asserting +// `<= kMaxRenderedValueChars` says only that elide() honours whatever number it is handed, so +// widening the budget widens the assertion with it and the description cap this feeds silently +// grows: at 480 a single value fills the whole aggregated description on its own and every other +// drifted entry is cut. That is the same class of defect this test was repaired to catch. +TEST(ParamDriftPolicy, LongValuesAreElided) { + constexpr std::size_t kBudget = 64; // pinned; see TheRenderedValueBudgetIsSixtyFourCharacters + const json big(std::vector(400, 7)); + ParamDriftPolicy p(ParamDriftConfig{}); + p.evaluate("/n", Params{{"arr", json(std::vector(4, 1))}}, true); + const auto out = p.evaluate("/n", Params{{"arr", big}}, true); + ASSERT_EQ(out.size(), 1u); + const auto got_at = out[0].find(" got="); + ASSERT_NE(got_at, std::string::npos); + const std::string rendered_got = out[0].substr(got_at + 5); + EXPECT_LE(rendered_got.size(), kBudget) + << "the changed value rendered to " << rendered_got.size() << " chars, past the per-value budget"; + EXPECT_LT(rendered_got.size(), big.dump().size()) + << "the 400-element array was rendered in full, so elide() is a no-op on the field that " + "actually carries a long value"; +} + +// The budget itself, pinned once with a literal so every other assertion may name the constant. +// +// Seven of these fit inside AggregatedFault's 480-character description and two fit inside the +// 150 characters one app may spend, which is what makes a multi-app, multi-parameter report +// readable at all. Raising it is not a free tuning choice: the description cap does not grow with +// it, so a wider value budget buys one entry room at the direct expense of every entry behind it. +TEST(ParamDriftPolicy, TheRenderedValueBudgetIsSixtyFourCharacters) { + EXPECT_EQ(ParamDriftPolicy::kMaxRenderedValueChars, 64u) + << "the per-value rendering budget moved. That is a change to what an operator can read in a " + "capped description, not an implementation detail: update this pin deliberately, and check " + "how many entries still fit under AggregatedFault::kMaxDescriptionChars when you do"; +} + +// The gap has a floor as well as microsecond resolution: a budget above a million reads per +// second truncates to zero again, and a zero gap is a reader that never waits. +TEST(ParamDriftPolicy, ReadGapNeverReachesZeroEvenForAnAbsurdBudget) { + EXPECT_EQ(read_gap(1000, 2000000), std::chrono::microseconds(1)); + EXPECT_GT(read_gap(1, 100000).count(), 0); +} + +// The reader spends one gap per round trip, so what the budget actually buys the watched node is +// `period / gap` round trips per tick period - and that is NOT `reads` unless the division comes +// out even. Flooring makes the gap shorter than asked and the rate correspondingly higher: +// `tick_interval_ms: 1` with `max_reads_per_tick: 600`, both accepted configurations, floors to a +// 1 us gap and 1000 round trips against a documented bound of 600. +// +// The whole accepted range is swept rather than one pair, because the overshoot only appears when +// the division is uneven and any single hand-picked pair is overwhelmingly likely to divide evenly. +// Both directions are pinned: the rate must never exceed the budget, and the gap must not be +// padded beyond the one microsecond the rounding can add, or the knob would silently throttle. +TEST(ParamDriftPolicy, ReadGapNeverLetsTheAchievableRateExceedTheBudget) { + const std::vector ticks{1, 2, 3, 5, 7, 10, 100, 125, 250, 333, 1000, 5000}; + const std::vector reads{1, 2, 3, 6, 7, 8, 9, 11, 13, 64, 600, 999, 1024, 7777, 100000}; + for (const int tick_ms : ticks) { + for (const int budget : reads) { + const auto gap = read_gap(tick_ms, budget).count(); + const std::int64_t period_us = static_cast(tick_ms) * 1000; + ASSERT_GT(gap, 0) << "tick_interval_ms=" << tick_ms << " max_reads_per_tick=" << budget; + EXPECT_LE(period_us / gap, budget) + << "tick_interval_ms=" << tick_ms << " with max_reads_per_tick=" << budget << " yields a " << gap + << " us gap, i.e. " << period_us / gap + << " round trips per tick period: the knob an operator lowered to protect a node is being " + "overshot"; + // Tightness, so "hold the bound" cannot be met by simply pacing far slower than asked. One + // slot per read plus at most the single microsecond rounding up can add. + EXPECT_LT(gap * budget, period_us + budget) + << "tick_interval_ms=" << tick_ms << " with max_reads_per_tick=" << budget << " spends " << gap + << " us per round trip, well beyond the slot the budget asks for"; + } + } +} + +// The exact configuration the bound used to break on, spelled out so the regression is named. +TEST(ParamDriftPolicy, ReadGapHoldsTheBoundOnASubMillisecondSlot) { + EXPECT_EQ(read_gap(1, 600), std::chrono::microseconds(2)) + << "a 1000 us tick period over 600 reads is 1.67 us per read; rounding down to 1 us buys the " + "sweep 1000 round trips per tick period against a bound of 600"; +} + +// is_number_integer() spans the whole 64-bit range and get() truncates, so the range check +// has to happen on the wide value. Otherwise 4294967297 arrives as 1, and -4294967295 arrives as +// 1 as well - a negative request passing a positive-only guard, both with no warning. +TEST(ParamDriftPolicy, OutOfRangeBudgetIsRejectedNotTruncated) { + for (const auto & bad : {json(4294967297LL), json(-4294967295LL), json(-1), json(0)}) { + const auto cfg = ParamDriftConfig::from_json(json{{"max_reads_per_tick", bad}}); + EXPECT_EQ(cfg.max_reads_per_tick, 8) << "accepted " << bad.dump(); + EXPECT_FALSE(cfg.warnings.empty()) << "accepted " << bad.dump() << " with no warning"; + } + const auto ok = ParamDriftConfig::from_json(json{{"max_reads_per_tick", 32}}); + EXPECT_EQ(ok.max_reads_per_tick, 32); + EXPECT_TRUE(ok.warnings.empty()); + + // The ceiling itself. An off-by-one here either rejects a legal budget or lets the value above + // it through, and neither shows up anywhere else - every other case in this test is orders of + // magnitude away from the bound. + const auto at_ceiling = + ParamDriftConfig::from_json(json{{"max_reads_per_tick", ParamDriftConfig::kMaxReadsPerTickCeiling}}); + EXPECT_EQ(at_ceiling.max_reads_per_tick, ParamDriftConfig::kMaxReadsPerTickCeiling); + EXPECT_TRUE(at_ceiling.warnings.empty()) << "the ceiling value itself must be accepted"; + + const auto past_ceiling = + ParamDriftConfig::from_json(json{{"max_reads_per_tick", ParamDriftConfig::kMaxReadsPerTickCeiling + 1}}); + EXPECT_EQ(past_ceiling.max_reads_per_tick, 8) << "a budget one past the ceiling was accepted"; + EXPECT_FALSE(past_ceiling.warnings.empty()); +} + +// nlohmann compares an integer against a float by casting the integer to double, which collapses +// values differing beyond 2^53 - exactly the missed drift the equality helper claims to avoid. +// Reachable from an expect pin written with a decimal point against an integer parameter. +TEST(ParamDriftPolicy, IntegerVersusFloatIsComparedExactly) { + EXPECT_FALSE(param_values_equal(json(9007199254740993LL), json(9007199254740992.0))); + EXPECT_TRUE(param_values_equal(json(9007199254740992LL), json(9007199254740992.0))); + EXPECT_TRUE(param_values_equal(json(20), json(20.0))); // the ordinary case must stay equal + EXPECT_FALSE(param_values_equal(json(1), json(1.5))); + EXPECT_FALSE(param_values_equal(json(1), json(std::numeric_limits::quiet_NaN()))); + EXPECT_FALSE(param_values_equal(json(1), json(std::numeric_limits::infinity()))); + + // The MIRRORED branch: a float on the left against an integer on the right swaps the operands + // before calling int_equals_float_exactly, and only the int-left form was covered - so the swap + // could have been the wrong way round in one direction and nothing here would have said so. A + // float-left NaN is the ordinary shape of it: a NaN-valued parameter read from a node, compared + // against an integer pin. + EXPECT_FALSE(param_values_equal(json(std::numeric_limits::quiet_NaN()), json(1))); + EXPECT_FALSE(param_values_equal(json(std::numeric_limits::infinity()), json(1))); + EXPECT_FALSE(param_values_equal(json(1.5), json(1))); + EXPECT_TRUE(param_values_equal(json(20.0), json(20))); + EXPECT_FALSE(param_values_equal(json(9007199254740992.0), json(9007199254740993LL))); + + // A negative pin against an unsigned parameter, which is the ordinary shape: the transport + // stores every positive integer unsigned. Converting the double to uint64_t before checking its + // sign is undefined behaviour, and on x86-64 -1.0 lands on UINT64_MAX - so the one parameter + // value that would then compare EQUAL to it is the largest one representable. + EXPECT_FALSE(param_values_equal(json(std::uint64_t{18446744073709551615ULL}), json(-1.0))); + EXPECT_FALSE(param_values_equal(json(std::uint64_t{5}), json(-1.0))); + EXPECT_TRUE(param_values_equal(json(std::uint64_t{5}), json(5.0))); // the branch still works + + // The 2^63 bounds. A double at or past +2^63 has no int64_t to compare against, so it is + // reported as different even where the two are numerically the same - erring towards raising a + // drift rather than missing one. -2^63 IS representable and must still compare equal. + EXPECT_FALSE(param_values_equal(json(std::uint64_t{9223372036854775808ULL}), json(9223372036854775808.0))); + EXPECT_FALSE(param_values_equal(json(1), json(-18446744073709551616.0))); // -2^64 + EXPECT_TRUE(param_values_equal(json(std::numeric_limits::min()), json(-9223372036854775808.0))); +} + +// Configuration that parses cleanly and can never do anything must say so. Silence is the worst +// outcome here: the detector looks configured and reports nothing, forever. +TEST(ParamDriftConfig, InertCombinationsAreReported) { + const auto nothing_to_read = ParamDriftConfig::from_json(json{{"baseline", false}}); + EXPECT_FALSE(nothing_to_read.warnings.empty()) << "baseline off with no expect reads nothing at all"; + + const auto dead_ignore = ParamDriftConfig::from_json( + json{{"baseline", false}, {"expect", {{"use_sim_time", false}}}, {"ignore", {"*_x"}}}); + EXPECT_FALSE(dead_ignore.warnings.empty()) << "ignore cannot filter anything when baseline is off"; + + // And the ordinary configurations stay quiet. + EXPECT_TRUE(ParamDriftConfig::from_json(json::object()).warnings.empty()); + EXPECT_TRUE(ParamDriftConfig::from_json(json{{"baseline", false}, {"expect", {{"a", 1}}}}).warnings.empty()); +} + +// An ignore pattern that matches every parameter name suppresses the whole self-captured half. +// The detector then looks configured and reports nothing, which at runtime is indistinguishable +// from it working and finding nothing - the exact silent failure it is built to rule out. +TEST(ParamDriftConfig, ACatchAllIgnoreIsReported) { + EXPECT_FALSE(ParamDriftConfig::from_json(json{{"ignore", {"*"}}}).warnings.empty()); + EXPECT_FALSE(ParamDriftConfig::from_json(json{{"ignore", {"**"}}}).warnings.empty()) + << "a glob of nothing but stars matches everything too"; + EXPECT_FALSE(ParamDriftConfig::from_json(json{{"ignore", {"debug_*", "*"}}}).warnings.empty()) + << "one catch-all among several patterns still swallows everything"; + + // Pins still report, so this is a different message - but still not silence. + EXPECT_FALSE(ParamDriftConfig::from_json(json{{"ignore", {"*"}}, {"expect", {{"a", 1}}}}).warnings.empty()); + + // An ordinary pattern stays quiet. + EXPECT_TRUE(ParamDriftConfig::from_json(json{{"ignore", {"debug_*"}}}).warnings.empty()); +} + +// A pin written two ways that flatten to the same parameter name loses one of them, and a pin +// with no value at all disappears. Both used to happen with no diagnostic. +TEST(ParamDriftConfig, LostExpectPinsAreReported) { + const auto collide = ParamDriftConfig::from_json(json::parse(R"({"expect":{"a":{"b":1},"a.b":2}})")); + EXPECT_FALSE(collide.warnings.empty()); + EXPECT_EQ(collide.expect.size(), 1u); + + const auto empty_pin = ParamDriftConfig::from_json(json::parse(R"({"expect":{"qos_overrides":{}}})")); + EXPECT_FALSE(empty_pin.warnings.empty()); + EXPECT_TRUE(empty_pin.expect.empty()); +} + +// Parameter NAMES come verbatim from the watched node and are never validated, so they carry the +// same two hazards values do. An invalid UTF-8 byte in the name, or a valid multi-byte name split +// by the aggregated description's byte-offset cut, makes the gateway's JSON writer throw - which +// answers 500 on GET /faults and takes every OTHER fault down with it. +TEST(ParamDriftPolicy, ParameterNamesAreSanitisedToo) { + ParamDriftPolicy p(ParamDriftConfig{}); + const std::string bad_name("na\xff\xfeme", 6); + p.evaluate("/n", Params{{bad_name, json(1.0)}}, true); + const auto out = p.evaluate("/n", Params{{bad_name, json(2.0)}}, true); + ASSERT_EQ(out.size(), 1u); + for (const char c : out[0]) { + EXPECT_LT(static_cast(c), 0x80u) << "non-ASCII byte from a parameter NAME"; + } + EXPECT_NO_THROW(json(out[0]).dump()); +} + +// A parameter a node DECLARED and never SET must not read as `null` in a fault description. +// +// The transport answers such a parameter with a JSON null, and a NaN double DUMPS as `null` too, so +// one description said the same thing for "the node never set this parameter" and "the value is +// NaN" - two different faults with two different remedies, and nothing in the text to tell them +// apart. Both are reachable from an `expect` pin, so both really do end up in front of an operator. +// +// The equality half is asserted here as well, and not assumed: the marker is carried through the +// same map a real value is, so whatever it is has to keep the pin's verdict intact - unequal to the +// pinned value (or the raise this rendering describes would silently disappear) and equal to +// itself (or a parameter that is unset every sweep would drift against its own baseline for ever). +TEST(ParamDriftPolicy, AnUnsetParameterIsRenderedDistinctlyFromANaNValue) { + ParamDriftConfig cfg; + cfg.baseline_capture = false; + cfg.expect = {{"threshold", json(1.0)}}; + ParamDriftPolicy p{cfg}; + + const auto unset = p.evaluate("/unset", Params{{"threshold", unset_param_value()}}, /*capture_allowed=*/false); + ASSERT_EQ(unset.size(), 1u) << "an expect pin on a parameter the node never set must still raise: a node with no " + "value at all does not satisfy a pin that says the value must be 1.0"; + const auto nan_drift = p.evaluate("/nan", Params{{"threshold", json(std::numeric_limits::quiet_NaN())}}, + /*capture_allowed=*/false); + ASSERT_EQ(nan_drift.size(), 1u) << "a NaN value against a pin of 1.0 is a drift too"; + + const std::string unset_got = rendered_got(unset[0]); + const std::string nan_got = rendered_got(nan_drift[0]); + ASSERT_FALSE(unset_got.empty()) << "no got= field in '" << unset[0] << "'"; + ASSERT_FALSE(nan_got.empty()) << "no got= field in '" << nan_drift[0] << "'"; + EXPECT_NE(unset_got, nan_got) << "an unset parameter and a NaN value both rendered as '" << unset_got + << "', so an operator cannot tell which fault they are looking at"; + EXPECT_EQ(unset_got, std::string(kUnsetParameterRendering)) << "an unset parameter rendered as '" << unset_got << "'"; + EXPECT_NE(unset_got, "null") << "the rendering of an unset parameter is exactly the text it must stop being"; + + // The description is cut at a BYTE offset by the aggregated fault, so the marker's rendering + // carries the same ASCII-only obligation every other rendered value does. + for (const char c : unset[0]) { + EXPECT_LT(static_cast(c), 0x80u) << "non-ASCII byte in the unset rendering"; + } + + // The verdict the rendering describes, pinned directly. + EXPECT_FALSE(param_values_equal(unset_param_value(), json(1.0))) << "an unset parameter would satisfy a pin"; + EXPECT_FALSE(param_values_equal(json(1.0), unset_param_value())) << "and the mirrored operand order"; + EXPECT_TRUE(param_values_equal(unset_param_value(), unset_param_value())) + << "a parameter that is unset on every sweep would drift against its own captured baseline for ever"; + + // A JSON OBJECT is the marker's whole safety argument: it is a shape no ROS parameter value and + // no `expect` pin can take, so it cannot collide with something a node or an operator wrote. A + // string sentinel could - a node is free to hold the string below. + EXPECT_TRUE(unset_param_value().is_object()); + EXPECT_FALSE(is_unset_param(json(kUnsetParameterRendering))) + << "a node holding this exact string would be reported as unset"; + EXPECT_FALSE(is_unset_param(json(nullptr))); + EXPECT_FALSE(is_unset_param(json(std::numeric_limits::quiet_NaN()))); +} + +// Which transport replies become the marker. Both signals are load-bearing and neither subsumes the +// other: the TYPE is what a live rclcpp node sends (`not_set` alongside a null value), and the null +// VALUE is the backstop for a transport that omits the type, since null is not a legal encoding of +// any ROS parameter value and so must never reach a description as one. A NaN double is caught by +// neither - it is a float that merely DUMPS as `null` - which is the distinction being kept. +TEST(ParamDriftPolicy, OnlyADeclaredButUnsetTransportEntryBecomesTheMarker) { + EXPECT_TRUE(is_unset_param(param_entry_value(json{{"name", "a"}, {"type", "not_set"}, {"value", nullptr}}))) + << "the shape a live rclcpp node produces for a parameter it declared and never set"; + EXPECT_TRUE(is_unset_param(param_entry_value(json{{"name", "a"}, {"value", nullptr}}))) + << "a null value with no type is still not a legal ROS parameter value"; + + EXPECT_FALSE(is_unset_param(param_entry_value(json{{"name", "a"}, {"type", "double"}, {"value", 1.0}}))); + EXPECT_EQ(param_entry_value(json{{"name", "a"}, {"type", "double"}, {"value", 1.0}}), json(1.0)); + EXPECT_EQ(param_entry_value(json{{"name", "a"}, {"type", "string"}, {"value", "not_set"}}), json("not_set")) + << "a STRING parameter whose value is the word 'not_set' is an ordinary value, not an unset parameter"; + EXPECT_FALSE(is_unset_param( + param_entry_value(json{{"name", "a"}, {"type", "double"}, {"value", std::numeric_limits::quiet_NaN()}}))) + << "a NaN double is a value the node HAS; folding it into the marker would recreate the ambiguity"; + EXPECT_FALSE(is_unset_param(param_entry_value(json{{"name", "a"}, {"type", "bool"}, {"value", false}}))) + << "false is a value, not an absence"; +} + +// Head-only truncation renders two different values identically whenever they share a long prefix - +// the common case for robot_description and for nav2 footprint arrays where only a late element +// differs. The fault would fire correctly and then prove nothing about what changed. +TEST(ParamDriftPolicy, ElisionKeepsBothEndsSoValuesStayDistinguishable) { + std::vector before(40, 0.5); + std::vector after = before; + after.back() = 9.5; // only the LAST element differs + ParamDriftPolicy p(ParamDriftConfig{}); + p.evaluate("/costmap", Params{{"footprint", json(before)}}, true); + const auto out = p.evaluate("/costmap", Params{{"footprint", json(after)}}, true); + ASSERT_EQ(out.size(), 1u); + const auto baseline_at = out[0].find("baseline="); + const auto got_at = out[0].find(" got="); + ASSERT_NE(baseline_at, std::string::npos); + ASSERT_NE(got_at, std::string::npos); + const std::string rendered_baseline = out[0].substr(baseline_at + 9, got_at - baseline_at - 9); + const std::string rendered_got = out[0].substr(got_at + 5); + EXPECT_NE(rendered_baseline, rendered_got) + << "both values rendered as '" << rendered_baseline << "', so the fault shows no difference"; +}