Additional diagnostic events

members/nullnet-client/src/commands/mod.rs

@@ -124,11 +124,14 @@ impl RtNetLinkHandle {
     }
 }
 
-pub(crate) async fn cleanup_network(rtnetlink_handle: &RtNetLinkHandle) {
+/// Returns the MSS-clamp install error, if there was one. It can't be reported
+/// from here — this runs before the control connection exists — so the caller
+/// emits the event once it does.
+pub(crate) async fn cleanup_network(rtnetlink_handle: &RtNetLinkHandle) -> Option<String> {
     dnat::init();
     nfqueue::init();
     egress::init();
-    install_mss_clamp();
+    let mss_error = install_mss_clamp();
     vxlan_cleanup_network();
     vlan_cleanup_network(rtnetlink_handle).await;
     // State a killed process never got to tear down, and that no `VxlanTeardown`
@@ -139,6 +142,7 @@ pub(crate) async fn cleanup_network(rtnetlink_handle: &RtNetLinkHandle) {
     purge_stale_xfrm();
     crate::host_mappings::purge_stale_mappings();
     egress::purge_stale_steers();
+    mss_error
 }
 
 /// SPI range `vxlan-setup.sh` can install: it offsets the net id by 1000 to
@@ -346,7 +350,7 @@ src 10.20.30.1/32 dst 10.20.30.2/32
 /// removed first — `-C` only matches a rule verbatim, so without that an
 /// upgrade would leave two clamps installed and the older one would win by
 /// position.
-fn install_mss_clamp() {
+fn install_mss_clamp() -> Option<String> {
     prune_superseded_mss_rules();
     // Must match OVERLAY_MTU in vxlan_scripts/vxlan-setup.sh (1080) minus the
     // 40-byte IP+TCP headers. The previous 1400 came from a theoretical
@@ -371,16 +375,23 @@ fn install_mss_clamp() {
     let mut check = vec!["iptables", "-t", "mangle", "-C", "FORWARD"];
     check.extend_from_slice(&rule);
     if sudo(&check).map(|s| s.success()).unwrap_or(false) {
-        return;
+        return None;
     }
     let mut add = vec!["iptables", "-t", "mangle", "-A", "FORWARD"];
     add.extend_from_slice(&rule);
     match sudo(&add) {
         Ok(s) if s.success() => {
-            println!("[mss] clamp installed on mangle/FORWARD: --set-mss {MSS}")
+            println!("[mss] clamp installed on mangle/FORWARD: --set-mss {MSS}");
+            None
+        }
+        Ok(s) => {
+            eprintln!("[mss] clamp install exited {s}");
+            Some(format!("iptables exited {s}"))
+        }
+        Err(e) => {
+            eprintln!("[mss] clamp install failed: {e}");
+            Some(e.to_string())
         }
-        Ok(s) => eprintln!("[mss] clamp install exited {s}"),
-        Err(e) => eprintln!("[mss] clamp install failed: {e}"),
     }
 }
 

members/nullnet-client/src/control_channel.rs

@@ -13,8 +13,8 @@ use nullnet_grpc_lib::NullnetGrpcInterface;
 use nullnet_grpc_lib::nullnet_grpc::{
     AgentContainerResumeFailed, AgentContainerSuspendFailed, AgentControlChannelAckFailed,
     AgentControlChannelClosed, AgentControlChannelEstablished, AgentDnatInstallFailed,
-    AgentDnatRemovalFailed, AgentGatewayForwardInstallFailed, AgentHostMappingFailed,
-    AgentVlanSetupCompleted, AgentVlanSetupFailed, AgentVlanTeardownFailed,
+    AgentDnatRemovalFailed, AgentEgressSteerInstallFailed, AgentGatewayForwardInstallFailed,
+    AgentHostMappingFailed, AgentVlanSetupCompleted, AgentVlanSetupFailed, AgentVlanTeardownFailed,
     AgentVxlanSetupCompleted, AgentVxlanSetupFailed, AgentVxlanTeardownFailed,
 };
 use nullnet_grpc_lib::nullnet_grpc::{
@@ -174,10 +174,11 @@ pub(crate) async fn control_channel(
                 // re-enter the NFQUEUE as NEW — newly-denied ones die there.
                 let verdicts = policy_verdicts.clone();
                 let cache = bridge_cache.clone();
+                let grpc = server.clone();
                 tokio::spawn(async move {
                     println!("[egress-policy] policy changed on server; re-verdicting flows");
                     verdicts.clear();
-                    flush_container_conntrack(cache.ips()).await;
+                    flush_container_conntrack(&grpc, cache.ips()).await;
                 });
             }
             None => {}
@@ -537,13 +538,34 @@ async fn handle_vxlan_setup(
                             cip,
                         );
                     }
+                } else {
+                    // Mirrors the DNAT path below: without steering the held
+                    // packet is never woken and drops at ACTIVE_TIMEOUT.
+                    fire_event(
+                        &grpc,
+                        AgentEventKind::EgressSteerInstallFailed(AgentEgressSteerInstallFailed {
+                            vxlan_id,
+                            docker_container: message.docker_container.clone(),
+                            error_message: "steer rules failed to install".to_string(),
+                        }),
+                    );
                 }
             }
             _ => {
                 eprintln!(
                     "[vxlan_setup] egress steer missing gateway/container_ip \
                      (gw={proxy_gw:?}, cip={container_ip:?}); steering not installed"
                 );
+                fire_event(
+                    &grpc,
+                    AgentEventKind::EgressSteerInstallFailed(AgentEgressSteerInstallFailed {
+                        vxlan_id,
+                        docker_container: message.docker_container.clone(),
+                        error_message: format!(
+                            "missing gateway/container_ip (gw={proxy_gw:?}, cip={container_ip:?})"
+                        ),
+                    }),
+                );
             }
         }
     } else if egress_intercept {

members/nullnet-client/src/egress_policy.rs

@@ -7,6 +7,10 @@
 //! so live flows re-enter the queue as NEW and get re-verdicted — flows the
 //! new policy denies die on their next packet.
 
+use nullnet_grpc_lib::NullnetGrpcInterface;
+use nullnet_grpc_lib::nullnet_grpc::{
+    AgentConntrackFlushFailed, AgentEvent, agent_event::Event as AgentEventKind,
+};
 use std::collections::HashMap;
 use std::net::Ipv4Addr;
 use std::sync::Mutex;
@@ -56,22 +60,40 @@ impl PolicyVerdicts {
 /// Delete the conntrack entries originating from each container bridge IP so
 /// every live flow re-enters the NFQUEUE as NEW and is re-verdicted. Exit
 /// code 1 just means "no entries matched" — only real failures are logged.
-pub async fn flush_container_conntrack(ips: Vec<Ipv4Addr>) {
+pub async fn flush_container_conntrack(grpc: &NullnetGrpcInterface, ips: Vec<Ipv4Addr>) {
     for ip in ips {
         let out = tokio::process::Command::new("conntrack")
             .args(["-D", "-s", &ip.to_string()])
             .output()
             .await;
-        match out {
-            Ok(o) if o.status.code() == Some(0) || o.status.code() == Some(1) => {}
-            Ok(o) => eprintln!(
-                "[egress-policy] conntrack -D -s {ip} exited {}: {}",
-                o.status,
-                String::from_utf8_lossy(&o.stderr).trim()
-            ),
+        // A failed flush leaves flows the new policy denies running until they
+        // close on their own, so the policy change is only partly in force.
+        let error_message = match out {
+            Ok(o) if o.status.code() == Some(0) || o.status.code() == Some(1) => continue,
+            Ok(o) => {
+                let stderr = String::from_utf8_lossy(&o.stderr).trim().to_string();
+                eprintln!(
+                    "[egress-policy] conntrack -D -s {ip} exited {}: {stderr}",
+                    o.status
+                );
+                format!("conntrack exited {}: {stderr}", o.status)
+            }
             Err(e) => {
                 eprintln!("[egress-policy] conntrack flush {ip}: {e} (is conntrack installed?)");
+                format!("{e} (is conntrack installed?)")
             }
-        }
+        };
+        let grpc = grpc.clone();
+        let event = AgentEvent {
+            event: Some(AgentEventKind::ConntrackFlushFailed(
+                AgentConntrackFlushFailed {
+                    ip: ip.to_string(),
+                    error_message,
+                },
+            )),
+        };
+        tokio::spawn(async move {
+            let _ = grpc.report_event(event).await;
+        });
     }
 }

members/nullnet-client/src/main.rs

@@ -16,8 +16,9 @@ use crate::triggers::TriggersState;
 use clap::Parser;
 use nullnet_grpc_lib::NullnetGrpcInterface;
 use nullnet_grpc_lib::nullnet_grpc::{
-    AgentEvent, AgentServicesListUpdateFailed, AgentServicesListUpdated, Container, Listener, Net,
-    NetType, ServiceReport, agent_event::Event as AgentEventKind,
+    AgentEvent, AgentFirewallRulesLoadFailed, AgentMssClampInstallFailed,
+    AgentServicesListUpdateFailed, AgentServicesListUpdated, Container, Listener, Net, NetType,
+    ServiceReport, agent_event::Event as AgentEventKind,
 };
 use nullnet_liberror::{Error, ErrorHandler, Location, location};
 use std::collections::HashMap;
@@ -77,7 +78,7 @@ async fn main() -> Result<(), Error> {
     let rtnetlink_handle = RtNetLinkHandle::new()?;
 
     // cleanup existing VLANs and VXLANs material
-    cleanup_network(&rtnetlink_handle).await;
+    let mss_error = cleanup_network(&rtnetlink_handle).await;
 
     // maps of all the peers
     let peers = Arc::new(RwLock::new(Peers::default()));
@@ -88,6 +89,22 @@ async fn main() -> Result<(), Error> {
     let grpc_server2 = grpc_server.clone();
     let grpc_server3 = grpc_server.clone();
 
+    // Deferred from `cleanup_network`, which runs before this connection exists.
+    // Without the clamp, oversized segments are silently black-holed once they
+    // enter a tunnel, which is near-impossible to trace from the symptom.
+    if let Some(error_message) = mss_error {
+        let grpc = grpc_server.clone();
+        tokio::spawn(async move {
+            let _ = grpc
+                .report_event(AgentEvent {
+                    event: Some(AgentEventKind::MssClampInstallFailed(
+                        AgentMssClampInstallFailed { error_message },
+                    )),
+                })
+                .await;
+        });
+    }
+
     let net_type = grpc_server.network_type().await.handle_err(location!())?;
 
     if net_type.net() == Net::Vlan {
@@ -108,6 +125,22 @@ async fn main() -> Result<(), Error> {
         }
         Err(e) => {
             eprintln!("Failed to enable eBPF firewall: {e:?}");
+            // Awaited, not fire-and-forget: the exit below would kill a spawned
+            // task before it ever reached the server. Bounded, because
+            // `report_event` has no request timeout of its own and a hung one
+            // would keep us from exiting at all.
+            let _ = tokio::time::timeout(
+                Duration::from_secs(5),
+                grpc_server.report_event(AgentEvent {
+                    event: Some(AgentEventKind::FirewallRulesLoadFailed(
+                        AgentFirewallRulesLoadFailed {
+                            path: "ebpf host firewall".to_string(),
+                            error_message: format!("{e:?}"),
+                        },
+                    )),
+                }),
+            )
+            .await;
             process::exit(1);
         }
     };