From 22e910493f2eddbaf43251e896e45f9191c08f9e Mon Sep 17 00:00:00 2001 From: Richard Evans Date: Tue, 28 Jul 2026 11:52:50 -0400 Subject: [PATCH] systemvm: fix subprocess deadlock in router health check scripts The health check scripts run shell commands with Popen(stdout=PIPE) and call wait() before reading the pipe. When the command output exceeds the OS pipe capacity (64 KiB on Linux), the child blocks writing to the full pipe and the parent blocks in wait() forever. This is the deadlock warned about in the Python subprocess documentation. On routers with large iptables rule sets this makes iptables_check.py hang on every run of the advanced health check. A new stuck process chain accumulates each advanced-check interval (10 minutes by default) until the router exhausts memory and swap, keepalived's track script starves, and the router enters FAULT and reboots. On a 512 MB router this produced a reboot roughly every 9 hours, and redundant pairs failed together because their timers are synchronized at creation. Drain the pipe with communicate() and check returncode instead of calling wait() with an unread pipe. The same latent pattern is also fixed in memory_usage_check.py, cpu_usage_check.py and gateways_check.py. Fixes: #13735 --- systemvm/debian/root/health_checks/cpu_usage_check.py | 5 +++-- systemvm/debian/root/health_checks/gateways_check.py | 3 ++- systemvm/debian/root/health_checks/iptables_check.py | 5 +++-- systemvm/debian/root/health_checks/memory_usage_check.py | 5 +++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/systemvm/debian/root/health_checks/cpu_usage_check.py b/systemvm/debian/root/health_checks/cpu_usage_check.py index 270ea2086107..c753b86fbbce 100644 --- a/systemvm/debian/root/health_checks/cpu_usage_check.py +++ b/systemvm/debian/root/health_checks/cpu_usage_check.py @@ -37,8 +37,9 @@ def main(): "'{ split($1, vs, \",\"); idle=vs[length(vs)]; " \ "sub(\"%\", \"\", idle); printf \"%.2f\", 100 - idle }'" pout = Popen(cmd, shell=True, stdout=PIPE) - if pout.wait() == 0: - currentUsage = float(pout.communicate()[0].decode().strip()) + stdout, _ = pout.communicate() + if pout.returncode == 0: + currentUsage = float(stdout.decode().strip()) if currentUsage > maxCpuUsage: print("CPU Usage " + str(currentUsage) + "% has crossed threshold of " + str(maxCpuUsage) + "%") diff --git a/systemvm/debian/root/health_checks/gateways_check.py b/systemvm/debian/root/health_checks/gateways_check.py index e3b661b7498a..19841c97b1de 100644 --- a/systemvm/debian/root/health_checks/gateways_check.py +++ b/systemvm/debian/root/health_checks/gateways_check.py @@ -36,7 +36,8 @@ def main(): for i in range(5): pingCmd = "ping " + gw + " -c 5 -w 10" pout = Popen(pingCmd, shell=True, stdout=PIPE) - if pout.wait() == 0: + pout.communicate() + if pout.returncode == 0: reachableGw = True break diff --git a/systemvm/debian/root/health_checks/iptables_check.py b/systemvm/debian/root/health_checks/iptables_check.py index 27e06f8352b6..ebf7b4bd8257 100644 --- a/systemvm/debian/root/health_checks/iptables_check.py +++ b/systemvm/debian/root/health_checks/iptables_check.py @@ -41,13 +41,14 @@ def main(): fetchIpTableEntriesCmd = "iptables-save | grep " + destIp pout = Popen(fetchIpTableEntriesCmd, shell=True, stdout=PIPE) - if pout.wait() != 0: + stdout, _ = pout.communicate() + if pout.returncode != 0: failedCheck = True failureMessage = failureMessage + "Unable to execute iptables-save command " \ "for fetching rules by " + fetchIpTableEntriesCmd + "\n" continue - ipTablesMatchingEntries = pout.communicate()[0].decode().strip().split('\n') + ipTablesMatchingEntries = stdout.decode().strip().split('\n') for pfEntryListExpected in entriesExpected: foundPfEntryList = False for ipTableEntry in ipTablesMatchingEntries: diff --git a/systemvm/debian/root/health_checks/memory_usage_check.py b/systemvm/debian/root/health_checks/memory_usage_check.py index eba0d5e49dde..f468160648f6 100644 --- a/systemvm/debian/root/health_checks/memory_usage_check.py +++ b/systemvm/debian/root/health_checks/memory_usage_check.py @@ -35,9 +35,10 @@ def main(): maxMemoryUsage = float(data["maxMemoryUsage"]) cmd = "free | awk 'FNR == 2 { print $3 * 100 / $2 }'" pout = Popen(cmd, shell=True, stdout=PIPE) + stdout, _ = pout.communicate() - if pout.wait() == 0: - currentUsage = float(pout.communicate()[0].decode().strip()) + if pout.returncode == 0: + currentUsage = float(stdout.decode().strip()) if currentUsage > maxMemoryUsage: print("Memory Usage " + str(currentUsage) + "% has crossed threshold of " + str(maxMemoryUsage) + "%")