Skip to content

fix(daemon): contain zombie generations from abandoned requests, name mute endpoint holders #1920

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
OhOkThisIsFine wants to merge 1 commit into
base: main
Choose a base branch
from
+434 −14
Open

fix(daemon): contain zombie generations from abandoned requests, name mute endpoint holders #1920

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 38 additions & 9 deletions src/daemon/bootstrap.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,11 +359,20 @@ static cbm_daemon_bootstrap_status_t bootstrap_finish_probe(

static cbm_daemon_bootstrap_probe_status_t bootstrap_probe(
const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops,
cbm_daemon_runtime_client_t **client_out, cbm_daemon_runtime_connect_result_t *connect_result) {
cbm_daemon_runtime_client_t **client_out, cbm_daemon_runtime_connect_result_t *connect_result,
uint64_t *muted_holder_pid_io) {
memset(connect_result, 0, sizeof(*connect_result));
*client_out = NULL;
return ops->probe(ops->context, config->endpoint, config->identity, config->connect_timeout_ms,
client_out, connect_result);
cbm_daemon_bootstrap_probe_status_t status =
ops->probe(ops->context, config->endpoint, config->identity, config->connect_timeout_ms,
client_out, connect_result);
/* Sticky across probes: a later fast-path probe never attempts a connect
* and reports no holder, but a mute holder seen once must survive into
* the final timeout diagnostic. */
if (muted_holder_pid_io && connect_result->muted_endpoint_holder_pid != 0) {
*muted_holder_pid_io = connect_result->muted_endpoint_holder_pid;
}
return status;
}

static bool bootstrap_probe_is_finishable(cbm_daemon_bootstrap_probe_status_t probe) {
Expand Down Expand Up @@ -432,8 +441,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(

cbm_daemon_runtime_client_t *client = NULL;
cbm_daemon_runtime_connect_result_t connect_result;
uint64_t muted_holder_pid = 0;
cbm_daemon_bootstrap_probe_status_t probe =
bootstrap_probe(config, ops, &client, &connect_result);
bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
if (bootstrap_probe_is_finishable(probe)) {
cbm_daemon_bootstrap_status_t status =
bootstrap_finish_probe(probe, client, &connect_result, ops, result_out);
Expand Down Expand Up @@ -461,7 +471,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
* replacements from being launched. */
generation_observed = true;
bootstrap_pause(deadline);
probe = bootstrap_probe(config, ops, &client, &connect_result);
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
continue;
}

Expand All @@ -473,7 +483,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
}
if (lock_status == 0) {
bootstrap_pause(deadline);
probe = bootstrap_probe(config, ops, &client, &connect_result);
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
continue;
}
if (lock_status != 1 || !startup_lock) {
Expand All @@ -482,7 +492,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
}

lock_acquired = true;
probe = bootstrap_probe(config, ops, &client, &connect_result);
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
if (probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL) {
generation_observed = true;
Expand Down Expand Up @@ -516,7 +526,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
* bootstrap against a daemon that is trying to cleanly stand down. */
do {
bootstrap_pause(deadline);
probe = bootstrap_probe(config, ops, &client, &connect_result);
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
if (probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL) {
generation_observed = true;
Expand Down Expand Up @@ -545,7 +555,16 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
}

result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED;
if (generation_observed) {
if (muted_holder_pid != 0) {
/* The one diagnostic the 2026-08-29 zombie recovery had to assemble by
* hand from process, pipe, and log correlation: name the pid that
* holds the endpoint without answering, and say what to do with it. */
(void)snprintf(result_out->message, sizeof(result_out->message),
"CBM daemon endpoint is held by pid %llu but that process answered no "
"rendezvous within %u ms; the daemon runtime is likely dead — stop that "
"process, then retry",
(unsigned long long)muted_holder_pid, config->startup_timeout_ms);
} else if (generation_observed) {
(void)snprintf(result_out->message, sizeof(result_out->message),
"CBM daemon is active or starting but could not accept this client "
"within %u ms",
Expand Down Expand Up @@ -578,6 +597,16 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect
* existing generation answered. Never reinterpret them as absence. */
return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
}
if (connect_result->status == CBM_DAEMON_RUNTIME_CONNECT_ERROR &&
connect_result->muted_endpoint_holder_pid != 0) {
/* The transport connected to a live process that answered nothing.
* That endpoint is OWNED, whatever the advisory locks read right now
* (a wedged generation can hold pipes while its lock files churn).
* Classifying this as absence made the 2026-08-29 zombie invisible:
* the starter spawned doomed competitors for 30 s and then reported a
* bare timeout with no holder named. */
return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
}
if (lifetime_status == 1) {
return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
}
Expand Down
15 changes: 14 additions & 1 deletion src/daemon/frontend.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,19 @@ enum {
* cancelled and the frontend exits cleanly. */
FRONTEND_EOF_DRAIN_MS = 15000,
FRONTEND_MAINTENANCE_POLL_MS = 10,
/* Cadence for the quiet no-maintenance steady state. Every presence probe
* try-acquires the maintenance marker lock, and on Windows that
* revalidates the whole ancestor directory chain (a CreateFileW +
* identity check per path component). At the old 10 ms cadence this
* burned ~20% of a core for the life of every MCP session — 7762
* CPU-seconds on one 9.5 h session in the 2026-08-29 incident. The
* budget: an activation drains participants within
* CLI_ACTIVATION_DRAIN_TIMEOUT_MS (15 s) and the monitor's own exit grace
* is FRONTEND_MAINTENANCE_GRACE_MS (3 s), so a 250 ms detection latency
* spends under 2% of that window while cutting the probe cost 25-fold.
* The 10 ms cadence still paces the monitor's post-detection grace loop,
* which does no lock probing. */
FRONTEND_MAINTENANCE_IDLE_POLL_MS = 250,
/* The owner thread may be draining a supervised process tree. Preserve the
* supervisor's complete graceful + forced-settle window before the monitor
* fail-stops the process, plus scheduling/teardown margin. */
Expand Down Expand Up @@ -162,7 +175,7 @@ static void *frontend_maintenance_monitor_worker(void *opaque) {
cbm_version_cohort_maintenance_presence_t presence =
frontend_observe_maintenance(monitor->manager, true);
if (presence == CBM_VERSION_COHORT_MAINTENANCE_ABSENT) {
cbm_usleep(FRONTEND_MAINTENANCE_POLL_MS * 1000U);
cbm_usleep(FRONTEND_MAINTENANCE_IDLE_POLL_MS * 1000U);
continue;
}

Expand Down
98 changes: 95 additions & 3 deletions src/daemon/runtime.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,19 @@ static atomic_bool runtime_force_peer_image_mismatch_seam;
void cbm_daemon_runtime_force_peer_image_mismatch_for_testing(bool force) {
atomic_store(&runtime_force_peer_image_mismatch_seam, force);
}
/* The abandoned-request containment path ends in process termination, which an
* in-process harness cannot observe. The timeout override makes the ceiling
* reachable in test time; the hook replaces termination with a recordable
* callback. Both stay inert (zero/NULL) outside tests. */
static _Atomic uint32_t runtime_abandoned_request_join_timeout_seam;
void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms) {
atomic_store(&runtime_abandoned_request_join_timeout_seam, timeout_ms);
}
static _Atomic(cbm_daemon_runtime_containment_hook_t) runtime_containment_hook_seam;
void cbm_daemon_runtime_set_containment_hook_for_testing(
cbm_daemon_runtime_containment_hook_t hook) {
atomic_store(&runtime_containment_hook_seam, hook);
}
#endif

#ifdef _WIN32
Expand Down Expand Up @@ -72,6 +85,16 @@ enum {
* a local cooperative peer is already blocked in read and consumes the
* rejection within milliseconds. */
RUNTIME_REJECT_DRAIN_TIMEOUT_MS = 250,
/* Ceiling on how long a disconnecting worker waits for its in-flight
* application request to observe cancellation. session_cancel has already
* run by the time this wait starts, so a compliant handler returns within
* milliseconds; the wait is sized for a handler that only polls its cancel
* flag between long pipeline stages. A handler that ignores cancellation
* past this ceiling turned the daemon into a permanent zombie once
* (2026-08-29): the unbounded join blocked the disconnect path, admission
* wedged behind it, and the dead generation held the endpoint pipes and
* the UI port for nine hours with nothing logged. */
RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS = 30000,
RUNTIME_PATH_CAP = 4096,

RENDEZVOUS_REQUEST_ABI_OFFSET = 0,
Expand Down Expand Up @@ -1313,13 +1336,62 @@ static void *runtime_application_worker(void *opaque) {
return NULL;
}

static _Noreturn void runtime_cleanup_fail_stop(const char *component);

static void runtime_contain_unresponsive_application(cbm_daemon_runtime_worker_t *worker) {
char peer_pid[32];
char token[32];
(void)snprintf(peer_pid, sizeof(peer_pid), "%llu", (unsigned long long)worker->peer_process_id);
(void)snprintf(token, sizeof(token), "%llu",
(unsigned long long)worker->application_request_token);
cbm_log_error("daemon.application_request_unresponsive", "peer_pid", peer_pid, "request_token",
token);
#ifdef CBM_ENABLE_TEST_SEAMS
cbm_daemon_runtime_containment_hook_t hook = atomic_load(&runtime_containment_hook_seam);
if (hook) {
hook("application_request_join");
return;
}
#endif
runtime_cleanup_fail_stop("application_request_join");
}

static bool runtime_worker_reap_application(cbm_daemon_runtime_worker_t *worker, bool wait) {
if (!worker->application_thread_started) {
return true;
}
if (!wait && !atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
return false;
}
if (!atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
/* The peer is gone and session_cancel already ran, so the handler owes
* a prompt return; only a handler that ignores cancellation reaches
* the ceiling. An unbounded join here once turned one wedged request
* into a whole-daemon zombie (2026-08-29): the disconnect path
* blocked, the worker slot never released, and the dead generation
* held the endpoint pipes and UI port for nine hours with nothing
* logged while every new client timed out bare. Containment
* terminates the process instead — the kernel releases every native
* claim and the next client starts a fresh generation. */
uint32_t timeout_ms = RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS;
#ifdef CBM_ENABLE_TEST_SEAMS
uint32_t seam_timeout = atomic_load(&runtime_abandoned_request_join_timeout_seam);
if (seam_timeout != 0) {
timeout_ms = seam_timeout;
}
#endif
uint64_t deadline = runtime_deadline_after(timeout_ms);
while (!atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
if (cbm_now_ms() >= deadline) {
/* Terminal in production. A test containment hook returns, and
* the join then waits for the harness to release the handler
* so teardown stays leak-free under the sanitizers. */
runtime_contain_unresponsive_application(worker);
deadline = UINT64_MAX;
}
runtime_wait_tick(deadline);
}
}
if (cbm_thread_join(&worker->application_thread) != 0) {
return false;
}
Expand Down Expand Up @@ -2569,8 +2641,11 @@ static bool runtime_control_request_send(const cbm_daemon_ipc_endpoint_t *endpoi
const cbm_daemon_build_identity_t *identity,
cbm_daemon_runtime_operation_t operation,
uint32_t timeout_ms, uint32_t response_size,
uint8_t **payload_out) {
uint8_t **payload_out, uint64_t *muted_holder_pid_out) {
*payload_out = NULL;
if (muted_holder_pid_out) {
*muted_holder_pid_out = 0;
}
if (!endpoint || !identity || !identity->build_fingerprint ||
timeout_ms == CBM_DAEMON_IPC_WAIT_FOREVER) {
return false;
Expand All @@ -2590,6 +2665,15 @@ static bool runtime_control_request_send(const cbm_daemon_ipc_endpoint_t *endpoi
int received = sent ? cbm_daemon_ipc_receive_frame_bounded(connection, timeout_ms,
response_size, &frame, &payload)
: 0;
if (muted_holder_pid_out && sent &&
(received != 1 || (frame.type == CBM_DAEMON_FRAME_RESPONSE && frame.flags != operation))) {
/* Connected but not served: either total silence (dead runtime), or a
* wrong-operation reject frame (a wedged generation whose accept path
* still answers inline while every worker slot is stuck). Both are
* the zombie class — surface the holder's kernel-reported pid so
* `daemon status` can name it instead of reporting "not running". */
*muted_holder_pid_out = cbm_daemon_ipc_connection_peer_pid(connection);
}
bool valid = received == 1 && frame.type == CBM_DAEMON_FRAME_RESPONSE &&
frame.flags == operation && frame.length == response_size && payload &&
payload[0] == 1U;
Expand Down Expand Up @@ -2617,7 +2701,8 @@ bool cbm_daemon_runtime_request_status(const cbm_daemon_ipc_endpoint_t *endpoint
memset(status_out, 0, sizeof(*status_out));
uint8_t *payload = NULL;
if (!runtime_control_request_send(endpoint, identity, CBM_DAEMON_RUNTIME_OP_STATUS, timeout_ms,
CBM_DAEMON_STATUS_RESPONSE_SIZE, &payload)) {
CBM_DAEMON_STATUS_RESPONSE_SIZE, &payload,
&status_out->muted_endpoint_holder_pid)) {
return false;
}
status_out->permanent = (payload[1] & 0x01U) != 0U;
Expand Down Expand Up @@ -2648,7 +2733,7 @@ bool cbm_daemon_runtime_request_stop(const cbm_daemon_ipc_endpoint_t *endpoint,
memset(result_out, 0, sizeof(*result_out));
uint8_t *payload = NULL;
if (!runtime_control_request_send(endpoint, identity, CBM_DAEMON_RUNTIME_OP_STOP, timeout_ms,
CBM_DAEMON_STOP_RESPONSE_SIZE, &payload)) {
CBM_DAEMON_STOP_RESPONSE_SIZE, &payload, NULL)) {
return false;
}
result_out->accepted = (payload[1] & 0x01U) != 0U;
Expand Down Expand Up @@ -2691,6 +2776,13 @@ cbm_daemon_runtime_client_t *cbm_daemon_runtime_client_connect(
frame.length == CBM_DAEMON_RENDEZVOUS_RESPONSE_SIZE &&
runtime_hello_response_decode(payload, result_out);
free(payload);
if (received != 1) {
/* The kernel completed the pipe/socket connect, so a server process
* exists, yet the HELLO went unanswered. Name that holder: a dead
* runtime behind a live endpoint is otherwise indistinguishable from
* absence, and the 2026-08-29 zombie hid behind exactly that gap. */
result_out->muted_endpoint_holder_pid = cbm_daemon_ipc_connection_peer_pid(connection);
}
if (!valid || result_out->status != CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED) {
cbm_daemon_ipc_connection_close(connection);
return NULL;
Expand Down
20 changes: 20 additions & 0 deletions src/daemon/runtime.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,12 @@ typedef struct {
cbm_daemon_client_id_t client_id;
/* Kernel-authenticated PID of this client as observed by the daemon. */
uint64_t authenticated_process_id;
/* Nonzero only when the transport connect succeeded but no valid response
* arrived in time: the kernel-reported PID of the process that holds the
* endpoint. A held-but-mute endpoint is a live process whose runtime is
* wedged (2026-08-29 zombie incident) — callers must report this PID
* rather than fold the failure into plain absence. */
uint64_t muted_endpoint_holder_pid;
cbm_daemon_conflict_t conflict;
char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE];
} cbm_daemon_runtime_connect_result_t;
Expand Down Expand Up @@ -265,6 +271,11 @@ typedef struct {
uint32_t client_pids[CBM_DAEMON_CONTROL_CLIENT_CAP];
char build_fingerprint[CBM_DAEMON_BUILD_FINGERPRINT_SIZE];
char semantic_version[CBM_DAEMON_SEMVER_SIZE];
/* Meaningful even when the status request itself FAILED: nonzero when the
* endpoint accepted the transport connect but answered nothing — the
* kernel-reported PID of the mute holder. `daemon status` uses this to
* name a zombie generation instead of reporting bare "not running". */
uint64_t muted_endpoint_holder_pid;
} cbm_daemon_runtime_status_t;

typedef struct {
Expand Down Expand Up @@ -421,6 +432,15 @@ void cbm_daemon_runtime_force_peer_image_unverified_for_testing(bool force);
* i.e. the tamper case that must still be rejected after unverifiable images
* became admissible. */
void cbm_daemon_runtime_force_peer_image_mismatch_for_testing(bool force);
/* Abandoned-request containment seams (2026-08-29 zombie incident). The
* timeout override shrinks the join ceiling to test scale; zero restores the
* production constant. The hook replaces the terminal containment stop with a
* recordable callback so an in-process harness survives the trigger; NULL
* restores process termination. */
typedef void (*cbm_daemon_runtime_containment_hook_t)(const char *component);
void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms);
void cbm_daemon_runtime_set_containment_hook_for_testing(
cbm_daemon_runtime_containment_hook_t hook);
#endif

#endif /* CBM_DAEMON_RUNTIME_H */
6 changes: 5 additions & 1 deletion src/daemon/version_cohort.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,11 @@ cbm_version_cohort_status_t cbm_version_cohort_acquire(cbm_version_cohort_manage
* which a mismatched client can join. Name it: "was the lifetime lock
* held?" is exactly what separates a local run (conflict raised) from
* a CI run (client admitted), and it was not observable in any log. */
cbm_log_info("version_cohort.claimed_unheld", "build",
/* Key wording matters here: the old name "claimed_unheld" read as a
* stale-claim anomaly and derailed the 2026-08-29 zombie diagnosis,
* when the record actually marks the HEALTHY fresh-claim path that
* every normal start logs. */
cbm_log_info("version_cohort.claimed_fresh", "prior_holder", "none", "build",
identity->build_fingerprint ? identity->build_fingerprint : "<null>");
status = version_cohort_claim_new(lease, identity, deadline_ms);
} else if (status == CBM_VERSION_COHORT_BUSY) {
Expand Down
Loading
Loading