From fc1b1ee74efde8632977f936858d2e04216d8079 Mon Sep 17 00:00:00 2001 From: OhOkThisIsFine <102485413+OhOkThisIsFine@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:38:35 -0700 Subject: [PATCH] fix(daemon): contain zombie generations from abandoned requests, name mute endpoint holders Fixes the 2026-08-29 daemon zombie class: a client disconnected while its application request was in flight, the request never observed cancellation, and runtime_worker_finish joined the request thread with no deadline. The disconnect path wedged forever, the generation stayed formally RUNNING, and a dead process held the endpoint pipes and UI port for nine hours while every new client timed out with no diagnostic. Four changes: - runtime: the disconnect-path reap of an in-flight application request now waits at most RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS (30 s), then logs daemon.application_request_unresponsive (peer pid, request token) and fail-stops the process. The kernel releases every native claim and the next client starts a fresh generation. Test seams expose the ceiling and replace the terminal stop with a recordable hook. - diagnostics: a transport connect that reaches a live process but gets no valid answer now resolves the holder's kernel pid (GetNamedPipeServerProcessId / SO_PEERCRED). `daemon start` classifies a mute holder as RESERVED (no doomed competitor spawns) and names the pid in its timeout message; `daemon status` prints "not responding (endpoint held by pid N)" with recovery guidance instead of "not running". - frontend: the maintenance monitor's quiet-state poll drops from 10 ms to 250 ms. Each probe revalidates the whole ancestor directory chain on Windows; at 100 Hz that burned ~20% of a core for the life of every MCP session (7762 CPU-seconds on one 9.5 h session). Detection latency spends 0.25 s of the 15 s activation drain budget. - version_cohort: the healthy fresh-claim record is renamed from "claimed_unheld" (misread as a stale-claim anomaly during the incident diagnosis) to "claimed_fresh" with prior_holder=none. Co-Authored-By: Claude Fable 5 Signed-off-by: OhOkThisIsFine <102485413+OhOkThisIsFine@users.noreply.github.com> --- src/daemon/bootstrap.c | 47 +++++-- src/daemon/frontend.c | 15 ++- src/daemon/runtime.c | 98 ++++++++++++++- src/daemon/runtime.h | 20 +++ src/daemon/version_cohort.c | 6 +- src/main.c | 13 ++ tests/test_daemon_bootstrap.c | 27 +++++ tests/test_daemon_runtime.c | 222 ++++++++++++++++++++++++++++++++++ 8 files changed, 434 insertions(+), 14 deletions(-) diff --git a/src/daemon/bootstrap.c b/src/daemon/bootstrap.c index ee61261d3..50f189036 100644 --- a/src/daemon/bootstrap.c +++ b/src/daemon/bootstrap.c @@ -359,11 +359,20 @@ static cbm_daemon_bootstrap_status_t bootstrap_finish_probe( static cbm_daemon_bootstrap_probe_status_t bootstrap_probe( const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops, - cbm_daemon_runtime_client_t **client_out, cbm_daemon_runtime_connect_result_t *connect_result) { + cbm_daemon_runtime_client_t **client_out, cbm_daemon_runtime_connect_result_t *connect_result, + uint64_t *muted_holder_pid_io) { memset(connect_result, 0, sizeof(*connect_result)); *client_out = NULL; - return ops->probe(ops->context, config->endpoint, config->identity, config->connect_timeout_ms, - client_out, connect_result); + cbm_daemon_bootstrap_probe_status_t status = + ops->probe(ops->context, config->endpoint, config->identity, config->connect_timeout_ms, + client_out, connect_result); + /* Sticky across probes: a later fast-path probe never attempts a connect + * and reports no holder, but a mute holder seen once must survive into + * the final timeout diagnostic. */ + if (muted_holder_pid_io && connect_result->muted_endpoint_holder_pid != 0) { + *muted_holder_pid_io = connect_result->muted_endpoint_holder_pid; + } + return status; } static bool bootstrap_probe_is_finishable(cbm_daemon_bootstrap_probe_status_t probe) { @@ -432,8 +441,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( cbm_daemon_runtime_client_t *client = NULL; cbm_daemon_runtime_connect_result_t connect_result; + uint64_t muted_holder_pid = 0; cbm_daemon_bootstrap_probe_status_t probe = - bootstrap_probe(config, ops, &client, &connect_result); + bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid); if (bootstrap_probe_is_finishable(probe)) { cbm_daemon_bootstrap_status_t status = bootstrap_finish_probe(probe, client, &connect_result, ops, result_out); @@ -461,7 +471,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( * replacements from being launched. */ generation_observed = true; bootstrap_pause(deadline); - probe = bootstrap_probe(config, ops, &client, &connect_result); + probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid); continue; } @@ -473,7 +483,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( } if (lock_status == 0) { bootstrap_pause(deadline); - probe = bootstrap_probe(config, ops, &client, &connect_result); + probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid); continue; } if (lock_status != 1 || !startup_lock) { @@ -482,7 +492,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( } lock_acquired = true; - probe = bootstrap_probe(config, ops, &client, &connect_result); + probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid); if (probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED || probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL) { generation_observed = true; @@ -516,7 +526,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( * bootstrap against a daemon that is trying to cleanly stand down. */ do { bootstrap_pause(deadline); - probe = bootstrap_probe(config, ops, &client, &connect_result); + probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid); if (probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED || probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL) { generation_observed = true; @@ -545,7 +555,16 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( } result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED; - if (generation_observed) { + if (muted_holder_pid != 0) { + /* The one diagnostic the 2026-08-29 zombie recovery had to assemble by + * hand from process, pipe, and log correlation: name the pid that + * holds the endpoint without answering, and say what to do with it. */ + (void)snprintf(result_out->message, sizeof(result_out->message), + "CBM daemon endpoint is held by pid %llu but that process answered no " + "rendezvous within %u ms; the daemon runtime is likely dead — stop that " + "process, then retry", + (unsigned long long)muted_holder_pid, config->startup_timeout_ms); + } else if (generation_observed) { (void)snprintf(result_out->message, sizeof(result_out->message), "CBM daemon is active or starting but could not accept this client " "within %u ms", @@ -578,6 +597,16 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect * existing generation answered. Never reinterpret them as absence. */ return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED; } + if (connect_result->status == CBM_DAEMON_RUNTIME_CONNECT_ERROR && + connect_result->muted_endpoint_holder_pid != 0) { + /* The transport connected to a live process that answered nothing. + * That endpoint is OWNED, whatever the advisory locks read right now + * (a wedged generation can hold pipes while its lock files churn). + * Classifying this as absence made the 2026-08-29 zombie invisible: + * the starter spawned doomed competitors for 30 s and then reported a + * bare timeout with no holder named. */ + return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED; + } if (lifetime_status == 1) { return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED; } diff --git a/src/daemon/frontend.c b/src/daemon/frontend.c index 17e0a8d2f..2dd986901 100644 --- a/src/daemon/frontend.c +++ b/src/daemon/frontend.c @@ -40,6 +40,19 @@ enum { * cancelled and the frontend exits cleanly. */ FRONTEND_EOF_DRAIN_MS = 15000, FRONTEND_MAINTENANCE_POLL_MS = 10, + /* Cadence for the quiet no-maintenance steady state. Every presence probe + * try-acquires the maintenance marker lock, and on Windows that + * revalidates the whole ancestor directory chain (a CreateFileW + + * identity check per path component). At the old 10 ms cadence this + * burned ~20% of a core for the life of every MCP session — 7762 + * CPU-seconds on one 9.5 h session in the 2026-08-29 incident. The + * budget: an activation drains participants within + * CLI_ACTIVATION_DRAIN_TIMEOUT_MS (15 s) and the monitor's own exit grace + * is FRONTEND_MAINTENANCE_GRACE_MS (3 s), so a 250 ms detection latency + * spends under 2% of that window while cutting the probe cost 25-fold. + * The 10 ms cadence still paces the monitor's post-detection grace loop, + * which does no lock probing. */ + FRONTEND_MAINTENANCE_IDLE_POLL_MS = 250, /* The owner thread may be draining a supervised process tree. Preserve the * supervisor's complete graceful + forced-settle window before the monitor * fail-stops the process, plus scheduling/teardown margin. */ @@ -162,7 +175,7 @@ static void *frontend_maintenance_monitor_worker(void *opaque) { cbm_version_cohort_maintenance_presence_t presence = frontend_observe_maintenance(monitor->manager, true); if (presence == CBM_VERSION_COHORT_MAINTENANCE_ABSENT) { - cbm_usleep(FRONTEND_MAINTENANCE_POLL_MS * 1000U); + cbm_usleep(FRONTEND_MAINTENANCE_IDLE_POLL_MS * 1000U); continue; } diff --git a/src/daemon/runtime.c b/src/daemon/runtime.c index 2ecf32627..7db0f0225 100644 --- a/src/daemon/runtime.c +++ b/src/daemon/runtime.c @@ -36,6 +36,19 @@ static atomic_bool runtime_force_peer_image_mismatch_seam; void cbm_daemon_runtime_force_peer_image_mismatch_for_testing(bool force) { atomic_store(&runtime_force_peer_image_mismatch_seam, force); } +/* The abandoned-request containment path ends in process termination, which an + * in-process harness cannot observe. The timeout override makes the ceiling + * reachable in test time; the hook replaces termination with a recordable + * callback. Both stay inert (zero/NULL) outside tests. */ +static _Atomic uint32_t runtime_abandoned_request_join_timeout_seam; +void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms) { + atomic_store(&runtime_abandoned_request_join_timeout_seam, timeout_ms); +} +static _Atomic(cbm_daemon_runtime_containment_hook_t) runtime_containment_hook_seam; +void cbm_daemon_runtime_set_containment_hook_for_testing( + cbm_daemon_runtime_containment_hook_t hook) { + atomic_store(&runtime_containment_hook_seam, hook); +} #endif #ifdef _WIN32 @@ -72,6 +85,16 @@ enum { * a local cooperative peer is already blocked in read and consumes the * rejection within milliseconds. */ RUNTIME_REJECT_DRAIN_TIMEOUT_MS = 250, + /* Ceiling on how long a disconnecting worker waits for its in-flight + * application request to observe cancellation. session_cancel has already + * run by the time this wait starts, so a compliant handler returns within + * milliseconds; the wait is sized for a handler that only polls its cancel + * flag between long pipeline stages. A handler that ignores cancellation + * past this ceiling turned the daemon into a permanent zombie once + * (2026-08-29): the unbounded join blocked the disconnect path, admission + * wedged behind it, and the dead generation held the endpoint pipes and + * the UI port for nine hours with nothing logged. */ + RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS = 30000, RUNTIME_PATH_CAP = 4096, RENDEZVOUS_REQUEST_ABI_OFFSET = 0, @@ -1313,6 +1336,26 @@ static void *runtime_application_worker(void *opaque) { return NULL; } +static _Noreturn void runtime_cleanup_fail_stop(const char *component); + +static void runtime_contain_unresponsive_application(cbm_daemon_runtime_worker_t *worker) { + char peer_pid[32]; + char token[32]; + (void)snprintf(peer_pid, sizeof(peer_pid), "%llu", (unsigned long long)worker->peer_process_id); + (void)snprintf(token, sizeof(token), "%llu", + (unsigned long long)worker->application_request_token); + cbm_log_error("daemon.application_request_unresponsive", "peer_pid", peer_pid, "request_token", + token); +#ifdef CBM_ENABLE_TEST_SEAMS + cbm_daemon_runtime_containment_hook_t hook = atomic_load(&runtime_containment_hook_seam); + if (hook) { + hook("application_request_join"); + return; + } +#endif + runtime_cleanup_fail_stop("application_request_join"); +} + static bool runtime_worker_reap_application(cbm_daemon_runtime_worker_t *worker, bool wait) { if (!worker->application_thread_started) { return true; @@ -1320,6 +1363,35 @@ static bool runtime_worker_reap_application(cbm_daemon_runtime_worker_t *worker, if (!wait && !atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) { return false; } + if (!atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) { + /* The peer is gone and session_cancel already ran, so the handler owes + * a prompt return; only a handler that ignores cancellation reaches + * the ceiling. An unbounded join here once turned one wedged request + * into a whole-daemon zombie (2026-08-29): the disconnect path + * blocked, the worker slot never released, and the dead generation + * held the endpoint pipes and UI port for nine hours with nothing + * logged while every new client timed out bare. Containment + * terminates the process instead — the kernel releases every native + * claim and the next client starts a fresh generation. */ + uint32_t timeout_ms = RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS; +#ifdef CBM_ENABLE_TEST_SEAMS + uint32_t seam_timeout = atomic_load(&runtime_abandoned_request_join_timeout_seam); + if (seam_timeout != 0) { + timeout_ms = seam_timeout; + } +#endif + uint64_t deadline = runtime_deadline_after(timeout_ms); + while (!atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) { + if (cbm_now_ms() >= deadline) { + /* Terminal in production. A test containment hook returns, and + * the join then waits for the harness to release the handler + * so teardown stays leak-free under the sanitizers. */ + runtime_contain_unresponsive_application(worker); + deadline = UINT64_MAX; + } + runtime_wait_tick(deadline); + } + } if (cbm_thread_join(&worker->application_thread) != 0) { return false; } @@ -2569,8 +2641,11 @@ static bool runtime_control_request_send(const cbm_daemon_ipc_endpoint_t *endpoi const cbm_daemon_build_identity_t *identity, cbm_daemon_runtime_operation_t operation, uint32_t timeout_ms, uint32_t response_size, - uint8_t **payload_out) { + uint8_t **payload_out, uint64_t *muted_holder_pid_out) { *payload_out = NULL; + if (muted_holder_pid_out) { + *muted_holder_pid_out = 0; + } if (!endpoint || !identity || !identity->build_fingerprint || timeout_ms == CBM_DAEMON_IPC_WAIT_FOREVER) { return false; @@ -2590,6 +2665,15 @@ static bool runtime_control_request_send(const cbm_daemon_ipc_endpoint_t *endpoi int received = sent ? cbm_daemon_ipc_receive_frame_bounded(connection, timeout_ms, response_size, &frame, &payload) : 0; + if (muted_holder_pid_out && sent && + (received != 1 || (frame.type == CBM_DAEMON_FRAME_RESPONSE && frame.flags != operation))) { + /* Connected but not served: either total silence (dead runtime), or a + * wrong-operation reject frame (a wedged generation whose accept path + * still answers inline while every worker slot is stuck). Both are + * the zombie class — surface the holder's kernel-reported pid so + * `daemon status` can name it instead of reporting "not running". */ + *muted_holder_pid_out = cbm_daemon_ipc_connection_peer_pid(connection); + } bool valid = received == 1 && frame.type == CBM_DAEMON_FRAME_RESPONSE && frame.flags == operation && frame.length == response_size && payload && payload[0] == 1U; @@ -2617,7 +2701,8 @@ bool cbm_daemon_runtime_request_status(const cbm_daemon_ipc_endpoint_t *endpoint memset(status_out, 0, sizeof(*status_out)); uint8_t *payload = NULL; if (!runtime_control_request_send(endpoint, identity, CBM_DAEMON_RUNTIME_OP_STATUS, timeout_ms, - CBM_DAEMON_STATUS_RESPONSE_SIZE, &payload)) { + CBM_DAEMON_STATUS_RESPONSE_SIZE, &payload, + &status_out->muted_endpoint_holder_pid)) { return false; } status_out->permanent = (payload[1] & 0x01U) != 0U; @@ -2648,7 +2733,7 @@ bool cbm_daemon_runtime_request_stop(const cbm_daemon_ipc_endpoint_t *endpoint, memset(result_out, 0, sizeof(*result_out)); uint8_t *payload = NULL; if (!runtime_control_request_send(endpoint, identity, CBM_DAEMON_RUNTIME_OP_STOP, timeout_ms, - CBM_DAEMON_STOP_RESPONSE_SIZE, &payload)) { + CBM_DAEMON_STOP_RESPONSE_SIZE, &payload, NULL)) { return false; } result_out->accepted = (payload[1] & 0x01U) != 0U; @@ -2691,6 +2776,13 @@ cbm_daemon_runtime_client_t *cbm_daemon_runtime_client_connect( frame.length == CBM_DAEMON_RENDEZVOUS_RESPONSE_SIZE && runtime_hello_response_decode(payload, result_out); free(payload); + if (received != 1) { + /* The kernel completed the pipe/socket connect, so a server process + * exists, yet the HELLO went unanswered. Name that holder: a dead + * runtime behind a live endpoint is otherwise indistinguishable from + * absence, and the 2026-08-29 zombie hid behind exactly that gap. */ + result_out->muted_endpoint_holder_pid = cbm_daemon_ipc_connection_peer_pid(connection); + } if (!valid || result_out->status != CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED) { cbm_daemon_ipc_connection_close(connection); return NULL; diff --git a/src/daemon/runtime.h b/src/daemon/runtime.h index 997b5d90c..4d53da9a4 100644 --- a/src/daemon/runtime.h +++ b/src/daemon/runtime.h @@ -214,6 +214,12 @@ typedef struct { cbm_daemon_client_id_t client_id; /* Kernel-authenticated PID of this client as observed by the daemon. */ uint64_t authenticated_process_id; + /* Nonzero only when the transport connect succeeded but no valid response + * arrived in time: the kernel-reported PID of the process that holds the + * endpoint. A held-but-mute endpoint is a live process whose runtime is + * wedged (2026-08-29 zombie incident) — callers must report this PID + * rather than fold the failure into plain absence. */ + uint64_t muted_endpoint_holder_pid; cbm_daemon_conflict_t conflict; char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE]; } cbm_daemon_runtime_connect_result_t; @@ -265,6 +271,11 @@ typedef struct { uint32_t client_pids[CBM_DAEMON_CONTROL_CLIENT_CAP]; char build_fingerprint[CBM_DAEMON_BUILD_FINGERPRINT_SIZE]; char semantic_version[CBM_DAEMON_SEMVER_SIZE]; + /* Meaningful even when the status request itself FAILED: nonzero when the + * endpoint accepted the transport connect but answered nothing — the + * kernel-reported PID of the mute holder. `daemon status` uses this to + * name a zombie generation instead of reporting bare "not running". */ + uint64_t muted_endpoint_holder_pid; } cbm_daemon_runtime_status_t; typedef struct { @@ -421,6 +432,15 @@ void cbm_daemon_runtime_force_peer_image_unverified_for_testing(bool force); * i.e. the tamper case that must still be rejected after unverifiable images * became admissible. */ void cbm_daemon_runtime_force_peer_image_mismatch_for_testing(bool force); +/* Abandoned-request containment seams (2026-08-29 zombie incident). The + * timeout override shrinks the join ceiling to test scale; zero restores the + * production constant. The hook replaces the terminal containment stop with a + * recordable callback so an in-process harness survives the trigger; NULL + * restores process termination. */ +typedef void (*cbm_daemon_runtime_containment_hook_t)(const char *component); +void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms); +void cbm_daemon_runtime_set_containment_hook_for_testing( + cbm_daemon_runtime_containment_hook_t hook); #endif #endif /* CBM_DAEMON_RUNTIME_H */ diff --git a/src/daemon/version_cohort.c b/src/daemon/version_cohort.c index f642001db..f3536ead1 100644 --- a/src/daemon/version_cohort.c +++ b/src/daemon/version_cohort.c @@ -463,7 +463,11 @@ cbm_version_cohort_status_t cbm_version_cohort_acquire(cbm_version_cohort_manage * which a mismatched client can join. Name it: "was the lifetime lock * held?" is exactly what separates a local run (conflict raised) from * a CI run (client admitted), and it was not observable in any log. */ - cbm_log_info("version_cohort.claimed_unheld", "build", + /* Key wording matters here: the old name "claimed_unheld" read as a + * stale-claim anomaly and derailed the 2026-08-29 zombie diagnosis, + * when the record actually marks the HEALTHY fresh-claim path that + * every normal start logs. */ + cbm_log_info("version_cohort.claimed_fresh", "prior_holder", "none", "build", identity->build_fingerprint ? identity->build_fingerprint : ""); status = version_cohort_claim_new(lease, identity, deadline_ms); } else if (status == CBM_VERSION_COHORT_BUSY) { diff --git a/src/main.c b/src/main.c index 1304c2093..1e06d86ff 100644 --- a/src/main.c +++ b/src/main.c @@ -2258,6 +2258,19 @@ static int main_run_daemon_ctl(int argc, char **argv, const cbm_daemon_ipc_endpo if (strcmp(subcommand, "status") == 0) { if (!active) { + if (status.muted_endpoint_holder_pid != 0) { + /* Alive process, dead runtime (2026-08-29 zombie class). + * "not running" here sent the operator hunting through + * processes, pipes, and logs by hand; name the holder and the + * recovery instead. */ + printf("daemon: not responding (endpoint held by pid %llu)\n", + (unsigned long long)status.muted_endpoint_holder_pid); + printf("hint: that process holds the daemon endpoint but answered nothing; " + "its runtime is likely dead. Terminate pid %llu, then run " + "`codebase-memory-mcp daemon start`.\n", + (unsigned long long)status.muted_endpoint_holder_pid); + return EXIT_FAILURE; + } printf("daemon: not running\n"); printf("hint: `codebase-memory-mcp daemon start` keeps a daemon warm so CLI " "commands and hooks skip the per-command startup cost.\n"); diff --git a/tests/test_daemon_bootstrap.c b/tests/test_daemon_bootstrap.c index 29f6a6f08..a9c8a368e 100644 --- a/tests/test_daemon_bootstrap.c +++ b/tests/test_daemon_bootstrap.c @@ -780,6 +780,32 @@ TEST(daemon_bootstrap_rejected_connect_is_reserved_and_never_unavailable) { PASS(); } +/* 2026-08-29 zombie regression: a connect that reached a live process but got + * no answer names that holder in muted_endpoint_holder_pid. That endpoint is + * owned regardless of what the advisory locks read, so classification must be + * RESERVED on every lifetime answer — never absence, never a spawn license. */ +TEST(daemon_bootstrap_mute_endpoint_holder_is_reserved_and_never_unavailable) { + cbm_daemon_runtime_connect_result_t mute = {0}; + mute.status = CBM_DAEMON_RUNTIME_CONNECT_ERROR; + mute.muted_endpoint_holder_pid = 4242; + ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&mute, 1), + CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED); + ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&mute, 0), + CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED); + ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&mute, -1), + CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED); + + /* A protocol-level rejection still wins over the holder pid: an answering + * generation is more precise evidence than a silent one. */ + cbm_daemon_runtime_connect_result_t rejected = {0}; + rejected.status = CBM_DAEMON_RUNTIME_CONNECT_REJECTED; + rejected.muted_endpoint_holder_pid = 4242; + snprintf(rejected.message, sizeof(rejected.message), "CBM daemon is stopping"); + ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&rejected, 0), + CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL); + PASS(); +} + TEST(daemon_bootstrap_concurrent_first_clients_spawn_one_daemon) { bootstrap_endpoint_fixture_t fixture; ASSERT_TRUE(bootstrap_endpoint_fixture_start(&fixture, "startup-race")); @@ -872,6 +898,7 @@ SUITE(daemon_bootstrap) { RUN_TEST(daemon_bootstrap_reserved_then_absent_spawns_replacement); RUN_TEST(daemon_bootstrap_releases_handoff_when_spawned_generation_is_reserved); RUN_TEST(daemon_bootstrap_rejected_connect_is_reserved_and_never_unavailable); + RUN_TEST(daemon_bootstrap_mute_endpoint_holder_is_reserved_and_never_unavailable); RUN_TEST(daemon_bootstrap_concurrent_first_clients_spawn_one_daemon); #ifdef __APPLE__ RUN_TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous); diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index 5f41d914e..ba00b50e8 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -3943,6 +3943,224 @@ TEST(daemon_runtime_noncooperative_callback_does_not_detach_or_unbound_stop) { PASS(); } +#if defined(CBM_ENABLE_TEST_SEAMS) +static atomic_int runtime_test_containment_fired; +static atomic_bool runtime_test_containment_component_exact; + +static void runtime_test_containment_hook(const char *component) { + if (component && strcmp(component, "application_request_join") == 0) { + atomic_store_explicit(&runtime_test_containment_component_exact, true, + memory_order_release); + } + atomic_fetch_add_explicit(&runtime_test_containment_fired, 1, memory_order_release); +} + +/* 2026-08-29 zombie regression: a disconnecting worker joined its in-flight + * application request with no deadline. A handler that ignored cancellation + * therefore wedged the disconnect path forever and the dead generation kept + * the endpoint. The join must now reach the containment boundary in bounded + * time; the seam hook stands in for the terminal process stop so the harness + * can then supply cooperation and prove a clean join and teardown. */ +TEST(daemon_runtime_abandoned_request_join_reaches_containment_in_bounded_time) { + static const uint8_t request[] = {'w', 'e', 'd', 'g', 'e'}; + enum { JOIN_BOUND_MS = 150, CONTAINMENT_OBSERVED_MAX_MS = 5000 }; + cbm_daemon_build_identity_t identity = + runtime_test_identity("2.4.0", runtime_test_self_build()); + runtime_application_context_t context; + runtime_application_context_init(&context, true); + atomic_store_explicit(&context.ignore_first_request_cancel, true, memory_order_release); + atomic_store(&runtime_test_containment_fired, 0); + atomic_store(&runtime_test_containment_component_exact, false); + atomic_bool request_thread_completed; + atomic_init(&request_thread_completed, false); + runtime_test_fixture_t fixture; + bool started = runtime_test_fixture_start_application(&fixture, "application-abandoned-join", + &identity, &context); + cbm_daemon_runtime_connect_result_t result = {0}; + cbm_daemon_runtime_client_t *client = NULL; + runtime_application_client_call_t call = { + .request = request, + .request_length = (uint32_t)sizeof(request), + .completed = &request_thread_completed, + .status = CBM_DAEMON_RUNTIME_APPLICATION_OK, + }; + cbm_thread_t request_thread; + int request_thread_create_rc = -1; + int request_thread_join_rc = -1; + bool request_thread_started = false; + bool callback_started = false; + bool close_begun = false; + bool containment_observed = false; + uint64_t containment_elapsed_ms = UINT64_MAX; + bool exited_after_release = false; + + cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(JOIN_BOUND_MS); + cbm_daemon_runtime_set_containment_hook_for_testing(runtime_test_containment_hook); + + if (started) { + client = cbm_daemon_runtime_client_connect(fixture.endpoint, &identity, + RUNTIME_TEST_TIMEOUT_MS, &result); + } + if (client) { + call.client = client; + request_thread_create_rc = cbm_thread_create( + &request_thread, 128U * 1024U, runtime_application_client_request_thread, &call); + request_thread_started = request_thread_create_rc == 0; + callback_started = + request_thread_started && + runtime_test_wait_atomic_bool(&context.first_request_started, RUNTIME_TEST_TIMEOUT_MS); + } + if (callback_started) { + uint64_t containment_started_ms = cbm_now_ms(); + close_begun = cbm_daemon_runtime_client_close_begin(client); + uint64_t deadline = cbm_now_ms() + CONTAINMENT_OBSERVED_MAX_MS; + while (atomic_load_explicit(&runtime_test_containment_fired, memory_order_acquire) == 0 && + cbm_now_ms() < deadline) { + struct timespec pause = {.tv_sec = 0, .tv_nsec = 1000000}; + (void)cbm_nanosleep(&pause, NULL); + } + containment_observed = + atomic_load_explicit(&runtime_test_containment_fired, memory_order_acquire) > 0; + containment_elapsed_ms = cbm_now_ms() - containment_started_ms; + } + + /* Cooperation after the boundary: the reap waits for the released handler + * so join and teardown stay provably leak-free under the sanitizers. */ + atomic_store_explicit(&context.release_first_request, true, memory_order_release); + if (request_thread_started) { + request_thread_join_rc = cbm_thread_join(&request_thread); + } + if (client) { + (void)cbm_daemon_runtime_client_close_finish(client, RUNTIME_TEST_TIMEOUT_MS); + client = NULL; + } + if (started) { + exited_after_release = + cbm_daemon_runtime_service_wait_exited(fixture.service, RUNTIME_TEST_TIMEOUT_MS); + } + free(call.response); + runtime_test_fixture_finish(&fixture); + cbm_daemon_runtime_set_containment_hook_for_testing(NULL); + cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(0); + + ASSERT_TRUE(started); + ASSERT_EQ(result.status, CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED); + ASSERT_EQ(request_thread_create_rc, 0); + ASSERT_TRUE(callback_started); + ASSERT_TRUE(close_begun); + ASSERT_TRUE(containment_observed); + ASSERT_TRUE(containment_elapsed_ms <= CONTAINMENT_OBSERVED_MAX_MS); + ASSERT_EQ(atomic_load(&runtime_test_containment_fired), 1); + ASSERT_TRUE(atomic_load(&runtime_test_containment_component_exact)); + ASSERT_EQ(request_thread_join_rc, 0); + ASSERT_TRUE(exited_after_release); + ASSERT_EQ(atomic_load(&context.opened), 1); + ASSERT_EQ(atomic_load(&context.requests), 1); + ASSERT_EQ(atomic_load(&context.cancelled), 1); + ASSERT_EQ(atomic_load(&context.closed), 1); + PASS(); +} +#endif + +typedef struct { + cbm_daemon_ipc_listener_t *listener; + cbm_daemon_ipc_connection_t *held[2]; + atomic_int held_count; + atomic_bool stop; +} runtime_mute_holder_t; + +/* Accepts up to two transport connections and never answers a frame: the + * kernel-level shape of the 2026-08-29 zombie (endpoint owned, runtime dead). */ +static void *runtime_test_mute_holder_thread(void *opaque) { + runtime_mute_holder_t *holder = opaque; + while (!atomic_load_explicit(&holder->stop, memory_order_acquire) && + atomic_load_explicit(&holder->held_count, memory_order_acquire) < 2) { + cbm_daemon_ipc_connection_t *connection = NULL; + int accepted = cbm_daemon_ipc_accept(holder->listener, 50, &connection); + if (accepted == 1 && connection) { + int index = atomic_load_explicit(&holder->held_count, memory_order_acquire); + holder->held[index] = connection; + atomic_store_explicit(&holder->held_count, index + 1, memory_order_release); + } + } + return NULL; +} + +static uint64_t runtime_test_self_process_id(void) { +#ifdef _WIN32 + return (uint64_t)GetCurrentProcessId(); +#else + return (uint64_t)getpid(); +#endif +} + +/* 2026-08-29 zombie regression: a held endpoint that answers nothing must + * name its holder. Both the HELLO connect and the one-shot status probe + * fail against a mute holder, and both must report the holder's + * kernel-authenticated pid instead of plain absence. */ +TEST(daemon_runtime_mute_endpoint_holder_pid_is_reported) { + enum { MUTE_CONNECT_TIMEOUT_MS = 500 }; + cbm_daemon_build_identity_t identity = + runtime_test_identity("2.4.0", runtime_test_self_build()); + char parent[RUNTIME_TEST_PATH_CAP] = {0}; + char key[CBM_DAEMON_KEY_SIZE] = {0}; + char runtime_dir[RUNTIME_TEST_PATH_CAP] = {0}; + cbm_daemon_ipc_endpoint_t *endpoint = NULL; + runtime_mute_holder_t holder = {0}; + atomic_init(&holder.held_count, 0); + atomic_init(&holder.stop, false); + cbm_thread_t holder_thread; + bool holder_started = false; + + bool prepared = th_secure_runtime_parent_new(parent, sizeof(parent), "mute-holder") && + cbm_daemon_rendezvous_key(key); + endpoint = prepared ? cbm_daemon_ipc_endpoint_new(key, parent) : NULL; + bool runtime_dir_copied = + endpoint && + runtime_test_copy_path(runtime_dir, cbm_daemon_ipc_endpoint_runtime_dir(endpoint)); + holder.listener = endpoint ? cbm_daemon_ipc_listen(endpoint) : NULL; + holder_started = + holder.listener && + cbm_thread_create(&holder_thread, 0, runtime_test_mute_holder_thread, &holder) == 0; + + cbm_daemon_runtime_connect_result_t connect_result = {0}; + cbm_daemon_runtime_client_t *client = NULL; + cbm_daemon_runtime_status_t status = {0}; + bool status_active = true; + if (holder_started) { + client = cbm_daemon_runtime_client_connect(endpoint, &identity, MUTE_CONNECT_TIMEOUT_MS, + &connect_result); + status_active = cbm_daemon_runtime_request_status(endpoint, &identity, + MUTE_CONNECT_TIMEOUT_MS, &status); + } + + atomic_store_explicit(&holder.stop, true, memory_order_release); + if (holder_started) { + (void)cbm_thread_join(&holder_thread); + } + for (size_t index = 0; index < 2; index++) { + if (holder.held[index]) { + cbm_daemon_ipc_connection_close(holder.held[index]); + } + } + if (holder.listener) { + cbm_daemon_ipc_listener_close(holder.listener); + } + cbm_daemon_ipc_endpoint_free(endpoint); + (void)cbm_rmdir(runtime_dir); + (void)cbm_rmdir(parent); + + ASSERT_TRUE(prepared); + ASSERT_TRUE(runtime_dir_copied); + ASSERT_TRUE(holder_started); + ASSERT_NULL(client); + ASSERT_EQ(connect_result.status, CBM_DAEMON_RUNTIME_CONNECT_ERROR); + ASSERT_EQ(connect_result.muted_endpoint_holder_pid, runtime_test_self_process_id()); + ASSERT_FALSE(status_active); + ASSERT_EQ(status.muted_endpoint_holder_pid, runtime_test_self_process_id()); + PASS(); +} + TEST(daemon_runtime_application_busy_cap_and_malformed_are_isolated) { static const uint8_t blocking_request[] = {'f', 'i', 'r', 's', 't'}; static const uint8_t busy_request[] = {'b', 'u', 's', 'y'}; @@ -4782,6 +5000,10 @@ SUITE(daemon_runtime) { RUN_TEST(daemon_runtime_close_begin_releases_admission_with_inflight_request); RUN_TEST(daemon_runtime_disconnect_cancels_blocked_non_index_child_and_preserves_other_session); RUN_TEST(daemon_runtime_noncooperative_callback_does_not_detach_or_unbound_stop); +#if defined(CBM_ENABLE_TEST_SEAMS) + RUN_TEST(daemon_runtime_abandoned_request_join_reaches_containment_in_bounded_time); +#endif + RUN_TEST(daemon_runtime_mute_endpoint_holder_pid_is_reported); RUN_TEST(daemon_runtime_application_busy_cap_and_malformed_are_isolated); RUN_TEST(daemon_runtime_malformed_and_zero_cancel_close_only_offending_connections); }