fix(stack): deflake service-state settling and control endpoint acquisition

packages/stack/docs/architecture.md

@@ -101,9 +101,10 @@ runtime logs, and `runtime/` for supervisor-owned runtime files. The
 `ManagedStackManager` is the only component that writes `stack.json`.
 
 Control ownership is the liveness and mutation authority. `acquireControl`
-returns `Owned` for the process that bound the deterministic endpoint or
-`Attached` for a live owner. An attached caller uses the owner's endpoint for
-runtime requests; it never edits the document directly.
+returns `Owned` for the process that bound one of the deterministic endpoint
+candidates or `Attached` for a live owner found on any candidate. An attached
+caller uses the owner's actual endpoint for runtime requests; it never edits
+the document directly.
 
 ### Start and attach
 
@@ -121,8 +122,8 @@ runtime requests; it never edits the document directly.
    `stack.start()` over the control transport when service startup is needed.
 
 `connectManagedStack` reads the document, probes the deterministic endpoint
-without binding it, and returns a `RemoteStack` only when the owner reports a
-ready running state. Read-only status and discovery therefore do not claim an
+candidates without binding them, and returns a `RemoteStack` against the
+owner's actual endpoint only when the owner reports a ready running state. Read-only status and discovery therefore do not claim an
 endpoint; mutating operations acquire control ownership.
 
 ### Update, stop, and delete
@@ -200,14 +201,18 @@ persisted endpoint, or another stack's reservation under the normal exact-port
 rules. A persisted automatic assignment in the control range is invalid and
 fails loudly rather than being silently migrated.
 
-The control endpoint is derived from the stack id and served on loopback. A
-persisted endpoint is accepted only when it matches that derivation. A rare
-hash collision or unrelated listener makes control acquisition fail with a
-typed conflict; a read-only probe treats the address as non-live and never
-claims it. An exact service port can still equal the future endpoint of an
-identity that has never started, so that low-probability conflict is rejected
-when ownership is acquired rather than forbidding every explicit port in the
-reserved range.
+The control endpoint is derived from the stack id and served on loopback. The
+derivation yields a short deterministic candidate sequence rather than a
+single port: an owner binds the first free candidate, skipping candidates
+occupied by other stacks or unrelated listeners, and readers scan the same
+sequence and match the published `ownershipId`. A hash collision between two
+stack ids therefore degrades to the collided stack binding its next candidate
+instead of failing. Acquisition fails with a typed conflict only when every
+candidate is occupied by a foreign listener; a read-only probe treats an
+address with no matching owner as non-live and never claims it. An exact
+service port can still equal a candidate of an identity that has never
+started, so every stack's full candidate set is reserved against exact-port
+requests rather than forbidding every explicit port in the reserved range.
 
 This is deliberately a small single-user localhost mechanism. The control
 protocol has no token authentication; ownership, endpoint identity, and

packages/stack/src/LocalStack.ts

@@ -793,6 +793,9 @@ export const localStackLayer = (
             ).toReversed()) {
               yield* runtime.orchestrator.stopService(target);
             }
+            // Settle the public projection before returning so callers observe
+            // the stop immediately, matching the start/restart/waitReady paths.
+            yield* syncRuntimeProjectedStates(runtime);
           }).pipe(withLifecycleLock),
         restartService: (name) =>
           Effect.gen(function* () {

packages/stack/src/Stack.unit.test.ts

@@ -3,7 +3,7 @@ import { BunServices } from "@effect/platform-bun";
 import { buildGraph } from "@supabase/process-compose";
 import { createHmac } from "node:crypto";
 import { mkdtempSync } from "node:fs";
-import { chmod, readFile, rm } from "node:fs/promises";
+import { readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { Deferred, Effect, Exit, Fiber, Layer, Stream } from "effect";
@@ -247,13 +247,18 @@ describe("Stack", () => {
       ).toBe("StackBuildError");
       expect((yield* readRuntimeConfig).env.SHARED).toBe("replacement-secret");
 
+      // Replace the workspace directory with a plain file so the config write
+      // fails for any user — permission bits alone are bypassed by root.
       const runtimeDirectory = join(runtimeRoot, "edge-runtime");
-      yield* Effect.promise(() => chmod(runtimeDirectory, 0o500));
+      yield* Effect.promise(async () => {
+        await rm(runtimeDirectory, { recursive: true, force: true });
+        await writeFile(runtimeDirectory, "");
+      });
       const failedBundle = functionsBundle(runtimeRoot, "failed-secret");
       const error = yield* stack.reloadFunctions({ functions: failedBundle }).pipe(Effect.flip);
       expect(error._tag).toBe("StackBuildError");
 
-      yield* Effect.promise(() => chmod(runtimeDirectory, 0o700));
+      yield* Effect.promise(() => rm(runtimeDirectory));
       yield* stack.reloadFunctions();
       expect((yield* readRuntimeConfig).env.SHARED).toBe("replacement-secret");
 
@@ -268,12 +273,7 @@ describe("Stack", () => {
       ).toBe(false);
     }).pipe(
       Effect.provide(layer),
-      Effect.ensuring(
-        Effect.promise(async () => {
-          await chmod(join(runtimeRoot, "edge-runtime"), 0o700).catch(() => {});
-          await rm(runtimeRoot, { recursive: true, force: true });
-        }),
-      ),
+      Effect.ensuring(Effect.promise(() => rm(runtimeRoot, { recursive: true, force: true }))),
       Effect.timeout("5 seconds"),
     );
   });
@@ -1087,14 +1087,10 @@ describe("Stack", () => {
     return Effect.gen(function* () {
       const stack = yield* Stack;
       yield* stack.start();
-      const authChanges = yield* stack.stateChanges("auth");
-      const stopped = yield* authChanges.pipe(
-        Stream.filter((state) => state.status === "Stopped"),
-        Stream.runHead,
-        Effect.forkChild({ startImmediately: true }),
-      );
+      // `stopService` settles the public projection before returning, so the
+      // stopped state is observable immediately without stream coordination.
       yield* stack.stopService("auth");
-      expect((yield* Fiber.join(stopped))._tag).toBe("Some");
+      expect((yield* stack.getState("auth")).status).toBe("Stopped");
 
       yield* stack.stop();
       yield* stack.start();

packages/stack/src/discovery.ts

@@ -74,7 +74,7 @@ const liveStatus = (
 ): Effect.Effect<boolean, ManagedStackManagerError, never> =>
   manager
     .probeControl(document.id)
-    .pipe(Effect.map((status) => status?.state === "running" && status.ready));
+    .pipe(Effect.map((probe) => probe?.status.state === "running" && probe.status.ready));
 
 export const listStacks = (opts: {
   readonly cacheRoot: string;

packages/stack/src/managed-control.integration.test.ts

@@ -7,10 +7,13 @@ import { describe, expect } from "vitest";
 import { DaemonServer } from "./DaemonServer.ts";
 import {
   acquireControl,
+  CONTROL_CANDIDATE_COUNT,
   controlEndpoint,
+  controlEndpointCandidates,
   ControlBindError,
   ControlTransport,
   ControlTransportError,
+  probeControl,
 } from "./managed/control.ts";
 import { controlTransportLayer } from "./platform-node.ts";
 import { httpTransportClientLayer } from "./HttpTransportClient.ts";
@@ -103,10 +106,15 @@ const spawnBoundChild = (port: number) => {
 };
 
 describe("managed control endpoint", () => {
-  it.live("derives one deterministic loopback endpoint from the ownership id", () => {
+  it.live("derives deterministic loopback candidates from the ownership id", () => {
     return Effect.sync(() => {
       const endpoint = Effect.runSync(controlEndpoint(STACK_ID));
       expect(endpoint.url).toBe("http://127.0.0.1:13737");
+      const candidates = Effect.runSync(controlEndpointCandidates(STACK_ID));
+      expect(candidates).toHaveLength(CONTROL_CANDIDATE_COUNT);
+      expect(candidates.map(({ port }) => port)).toEqual(
+        Array.from({ length: CONTROL_CANDIDATE_COUNT }, (_, offset) => 13737 + offset),
+      );
     });
   });
 
@@ -254,7 +262,7 @@ describe("managed control endpoint", () => {
     ),
   );
 
-  it.live("rejects a valid owner with a colliding deterministic endpoint", () =>
+  it.live("claims the next candidate when another stack owns the first", () =>
     Effect.scoped(
       live(
         Effect.gen(function* () {
@@ -272,15 +280,20 @@ describe("managed control endpoint", () => {
             ),
           );
           yield* Effect.promise(() => daemonRuntime.runPromise(DaemonServer));
-          const contender = yield* acquireControl({ stackId: COLLIDING_STACK_ID }).pipe(
-            Effect.exit,
-          );
-          expect(Exit.isFailure(contender)).toBe(true);
-          if (Exit.isFailure(contender)) {
-            expect(Cause.squash(contender.cause)).toMatchObject({
-              _tag: "ControlAddressConflictError",
-            });
-          }
+          const contender = yield* acquireControl({ stackId: COLLIDING_STACK_ID });
+          if (contender._tag !== "Owned") throw new Error("expected contender ownership");
+          expect(contender.endpoint.port).not.toBe(owner.endpoint.port);
+
+          // Readers locate each owner at its actual candidate.
+          const ownerProbe = yield* probeControl(STACK_ID);
+          expect(ownerProbe?.endpoint.port).toBe(owner.endpoint.port);
+          const contenderProbe = yield* probeControl(COLLIDING_STACK_ID);
+          expect(contenderProbe?.endpoint.port).toBe(contender.endpoint.port);
+
+          // A second caller for the collided stack attaches to its owner.
+          const attached = yield* acquireControl({ stackId: COLLIDING_STACK_ID });
+          expect(attached._tag).toBe("Attached");
+          expect(attached.endpoint.port).toBe(contender.endpoint.port);
           yield* Effect.promise(() => daemonRuntime.dispose());
         }),
       ),
@@ -306,15 +319,37 @@ describe("managed control endpoint", () => {
     ),
   );
 
-  it.live("rejects an unrelated listener without taking it over", () =>
+  it.live("claims the next candidate without taking over an unrelated listener", () =>
     live(
       Effect.scoped(
         Effect.gen(function* () {
-          const endpoint = yield* controlEndpoint(STACK_ID);
+          const candidates = yield* controlEndpointCandidates(STACK_ID);
           const unrelated = yield* Effect.acquireRelease(
-            Effect.promise(() => listenRaw(endpoint.port)),
+            Effect.promise(() => listenRaw(candidates[0]!.port)),
             (server) => Effect.promise(() => closeRaw(server)),
           );
+          const owner = yield* acquireControl({ stackId: STACK_ID });
+          if (owner._tag !== "Owned") throw new Error("expected control ownership");
+          expect(owner.endpoint.port).toBe(candidates[1]!.port);
+          expect(unrelated.listening).toBe(true);
+          const probe = yield* probeControl(STACK_ID);
+          expect(probe?.endpoint.port).toBe(candidates[1]!.port);
+        }),
+      ),
+    ),
+  );
+
+  it.live("fails once every candidate is occupied by unrelated listeners", () =>
+    live(
+      Effect.scoped(
+        Effect.gen(function* () {
+          const candidates = yield* controlEndpointCandidates(STACK_ID);
+          yield* Effect.forEach(candidates, (candidate) =>
+            Effect.acquireRelease(
+              Effect.promise(() => listenRaw(candidate.port)),
+              (server) => Effect.promise(() => closeRaw(server)),
+            ),
+          );
           const result = yield* acquireControl({
             stackId: STACK_ID,
           }).pipe(
@@ -325,7 +360,6 @@ describe("managed control endpoint", () => {
           );
           expect(result._tag).toBe("Left");
           if (result._tag === "Left") expect(result.error._tag).toBe("ControlAddressConflictError");
-          expect(unrelated.listening).toBe(true);
         }),
       ),
     ),
@@ -350,7 +384,7 @@ describe("managed control endpoint", () => {
           requestStop: () => Effect.void,
         });
         const exit = yield* acquireControl({ stackId: STACK_ID }).pipe(
-          Effect.timeout("2 seconds"),
+          Effect.timeout("10 seconds"),
           Effect.exit,
           Effect.provide(unavailable),
         );

packages/stack/src/managed-manager-lifecycle.integration.test.ts

@@ -133,7 +133,9 @@ describe("managed stack lifecycle journeys", () => {
               ? Effect.succeed(current)
               : Effect.fail(new Error("stop pending")),
           ),
-          Effect.retry(Schedule.spaced("10 millis").pipe(Schedule.upTo({ duration: "2 seconds" }))),
+          Effect.retry(
+            Schedule.spaced("10 millis").pipe(Schedule.upTo({ duration: "10 seconds" })),
+          ),
         );
         yield* owner.close;
         yield* Fiber.join(stopFiber);
@@ -179,7 +181,7 @@ describe("managed stack lifecycle journeys", () => {
         } satisfies FileSystem.FileSystem;
       }),
     ).pipe(Layer.provide(NodeFileSystem.layer));
-    const managerLayer = managedStackManagerLayer({ stateRoot }).pipe(
+    const managerLayer = managedStackManagerLayer({ stateRoot, preferCatalogDefaults: false }).pipe(
       Layer.provide(gatedFileSystemLayer),
       Layer.provide(NodePath.layer),
       Layer.provide(gitConfigStoreLayer),

packages/stack/src/managed-manager-projects.integration.test.ts

@@ -151,17 +151,23 @@ describe("managed stack projects journeys", () => {
       const baseTransport = yield* ControlTransport;
       const readStarted = yield* Deferred.make<void>();
       const continueRead = yield* Deferred.make<void>();
+      // Hold only the status probe's first read in flight. Later reads (the
+      // concurrent acquire scans its endpoint candidates before binding) must
+      // pass through, mirroring the real transport's bounded read timeout.
       let gateReads = false;
+      let gatedRead = false;
       const gatedTransport = Layer.succeed(ControlTransport, {
         ...baseTransport,
         read: (endpoint) =>
-          gateReads
-            ? Effect.gen(function* () {
-                yield* Deferred.succeed(readStarted, void 0);
-                yield* Deferred.await(continueRead);
-                return yield* baseTransport.read(endpoint);
-              })
-            : baseTransport.read(endpoint),
+          Effect.suspend(() => {
+            if (!gateReads || gatedRead) return baseTransport.read(endpoint);
+            gatedRead = true;
+            return Effect.gen(function* () {
+              yield* Deferred.succeed(readStarted, void 0);
+              yield* Deferred.await(continueRead);
+              return yield* baseTransport.read(endpoint);
+            });
+          }),
       });
 
       yield* Effect.scoped(