fix(infra): retry the VPC delete until DigitalOcean releases a cluster's subnets

src/infra/constants.ts

@@ -18,12 +18,18 @@ export const SHARED_KEY_NAME = "perfkit-shared";
 
 /** Droplets active and cloud-init finished within this, or exit 3. */
 export const READY_TIMEOUT_MS = 10 * 60 * 1000;
-/** How long teardown waits for a VPC to have no members. */
+/** How long teardown tries to delete a VPC (it waits for it to empty). */
 export const VPC_EMPTY_TIMEOUT_MS = 5 * 60 * 1000;
 export const POLL_INTERVAL_MS = 5 * 1000;
 /** Attempts per delete, and the waits between them. */
 export const DELETE_ATTEMPTS = 3;
 export const DELETE_BACKOFF_MS = [2 * 1000, 4 * 1000];
+/**
+ * The waits between VPC deletes. DigitalOcean refuses one with "active
+ * Subnets" for minutes after a DOKS cluster's delete returns, even when the
+ * VPC lists no members, so teardown keeps trying until its VPC timeout.
+ */
+export const VPC_DELETE_BACKOFF_MS = [5, 10, 20, 30, 60].map((s) => s * 1000);
 
 /** `perfkit-<run id>`: the VPC, the firewall and the run-only SSH key. */
 export function resourceNameOf(runId: string): string {
@@ -37,7 +43,7 @@ export function clusterNameOf(runId: string): string {
 
 /** A new DOKS cluster must be running within this, or exit 3. */
 export const CLUSTER_READY_TIMEOUT_MS = 20 * 60 * 1000;
-/** How long teardown waits for a VPC to empty after deleting a cluster. */
+/** How long teardown tries to delete a VPC after deleting a cluster. */
 export const VPC_EMPTY_AFTER_CLUSTER_TIMEOUT_MS = 10 * 60 * 1000;
 
 /** `perfkit-run-<run id>`: the tag both droplets of a run carry. */

src/infra/createDeleteVpc.ts

@@ -0,0 +1,85 @@
+/**
+ * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License.
+ */
+import { withErrorHandlingAndValidation } from "../errors/index.js";
+import { POLL_INTERVAL_MS, VPC_DELETE_BACKOFF_MS } from "./constants.js";
+import { messageOf } from "./errors.js";
+import { type DeleteVpcInput, deleteVpcInputSchema } from "./infra.schema.js";
+import type { DoClient, Now, Sleep } from "./types.js";
+
+type Step =
+  | { kind: "members"; members: unknown[] }
+  | { kind: "listError"; err: unknown }
+  | { kind: "deleteError"; err: unknown };
+
+/**
+ * Creates `deleteVpc({ id, timeoutMs })`: it deletes a VPC once DigitalOcean
+ * lets it, within `timeoutMs`. DigitalOcean deletes only an empty VPC, so it
+ * polls the VPC's members every 5 seconds until there are none, then deletes
+ * it. A refused delete (like "Can not delete VPC with active Subnets", which
+ * a DOKS cluster's subnets cause for minutes after the cluster is gone, with
+ * no members listed) is tried again after 5, 10, 20, 30 and then every 60
+ * seconds, until the timeout. The last try is at the deadline.
+ *
+ * @param deps - The DigitalOcean client and clock (injected)
+ * @returns Null once the VPC is deleted, or why it is not, like `after 10
+ *   minutes and 9 attempts: Can not delete VPC with active Subnets`
+ */
+export function createDeleteVpc(deps: {
+  client: DoClient;
+  sleep: Sleep;
+  now: Now;
+}) {
+  const fn = async ({
+    id,
+    timeoutMs
+  }: DeleteVpcInput): Promise<string | null> => {
+    const deadline = deps.now() + timeoutMs;
+    const minutes = timeoutMs / 60_000;
+
+    const step = async (): Promise<Step | null> => {
+      const members = await deps.client
+        .listVpcMembers(id)
+        .catch((err: unknown) => ({ err }));
+      if (!Array.isArray(members)) {
+        return { kind: "listError", err: members.err };
+      }
+      if (members.length > 0) return { kind: "members", members };
+      return deps.client.deleteVpc(id).then(
+        () => null,
+        (err: unknown) => ({ kind: "deleteError" as const, err })
+      );
+    };
+
+    const reasonOf = (last: Step, attempts: number): string => {
+      if (last.kind === "deleteError") {
+        return `after ${minutes} minutes and ${attempts} attempt${attempts === 1 ? "" : "s"}: ${messageOf(last.err)}`;
+      }
+      if (last.kind === "listError") {
+        return `after ${minutes} minutes: ${messageOf(last.err)}`;
+      }
+      const n = last.members.length;
+      return `after ${minutes} minutes: it still has ${n} member${n === 1 ? "" : "s"}`;
+    };
+
+    const run = async (attempts: number): Promise<string | null> => {
+      const last = await step();
+      if (last === null) return null;
+      const tried = attempts + (last.kind === "deleteError" ? 1 : 0);
+      const left = deadline - deps.now();
+      if (left <= 0) return reasonOf(last, tried);
+      const wait =
+        last.kind === "deleteError"
+          ? (VPC_DELETE_BACKOFF_MS[tried - 1] ?? VPC_DELETE_BACKOFF_MS.at(-1)!)
+          : POLL_INTERVAL_MS;
+      await deps.sleep(Math.min(wait, left));
+      return run(tried);
+    };
+
+    return run(0);
+  };
+
+  return withErrorHandlingAndValidation(fn, deleteVpcInputSchema) as (
+    input: DeleteVpcInput
+  ) => Promise<string | null>;
+}

src/infra/createTeardown.ts

@@ -6,10 +6,10 @@ import { withErrorHandlingAndValidation } from "../errors/index.js";
 import {
   DELETE_ATTEMPTS,
   DELETE_BACKOFF_MS,
-  POLL_INTERVAL_MS,
   VPC_EMPTY_AFTER_CLUSTER_TIMEOUT_MS,
   VPC_EMPTY_TIMEOUT_MS
 } from "./constants.js";
+import { createDeleteVpc } from "./createDeleteVpc.js";
 import { messageOf } from "./errors.js";
 import { runResourcesSchema } from "./infra.schema.js";
 import type {
@@ -31,10 +31,11 @@ type Outcome =
  * order: the firewall, the droplets and the Kubernetes cluster (with its
  * load balancers and volumes) at the same time, the run's tag, the VPC, and
  * the SSH key unless it is the shared one. Each delete is tried 3 times with
- * backoff. DigitalOcean deletes only an empty VPC, so it first polls the
- * VPC's members until there are none, for at most 5 minutes (10 after a
- * cluster); it skips the VPC when a droplet or the cluster in it could not
- * be deleted.
+ * backoff, except the VPC: DigitalOcean deletes only an empty VPC, and
+ * refuses one with a DOKS cluster's subnets for minutes after the cluster is
+ * gone, so it waits for the VPC to have no members and retries its delete
+ * with backoff for at most 5 minutes (10 after a cluster). It skips the VPC
+ * when a droplet or the cluster in it could not be deleted.
  *
  * It never throws for a failed delete: the result lists every failure, and
  * the caller exits 3 when there is one.
@@ -69,23 +70,7 @@ export function createTeardown(deps: {
     }
   };
 
-  const waitEmpty = async (
-    vpcId: string,
-    deadline: number,
-    minutes: number
-  ): Promise<string | null> => {
-    const members = await deps.client
-      .listVpcMembers(vpcId)
-      .catch((err: unknown) => err as Error);
-    if (Array.isArray(members) && members.length === 0) return null;
-    if (deps.now() >= deadline) {
-      return Array.isArray(members)
-        ? `after ${minutes} minutes: it still has ${members.length} member${members.length === 1 ? "" : "s"}`
-        : `after ${minutes} minutes: ${messageOf(members)}`;
-    }
-    await deps.sleep(POLL_INTERVAL_MS);
-    return waitEmpty(vpcId, deadline, minutes);
-  };
+  const deleteVpc = createDeleteVpc(deps);
 
   const fn = async (
     resources: z.output<typeof runResourcesSchema>
@@ -135,15 +120,11 @@ export function createTeardown(deps: {
       const waitMs = cluster
         ? VPC_EMPTY_AFTER_CLUSTER_TIMEOUT_MS
         : VPC_EMPTY_TIMEOUT_MS;
-      const notEmpty = await waitEmpty(
-        id,
-        deps.now() + waitMs,
-        waitMs / 60_000
-      );
+      const reason = await deleteVpc({ id, timeoutMs: waitMs });
       outcomes.push(
-        notEmpty === null
-          ? await attempt(resource, () => deps.client.deleteVpc(id))
-          : { ok: false, failure: { ...resource, reason: notEmpty } }
+        reason === null
+          ? { ok: true, resource }
+          : { ok: false, failure: { ...resource, reason } }
       );
     }
 

src/infra/index.ts

@@ -10,6 +10,7 @@ export { buildCloudInit } from "./buildCloudInit.js";
 export { buildFirewall } from "./buildFirewall.js";
 export * from "./constants.js";
 export { createAllowSsh } from "./createAllowSsh.js";
+export { createDeleteVpc } from "./createDeleteVpc.js";
 export { createDockerHubRegistry } from "./createDockerHubRegistry.js";
 export { createDoClient, type DoClientInput } from "./createDoClient.js";
 export { createGetPublicIp } from "./createGetPublicIp.js";

src/infra/infra.schema.ts

@@ -203,11 +203,17 @@ export const sweepInputSchema = z.strictObject({
   dryRun: z.boolean().default(false)
 });
 
+export const deleteVpcInputSchema = z.strictObject({
+  id: z.string().min(1),
+  timeoutMs: z.number().int().positive()
+});
+
 export type PlanRunInput = z.input<typeof planRunInputSchema>;
 export type ResolveImageDigestsInput = z.infer<
   typeof resolveImageDigestsInputSchema
 >;
 export type ProvisionOptions = z.input<typeof provisionOptionsSchema>;
 export type AllowSshInput = z.infer<typeof allowSshInputSchema>;
 export type SweepInput = z.input<typeof sweepInputSchema>;
+export type DeleteVpcInput = z.infer<typeof deleteVpcInputSchema>;
 export type WithRunOptions = z.input<typeof withRunInputSchema>["options"];

test/infra/createDeleteVpc.unit.test.ts

@@ -0,0 +1,132 @@
+/**
+ * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License.
+ */
+import { expect } from "chai";
+import { ValidationError } from "../../src/errors/index.js";
+import { createDeleteVpc } from "../../src/infra/index.js";
+import { FakeDigitalOcean, fakeClock, RUN_ID } from "./fakes.js";
+
+const SUBNETS = "Can not delete VPC with active Subnets";
+const TEN_MINUTES = 10 * 60 * 1000;
+
+function seeded() {
+  const { now, sleep, clock } = fakeClock();
+  const client = new FakeDigitalOcean(now);
+  client.vpcs.push({
+    id: "vpc-1",
+    name: `perfkit-${RUN_ID}`,
+    region: "nyc3",
+    ip_range: "10.0.0.0/20",
+    created_at: "2026-11-03T14:12:10Z"
+  });
+  const deleteVpc = createDeleteVpc({ client, sleep, now });
+  return { client, clock, deleteVpc };
+}
+
+describe("createDeleteVpc", () => {
+  describe("with valid input", () => {
+    it("should delete the VPC once it has no members", async () => {
+      // Arrange
+      const { client, clock, deleteVpc } = seeded();
+      client.seedDroplet({
+        name: `perfkit-${RUN_ID}-uut`,
+        tags: ["perfkit"],
+        createdAt: clock.now,
+        vpcId: "vpc-1"
+      });
+      client.lingerPolls = 2;
+      await client.deleteDroplet(client.droplets[0]!.id);
+
+      // Act
+      const reason = await deleteVpc({ id: "vpc-1", timeoutMs: TEN_MINUTES });
+
+      // Assert
+      expect(reason).to.equal(null);
+      expect(client.callsTo("listVpcMembers")).to.have.length(3);
+      expect(client.callsTo("deleteVpc")).to.deep.equal([["vpc-1"]]);
+      expect(clock.slept).to.deep.equal([5000, 5000]);
+      expect(client.leftovers().vpcs).to.deep.equal([]);
+    });
+
+    it("should retry a delete refused for active subnets with backoff", async () => {
+      // Arrange
+      const { client, clock, deleteVpc } = seeded();
+      client.failures.deleteVpc = (n) => (n <= 3 ? SUBNETS : undefined);
+
+      // Act
+      const reason = await deleteVpc({ id: "vpc-1", timeoutMs: TEN_MINUTES });
+
+      // Assert
+      expect(reason).to.equal(null);
+      expect(client.callsTo("deleteVpc")).to.have.length(4);
+      expect(clock.slept).to.deep.equal([5000, 10000, 20000]);
+      expect(client.leftovers().vpcs).to.deep.equal([]);
+    });
+
+    it("should give up on a refused delete at the timeout, trying last at the deadline", async () => {
+      // Arrange
+      const { client, clock, deleteVpc } = seeded();
+      client.failures.deleteVpc = () => SUBNETS;
+      const start = clock.now;
+
+      // Act
+      const reason = await deleteVpc({ id: "vpc-1", timeoutMs: TEN_MINUTES });
+
+      // Assert
+      expect(reason).to.equal(`after 10 minutes and 14 attempts: ${SUBNETS}`);
+      expect(client.callsTo("deleteVpc")).to.have.length(14);
+      expect(clock.now - start).to.equal(TEN_MINUTES);
+      expect(clock.slept.at(-1)).to.equal(55_000);
+    });
+
+    it("should give up when the VPC keeps its members", async () => {
+      // Arrange
+      const { client, clock, deleteVpc } = seeded();
+      client.seedDroplet({
+        name: `perfkit-${RUN_ID}-uut`,
+        tags: ["perfkit"],
+        createdAt: clock.now,
+        vpcId: "vpc-1"
+      });
+
+      // Act
+      const reason = await deleteVpc({
+        id: "vpc-1",
+        timeoutMs: 5 * 60 * 1000
+      });
+
+      // Assert
+      expect(reason).to.equal("after 5 minutes: it still has 1 member");
+      expect(client.callsTo("deleteVpc")).to.deep.equal([]);
+    });
+
+    it("should report a members lookup that keeps failing", async () => {
+      // Arrange
+      const { client, deleteVpc } = seeded();
+      client.failures.listVpcMembers = () => "Service Unavailable";
+
+      // Act
+      const reason = await deleteVpc({ id: "vpc-1", timeoutMs: 60_000 });
+
+      // Assert
+      expect(reason).to.equal("after 1 minutes: Service Unavailable");
+      expect(client.callsTo("deleteVpc")).to.deep.equal([]);
+    });
+  });
+
+  describe("with invalid input", () => {
+    it("should throw ValidationError and call nothing", async () => {
+      // Arrange
+      const { client, deleteVpc } = seeded();
+
+      // Act
+      const err = await deleteVpc({ id: "", timeoutMs: 0 }).catch(
+        (e: unknown) => e
+      );
+
+      // Assert
+      expect(err).to.be.instanceOf(ValidationError);
+      expect(client.calls).to.deep.equal([]);
+    });
+  });
+});

test/infra/createTeardown.unit.test.ts

@@ -127,6 +127,22 @@ describe("createTeardown", () => {
       expect(clock.now - start).to.equal(5 * 60 * 1000);
     });
 
+    it("should wait out a VPC delete refused for its active subnets", async () => {
+      // Arrange
+      const { client, teardown, resources } = seeded();
+      client.failures.deleteVpc = (n) =>
+        n <= 4 ? "Can not delete VPC with active Subnets" : undefined;
+
+      // Act
+      const result = await teardown(resources(false));
+
+      // Assert
+      expect(result.failures).to.deep.equal([]);
+      expect(result.deleted.map(({ kind }) => kind)).to.include("VPC");
+      expect(client.callsTo("deleteVpc")).to.have.length(5);
+      expect(client.leftovers().vpcs).to.deep.equal([]);
+    });
+
     it("should do nothing for a run that created nothing", async () => {
       // Arrange
       const { client, teardown } = seeded();