From e5b16a996c0affb82587835144bc00a743f4bfc2 Mon Sep 17 00:00:00 2001 From: Pedro Sanders Date: Tue, 10 Nov 2026 08:26:41 -0400 Subject: [PATCH] fix(infra): retry the VPC delete until DigitalOcean releases a cluster's subnets DigitalOcean refuses to delete a VPC with "Can not delete VPC with active Subnets" for minutes after a DOKS cluster's delete returns, even when the VPC lists no members. Teardown gave up after 3 tries over 6 seconds, so runs with a cluster ended with exit 3. createDeleteVpc waits for the VPC to have no members and retries a refused delete with backoff (5, 10, 20, 30, then every 60 seconds) until the existing VPC timeout: 5 minutes, or 10 after a cluster. Teardown and sweep both use it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WSt25UoKGj2mgDNQ4dHVrB --- src/infra/constants.ts | 10 +- src/infra/createDeleteVpc.ts | 85 +++++++++++++++ src/infra/createTeardown.ts | 41 ++------ src/infra/index.ts | 1 + src/infra/infra.schema.ts | 6 ++ test/infra/createDeleteVpc.unit.test.ts | 132 ++++++++++++++++++++++++ test/infra/createTeardown.unit.test.ts | 16 +++ 7 files changed, 259 insertions(+), 32 deletions(-) create mode 100644 src/infra/createDeleteVpc.ts create mode 100644 test/infra/createDeleteVpc.unit.test.ts diff --git a/src/infra/constants.ts b/src/infra/constants.ts index d317afa..0a4acb9 100644 --- a/src/infra/constants.ts +++ b/src/infra/constants.ts @@ -18,12 +18,18 @@ export const SHARED_KEY_NAME = "perfkit-shared"; /** Droplets active and cloud-init finished within this, or exit 3. */ export const READY_TIMEOUT_MS = 10 * 60 * 1000; -/** How long teardown waits for a VPC to have no members. */ +/** How long teardown tries to delete a VPC (it waits for it to empty). */ export const VPC_EMPTY_TIMEOUT_MS = 5 * 60 * 1000; export const POLL_INTERVAL_MS = 5 * 1000; /** Attempts per delete, and the waits between them. */ export const DELETE_ATTEMPTS = 3; export const DELETE_BACKOFF_MS = [2 * 1000, 4 * 1000]; +/** + * The waits between VPC deletes. DigitalOcean refuses one with "active + * Subnets" for minutes after a DOKS cluster's delete returns, even when the + * VPC lists no members, so teardown keeps trying until its VPC timeout. + */ +export const VPC_DELETE_BACKOFF_MS = [5, 10, 20, 30, 60].map((s) => s * 1000); /** `perfkit-`: the VPC, the firewall and the run-only SSH key. */ export function resourceNameOf(runId: string): string { @@ -37,7 +43,7 @@ export function clusterNameOf(runId: string): string { /** A new DOKS cluster must be running within this, or exit 3. */ export const CLUSTER_READY_TIMEOUT_MS = 20 * 60 * 1000; -/** How long teardown waits for a VPC to empty after deleting a cluster. */ +/** How long teardown tries to delete a VPC after deleting a cluster. */ export const VPC_EMPTY_AFTER_CLUSTER_TIMEOUT_MS = 10 * 60 * 1000; /** `perfkit-run-`: the tag both droplets of a run carry. */ diff --git a/src/infra/createDeleteVpc.ts b/src/infra/createDeleteVpc.ts new file mode 100644 index 0000000..6ecfaea --- /dev/null +++ b/src/infra/createDeleteVpc.ts @@ -0,0 +1,85 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { withErrorHandlingAndValidation } from "../errors/index.js"; +import { POLL_INTERVAL_MS, VPC_DELETE_BACKOFF_MS } from "./constants.js"; +import { messageOf } from "./errors.js"; +import { type DeleteVpcInput, deleteVpcInputSchema } from "./infra.schema.js"; +import type { DoClient, Now, Sleep } from "./types.js"; + +type Step = + | { kind: "members"; members: unknown[] } + | { kind: "listError"; err: unknown } + | { kind: "deleteError"; err: unknown }; + +/** + * Creates `deleteVpc({ id, timeoutMs })`: it deletes a VPC once DigitalOcean + * lets it, within `timeoutMs`. DigitalOcean deletes only an empty VPC, so it + * polls the VPC's members every 5 seconds until there are none, then deletes + * it. A refused delete (like "Can not delete VPC with active Subnets", which + * a DOKS cluster's subnets cause for minutes after the cluster is gone, with + * no members listed) is tried again after 5, 10, 20, 30 and then every 60 + * seconds, until the timeout. The last try is at the deadline. + * + * @param deps - The DigitalOcean client and clock (injected) + * @returns Null once the VPC is deleted, or why it is not, like `after 10 + * minutes and 9 attempts: Can not delete VPC with active Subnets` + */ +export function createDeleteVpc(deps: { + client: DoClient; + sleep: Sleep; + now: Now; +}) { + const fn = async ({ + id, + timeoutMs + }: DeleteVpcInput): Promise => { + const deadline = deps.now() + timeoutMs; + const minutes = timeoutMs / 60_000; + + const step = async (): Promise => { + const members = await deps.client + .listVpcMembers(id) + .catch((err: unknown) => ({ err })); + if (!Array.isArray(members)) { + return { kind: "listError", err: members.err }; + } + if (members.length > 0) return { kind: "members", members }; + return deps.client.deleteVpc(id).then( + () => null, + (err: unknown) => ({ kind: "deleteError" as const, err }) + ); + }; + + const reasonOf = (last: Step, attempts: number): string => { + if (last.kind === "deleteError") { + return `after ${minutes} minutes and ${attempts} attempt${attempts === 1 ? "" : "s"}: ${messageOf(last.err)}`; + } + if (last.kind === "listError") { + return `after ${minutes} minutes: ${messageOf(last.err)}`; + } + const n = last.members.length; + return `after ${minutes} minutes: it still has ${n} member${n === 1 ? "" : "s"}`; + }; + + const run = async (attempts: number): Promise => { + const last = await step(); + if (last === null) return null; + const tried = attempts + (last.kind === "deleteError" ? 1 : 0); + const left = deadline - deps.now(); + if (left <= 0) return reasonOf(last, tried); + const wait = + last.kind === "deleteError" + ? (VPC_DELETE_BACKOFF_MS[tried - 1] ?? VPC_DELETE_BACKOFF_MS.at(-1)!) + : POLL_INTERVAL_MS; + await deps.sleep(Math.min(wait, left)); + return run(tried); + }; + + return run(0); + }; + + return withErrorHandlingAndValidation(fn, deleteVpcInputSchema) as ( + input: DeleteVpcInput + ) => Promise; +} diff --git a/src/infra/createTeardown.ts b/src/infra/createTeardown.ts index 0b45e08..b088fd7 100644 --- a/src/infra/createTeardown.ts +++ b/src/infra/createTeardown.ts @@ -6,10 +6,10 @@ import { withErrorHandlingAndValidation } from "../errors/index.js"; import { DELETE_ATTEMPTS, DELETE_BACKOFF_MS, - POLL_INTERVAL_MS, VPC_EMPTY_AFTER_CLUSTER_TIMEOUT_MS, VPC_EMPTY_TIMEOUT_MS } from "./constants.js"; +import { createDeleteVpc } from "./createDeleteVpc.js"; import { messageOf } from "./errors.js"; import { runResourcesSchema } from "./infra.schema.js"; import type { @@ -31,10 +31,11 @@ type Outcome = * order: the firewall, the droplets and the Kubernetes cluster (with its * load balancers and volumes) at the same time, the run's tag, the VPC, and * the SSH key unless it is the shared one. Each delete is tried 3 times with - * backoff. DigitalOcean deletes only an empty VPC, so it first polls the - * VPC's members until there are none, for at most 5 minutes (10 after a - * cluster); it skips the VPC when a droplet or the cluster in it could not - * be deleted. + * backoff, except the VPC: DigitalOcean deletes only an empty VPC, and + * refuses one with a DOKS cluster's subnets for minutes after the cluster is + * gone, so it waits for the VPC to have no members and retries its delete + * with backoff for at most 5 minutes (10 after a cluster). It skips the VPC + * when a droplet or the cluster in it could not be deleted. * * It never throws for a failed delete: the result lists every failure, and * the caller exits 3 when there is one. @@ -69,23 +70,7 @@ export function createTeardown(deps: { } }; - const waitEmpty = async ( - vpcId: string, - deadline: number, - minutes: number - ): Promise => { - const members = await deps.client - .listVpcMembers(vpcId) - .catch((err: unknown) => err as Error); - if (Array.isArray(members) && members.length === 0) return null; - if (deps.now() >= deadline) { - return Array.isArray(members) - ? `after ${minutes} minutes: it still has ${members.length} member${members.length === 1 ? "" : "s"}` - : `after ${minutes} minutes: ${messageOf(members)}`; - } - await deps.sleep(POLL_INTERVAL_MS); - return waitEmpty(vpcId, deadline, minutes); - }; + const deleteVpc = createDeleteVpc(deps); const fn = async ( resources: z.output @@ -135,15 +120,11 @@ export function createTeardown(deps: { const waitMs = cluster ? VPC_EMPTY_AFTER_CLUSTER_TIMEOUT_MS : VPC_EMPTY_TIMEOUT_MS; - const notEmpty = await waitEmpty( - id, - deps.now() + waitMs, - waitMs / 60_000 - ); + const reason = await deleteVpc({ id, timeoutMs: waitMs }); outcomes.push( - notEmpty === null - ? await attempt(resource, () => deps.client.deleteVpc(id)) - : { ok: false, failure: { ...resource, reason: notEmpty } } + reason === null + ? { ok: true, resource } + : { ok: false, failure: { ...resource, reason } } ); } diff --git a/src/infra/index.ts b/src/infra/index.ts index d96ac53..7b936eb 100644 --- a/src/infra/index.ts +++ b/src/infra/index.ts @@ -10,6 +10,7 @@ export { buildCloudInit } from "./buildCloudInit.js"; export { buildFirewall } from "./buildFirewall.js"; export * from "./constants.js"; export { createAllowSsh } from "./createAllowSsh.js"; +export { createDeleteVpc } from "./createDeleteVpc.js"; export { createDockerHubRegistry } from "./createDockerHubRegistry.js"; export { createDoClient, type DoClientInput } from "./createDoClient.js"; export { createGetPublicIp } from "./createGetPublicIp.js"; diff --git a/src/infra/infra.schema.ts b/src/infra/infra.schema.ts index f688837..296ce02 100644 --- a/src/infra/infra.schema.ts +++ b/src/infra/infra.schema.ts @@ -203,6 +203,11 @@ export const sweepInputSchema = z.strictObject({ dryRun: z.boolean().default(false) }); +export const deleteVpcInputSchema = z.strictObject({ + id: z.string().min(1), + timeoutMs: z.number().int().positive() +}); + export type PlanRunInput = z.input; export type ResolveImageDigestsInput = z.infer< typeof resolveImageDigestsInputSchema @@ -210,4 +215,5 @@ export type ResolveImageDigestsInput = z.infer< export type ProvisionOptions = z.input; export type AllowSshInput = z.infer; export type SweepInput = z.input; +export type DeleteVpcInput = z.infer; export type WithRunOptions = z.input["options"]; diff --git a/test/infra/createDeleteVpc.unit.test.ts b/test/infra/createDeleteVpc.unit.test.ts new file mode 100644 index 0000000..06a96a5 --- /dev/null +++ b/test/infra/createDeleteVpc.unit.test.ts @@ -0,0 +1,132 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { ValidationError } from "../../src/errors/index.js"; +import { createDeleteVpc } from "../../src/infra/index.js"; +import { FakeDigitalOcean, fakeClock, RUN_ID } from "./fakes.js"; + +const SUBNETS = "Can not delete VPC with active Subnets"; +const TEN_MINUTES = 10 * 60 * 1000; + +function seeded() { + const { now, sleep, clock } = fakeClock(); + const client = new FakeDigitalOcean(now); + client.vpcs.push({ + id: "vpc-1", + name: `perfkit-${RUN_ID}`, + region: "nyc3", + ip_range: "10.0.0.0/20", + created_at: "2026-11-03T14:12:10Z" + }); + const deleteVpc = createDeleteVpc({ client, sleep, now }); + return { client, clock, deleteVpc }; +} + +describe("createDeleteVpc", () => { + describe("with valid input", () => { + it("should delete the VPC once it has no members", async () => { + // Arrange + const { client, clock, deleteVpc } = seeded(); + client.seedDroplet({ + name: `perfkit-${RUN_ID}-uut`, + tags: ["perfkit"], + createdAt: clock.now, + vpcId: "vpc-1" + }); + client.lingerPolls = 2; + await client.deleteDroplet(client.droplets[0]!.id); + + // Act + const reason = await deleteVpc({ id: "vpc-1", timeoutMs: TEN_MINUTES }); + + // Assert + expect(reason).to.equal(null); + expect(client.callsTo("listVpcMembers")).to.have.length(3); + expect(client.callsTo("deleteVpc")).to.deep.equal([["vpc-1"]]); + expect(clock.slept).to.deep.equal([5000, 5000]); + expect(client.leftovers().vpcs).to.deep.equal([]); + }); + + it("should retry a delete refused for active subnets with backoff", async () => { + // Arrange + const { client, clock, deleteVpc } = seeded(); + client.failures.deleteVpc = (n) => (n <= 3 ? SUBNETS : undefined); + + // Act + const reason = await deleteVpc({ id: "vpc-1", timeoutMs: TEN_MINUTES }); + + // Assert + expect(reason).to.equal(null); + expect(client.callsTo("deleteVpc")).to.have.length(4); + expect(clock.slept).to.deep.equal([5000, 10000, 20000]); + expect(client.leftovers().vpcs).to.deep.equal([]); + }); + + it("should give up on a refused delete at the timeout, trying last at the deadline", async () => { + // Arrange + const { client, clock, deleteVpc } = seeded(); + client.failures.deleteVpc = () => SUBNETS; + const start = clock.now; + + // Act + const reason = await deleteVpc({ id: "vpc-1", timeoutMs: TEN_MINUTES }); + + // Assert + expect(reason).to.equal(`after 10 minutes and 14 attempts: ${SUBNETS}`); + expect(client.callsTo("deleteVpc")).to.have.length(14); + expect(clock.now - start).to.equal(TEN_MINUTES); + expect(clock.slept.at(-1)).to.equal(55_000); + }); + + it("should give up when the VPC keeps its members", async () => { + // Arrange + const { client, clock, deleteVpc } = seeded(); + client.seedDroplet({ + name: `perfkit-${RUN_ID}-uut`, + tags: ["perfkit"], + createdAt: clock.now, + vpcId: "vpc-1" + }); + + // Act + const reason = await deleteVpc({ + id: "vpc-1", + timeoutMs: 5 * 60 * 1000 + }); + + // Assert + expect(reason).to.equal("after 5 minutes: it still has 1 member"); + expect(client.callsTo("deleteVpc")).to.deep.equal([]); + }); + + it("should report a members lookup that keeps failing", async () => { + // Arrange + const { client, deleteVpc } = seeded(); + client.failures.listVpcMembers = () => "Service Unavailable"; + + // Act + const reason = await deleteVpc({ id: "vpc-1", timeoutMs: 60_000 }); + + // Assert + expect(reason).to.equal("after 1 minutes: Service Unavailable"); + expect(client.callsTo("deleteVpc")).to.deep.equal([]); + }); + }); + + describe("with invalid input", () => { + it("should throw ValidationError and call nothing", async () => { + // Arrange + const { client, deleteVpc } = seeded(); + + // Act + const err = await deleteVpc({ id: "", timeoutMs: 0 }).catch( + (e: unknown) => e + ); + + // Assert + expect(err).to.be.instanceOf(ValidationError); + expect(client.calls).to.deep.equal([]); + }); + }); +}); diff --git a/test/infra/createTeardown.unit.test.ts b/test/infra/createTeardown.unit.test.ts index beb1814..8cfd9c7 100644 --- a/test/infra/createTeardown.unit.test.ts +++ b/test/infra/createTeardown.unit.test.ts @@ -127,6 +127,22 @@ describe("createTeardown", () => { expect(clock.now - start).to.equal(5 * 60 * 1000); }); + it("should wait out a VPC delete refused for its active subnets", async () => { + // Arrange + const { client, teardown, resources } = seeded(); + client.failures.deleteVpc = (n) => + n <= 4 ? "Can not delete VPC with active Subnets" : undefined; + + // Act + const result = await teardown(resources(false)); + + // Assert + expect(result.failures).to.deep.equal([]); + expect(result.deleted.map(({ kind }) => kind)).to.include("VPC"); + expect(client.callsTo("deleteVpc")).to.have.length(5); + expect(client.leftovers().vpcs).to.deep.equal([]); + }); + it("should do nothing for a run that created nothing", async () => { // Arrange const { client, teardown } = seeded();