diff --git a/deploy/publish.ts b/deploy/publish.ts index a791c57..09ae51f 100644 --- a/deploy/publish.ts +++ b/deploy/publish.ts @@ -3,9 +3,11 @@ import { ProgressBar } from "@std/cli/unstable-progress-bar"; import { Spinner } from "@std/cli/unstable-spinner"; import { isAbsolute, join, relative, resolve, SEPARATOR } from "@std/path"; import { green, red, yellow } from "@std/fmt/colors"; -import { authedFetch, createTrpcClient } from "../auth.ts"; +import { authedFetch, createTrpcClient, type TRPCClient } from "../auth.ts"; import { error, + type ErrorOptions, + ExitCode, selectProductionUrl, shouldUseSpinner, writeJsonResult, @@ -14,10 +16,221 @@ import type { GlobalContext } from "../main.ts"; import type { ConfigContext } from "../config.ts"; import type { AppDetail } from "./apps.ts"; -interface Revision { +export interface Revision { labels: Record; steps: { step: string }[]; - status: "cancelled" | "failed"; + status: "building" | "routeable" | "routed" | "cancelled" | "failed"; + /** Set by the console when a failed build could be classified. */ + build_failure?: { kind: string; message: string } | null; +} + +/** + * The label the console sets on a diff-sync revision once the builder has + * compared the uploaded manifest against its cache: a JSON-encoded array of + * the hashes whose files the CLI still has to upload (`"[]"` when none). + */ +export const MISSING_HASHES_LABEL = "deno.diffsync.missing_hashes"; + +/** + * How long to wait for the builder to report which files to upload. The + * report arrives while the build is still preparing, long before any build + * timeout, so a wait this long means the build never started. + */ +export const MISSING_HASHES_TIMEOUT_MS = 10 * 60 * 1000; + +export type MissingHashesFailure = + /** The revision failed or was cancelled before the builder reported. */ + | { reason: "revision_terminal"; revision: Revision } + /** The server ended the stream without the label or a terminal status. */ + | { reason: "stream_ended"; revision: Revision | undefined } + /** No report within `timeoutMs`. */ + | { reason: "timeout"; revision: Revision | undefined; timeoutMs: number } + /** A snapshot without labels, or a label that is not a JSON array of strings. */ + | { reason: "malformed_label"; revision: Revision | undefined; value: string } + /** The subscription itself failed. */ + | { + reason: "subscription_error"; + revision: Revision | undefined; + error: unknown; + }; + +export class MissingHashesError extends Error { + constructor(readonly failure: MissingHashesFailure) { + super(`missing hashes unavailable: ${failure.reason}`); + this.name = "MissingHashesError"; + } +} + +function parseMissingHashes(value: string): string[] | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + return undefined; + } + return Array.isArray(parsed) && + parsed.every((hash) => typeof hash === "string") + ? parsed + : undefined; +} + +function isTerminal(revision: Revision): boolean { + return revision.status === "failed" || revision.status === "cancelled"; +} + +/** + * Watches the revision until the console publishes the hashes the builder is + * missing (see {@link MISSING_HASHES_LABEL}). Settles on every exit of the + * stream: the label makes it resolve; a terminal revision, the stream ending, + * a subscription error, or the timeout reject with a {@link MissingHashesError} + * naming the cause, so the deploy fails with a message instead of waiting on + * a stream that will never deliver. + */ +export function waitForMissingHashes( + trpcClient: TRPCClient, + input: { org: string; app: string; revision: string }, + { timeoutMs = MISSING_HASHES_TIMEOUT_MS }: { timeoutMs?: number } = {}, +): Promise<{ missingHashes: string[]; revision: Revision }> { + const { promise, resolve, reject } = Promise.withResolvers< + { missingHashes: string[]; revision: Revision } + >(); + let revision: Revision | undefined; + // Assigned once `subscription` returns; a callback may run before that. + let sub: { unsubscribe(): void } | undefined = undefined; + let settled = false; + + const timer = setTimeout(() => { + fail({ reason: "timeout", revision, timeoutMs }); + }, timeoutMs); + + function finish() { + settled = true; + clearTimeout(timer); + sub?.unsubscribe(); + } + function fail(failure: MissingHashesFailure) { + if (settled) return; + finish(); + reject(new MissingHashesError(failure)); + } + // The console ends the stream once the revision is terminal, so a stream + // that ends after a failed or cancelled snapshot is reported as that. + function streamEnded() { + fail( + revision !== undefined && isTerminal(revision) + ? { reason: "revision_terminal", revision } + : { reason: "stream_ended", revision }, + ); + } + + sub = trpcClient.subscription("revisions.watchUntilReady", input, { + onData: (data: unknown) => { + if (settled) return; + const labels = (data as Partial | null)?.labels; + if ( + typeof labels !== "object" || labels === null || Array.isArray(labels) + ) { + fail({ + reason: "malformed_label", + revision, + value: Deno.inspect(data), + }); + return; + } + revision = data as Revision; + if (MISSING_HASHES_LABEL in revision.labels) { + const value = revision.labels[MISSING_HASHES_LABEL]; + const missingHashes = parseMissingHashes(value); + if (missingHashes === undefined) { + fail({ reason: "malformed_label", revision, value }); + return; + } + finish(); + resolve({ missingHashes, revision }); + } else if (isTerminal(revision)) { + // The builder reports before the build runs, so a revision that ended + // without the label will never carry it. + fail({ reason: "revision_terminal", revision }); + } + }, + // The CLI's tRPC client (`auth.ts`) reports and exits on errors in its + // error link, so this only runs for a client built without that link; it + // settles the wait either way. + onError: (err: unknown) => { + fail({ reason: "subscription_error", revision, error: err }); + }, + onStopped: streamEnded, + onComplete: streamEnded, + }); + // A callback may have settled the wait before `subscription` returned. + if (settled) sub.unsubscribe(); + + return promise; +} + +/** + * The error to report for a {@link MissingHashesFailure}: a message naming + * what happened, with the revision's console URL as the hint so the build + * logs are one click away. + */ +export function describeMissingHashesFailure( + failure: MissingHashesFailure, + revisionUrl: string, +): { message: string } & ErrorOptions { + const hint = `View ${revisionUrl} for details.`; + switch (failure.reason) { + case "revision_terminal": { + const { status, build_failure } = failure.revision; + const detail = build_failure?.message ? `\n${build_failure.message}` : ""; + return { + message: `The revision ${ + status === "cancelled" ? "was cancelled" : "failed" + } before the upload could start.${detail}`, + code: ExitCode.GENERIC, + errorCode: status === "cancelled" + ? "REVISION_CANCELLED" + : "REVISION_FAILED", + hint, + }; + } + case "stream_ended": + return { + message: + "The server closed the revision stream before reporting which files need uploading.", + code: ExitCode.NETWORK, + errorCode: "DIFFSYNC_STREAM_ENDED", + hint: + `The revision may have been cancelled or superseded by another deploy. ${hint} Then retry the deploy.`, + }; + case "timeout": + return { + message: `Timed out after ${ + Math.round(failure.timeoutMs / 60_000) + } minutes waiting for the build to report which files need uploading.`, + code: ExitCode.NETWORK, + errorCode: "DIFFSYNC_TIMEOUT", + hint: + `The build may be stuck or queued. ${hint} Then retry the deploy.`, + }; + case "malformed_label": + return { + message: + `The server reported the files to upload in an unexpected format: ${failure.value}`, + code: ExitCode.NETWORK, + errorCode: "DIFFSYNC_MALFORMED_RESPONSE", + hint, + }; + case "subscription_error": + return { + message: + `The revision stream failed before reporting which files need uploading: ${ + Deno.inspect(failure.error) + }`, + code: ExitCode.NETWORK, + errorCode: "DIFFSYNC_SUBSCRIPTION_ERROR", + hint: `${hint} Then retry the deploy.`, + }; + } } type Chunk = { @@ -214,42 +427,28 @@ export async function publish( `You can view the revision here:\n ${context.endpoint}/${org}/${app}/builds/${revisionId}\n`, ); - const missingHashesPromise = Promise.withResolvers(); - const existingFilesSpinner = startSpinner( "Loading previously uploaded files...", ); - let revision: Revision | undefined = undefined; - const sub = trpcClient.subscription( - "revisions.watchUntilReady", - { + let missingHashes: string[]; + let revision: Revision | undefined; + try { + ({ missingHashes, revision } = await waitForMissingHashes(trpcClient, { org, app, revision: revisionId, - }, - { - onData: (data: unknown) => { - const typedData = data as Revision; - revision = typedData; - if ("deno.diffsync.missing_hashes" in typedData.labels) { - missingHashesPromise.resolve( - JSON.parse(typedData.labels["deno.diffsync.missing_hashes"]), - ); - sub.unsubscribe(); - } - }, - onError: (err: unknown) => { - sub.unsubscribe(); - error(context, Deno.inspect(err)); - }, - onStopped: () => { - sub.unsubscribe(); - }, - }, - ); - - const missingHashes = await missingHashesPromise.promise; + })); + } catch (err) { + existingFilesSpinner.stop(); + if (!(err instanceof MissingHashesError)) throw err; + if (context.debug) console.error(err.failure); + const { message, ...opts } = describeMissingHashesFailure( + err.failure, + `${context.endpoint}/${org}/${app}/builds/${revisionId}`, + ); + error(context, message, opts); + } existingFilesSpinner.stop(); log(`${green("✔")} Loaded previously uploaded files`); diff --git a/tests/missing_hashes.test.ts b/tests/missing_hashes.test.ts new file mode 100644 index 0000000..3a3ec70 --- /dev/null +++ b/tests/missing_hashes.test.ts @@ -0,0 +1,285 @@ +import { + assert, + assertEquals, + assertRejects, + assertStringIncludes, +} from "@std/assert"; +import type { TRPCClient } from "../auth.ts"; +import { + describeMissingHashesFailure, + MISSING_HASHES_LABEL, + MissingHashesError, + type Revision, + waitForMissingHashes, +} from "../deploy/publish.ts"; + +interface Observer { + onData: (data: unknown) => void; + onError: (err: unknown) => void; + onStopped: () => void; + onComplete: () => void; +} + +/** A tRPC client whose `revisions.watchUntilReady` stream the test drives. */ +function fakeWatchClient() { + const state = { + observer: undefined as Observer | undefined, + input: undefined as unknown, + unsubscribed: 0, + }; + const client = { + subscription(path: string, input: unknown, observer: Observer) { + assertEquals(path, "revisions.watchUntilReady"); + state.input = input; + state.observer = observer; + return { + unsubscribe() { + state.unsubscribed++; + }, + }; + }, + } as unknown as TRPCClient; + // The console ends the stream after a terminal snapshot: `stopped`, then + // `complete`, as `httpSubscriptionLink` delivers it. + const end = () => { + state.observer!.onStopped(); + state.observer!.onComplete(); + }; + return { client, state, end }; +} + +const INPUT = { org: "acme", app: "web", revision: "rev-1" }; +const URL = "https://console.deno.com/acme/web/builds/rev-1"; + +function building(labels: Record = {}): Revision { + return { labels, steps: [{ step: "preparing" }], status: "building" }; +} + +async function failure(promise: Promise) { + const err = await assertRejects(() => promise, MissingHashesError); + return err.failure; +} + +Deno.test("waitForMissingHashes: resolves when the label arrives", async () => { + const { client, state } = fakeWatchClient(); + const wait = waitForMissingHashes(client, INPUT); + assertEquals(state.input, INPUT); + state.observer!.onData(building()); + state.observer!.onData( + building({ [MISSING_HASHES_LABEL]: JSON.stringify(["a1", "b2"]) }), + ); + const { missingHashes, revision } = await wait; + assertEquals(missingHashes, ["a1", "b2"]); + assertEquals(revision.labels[MISSING_HASHES_LABEL], '["a1","b2"]'); + assertEquals(state.unsubscribed, 1); + // Whatever the stream does after that is ignored. + state.observer!.onData(building()); + state.observer!.onStopped(); + state.observer!.onError(new Error("late")); + assertEquals(state.unsubscribed, 1); +}); + +Deno.test("waitForMissingHashes: an empty report means nothing to upload", async () => { + const { client, state } = fakeWatchClient(); + const wait = waitForMissingHashes(client, INPUT); + state.observer!.onData(building({ [MISSING_HASHES_LABEL]: "[]" })); + assertEquals((await wait).missingHashes, []); +}); + +Deno.test("waitForMissingHashes: a report in the first snapshot settles before subscribe returns", async () => { + const client = { + subscription(_path: string, _input: unknown, observer: Observer) { + observer.onData(building({ [MISSING_HASHES_LABEL]: '["a1"]' })); + return { unsubscribe: () => unsubscribed++ }; + }, + } as unknown as TRPCClient; + let unsubscribed = 0; + const { missingHashes } = await waitForMissingHashes(client, INPUT); + assertEquals(missingHashes, ["a1"]); + assertEquals(unsubscribed, 1); +}); + +Deno.test("waitForMissingHashes: rejects when the stream ends without the label", async () => { + const { client, state, end } = fakeWatchClient(); + const wait = waitForMissingHashes(client, INPUT); + state.observer!.onData(building()); + end(); + const f = await failure(wait); + assertEquals(f.reason, "stream_ended"); + assertEquals(f.revision?.status, "building"); + assertEquals(state.unsubscribed, 1); +}); + +Deno.test("waitForMissingHashes: rejects when the stream ends before any snapshot", async () => { + const { client, end } = fakeWatchClient(); + const wait = waitForMissingHashes(client, INPUT); + end(); + const f = await failure(wait); + assertEquals(f, { reason: "stream_ended", revision: undefined }); +}); + +Deno.test("waitForMissingHashes: a failed snapshot rejects with the revision", async () => { + const { client, state, end } = fakeWatchClient(); + const wait = waitForMissingHashes(client, INPUT); + const failed: Revision = { + labels: {}, + steps: [{ step: "preparing" }], + status: "failed", + build_failure: { kind: "process_exit", message: "git clone failed" }, + }; + state.observer!.onData(failed); + // Rejected on the snapshot itself, not only once the stream ends. + const f = await failure(wait); + assertEquals(f.reason, "revision_terminal"); + assertEquals(f.revision, failed); + assertEquals(state.unsubscribed, 1); + end(); + assertEquals(state.unsubscribed, 1); +}); + +Deno.test("waitForMissingHashes: a terminal snapshot carrying the label still resolves", async () => { + const { client, state } = fakeWatchClient(); + const wait = waitForMissingHashes(client, INPUT); + state.observer!.onData({ + labels: { [MISSING_HASHES_LABEL]: '["a1"]' }, + steps: [{ step: "preparing" }], + status: "failed", + }); + assertEquals((await wait).missingHashes, ["a1"]); +}); + +Deno.test("waitForMissingHashes: a cancelled stream end names the revision", async () => { + const { client, state, end } = fakeWatchClient(); + const wait = waitForMissingHashes(client, INPUT); + state.observer!.onData({ ...building(), status: "cancelled" }); + end(); + const f = await failure(wait); + assertEquals(f.reason, "revision_terminal"); + assertEquals(f.revision?.status, "cancelled"); +}); + +Deno.test("waitForMissingHashes: rejects on a subscription error", async () => { + const { client, state } = fakeWatchClient(); + const wait = waitForMissingHashes(client, INPUT); + const cause = new Error("connection reset"); + state.observer!.onError(cause); + const f = await failure(wait); + assertEquals(f.reason, "subscription_error"); + assert(f.reason === "subscription_error" && f.error === cause); + assertEquals(state.unsubscribed, 1); +}); + +Deno.test("waitForMissingHashes: rejects a label that is not a string array", async () => { + for (const value of ["not json", "{}", '["a1", 2]', "null"]) { + const { client, state } = fakeWatchClient(); + const wait = waitForMissingHashes(client, INPUT); + state.observer!.onData(building({ [MISSING_HASHES_LABEL]: value })); + const f = await failure(wait); + assertEquals(f.reason, "malformed_label", value); + assert(f.reason === "malformed_label" && f.value === value); + } +}); + +Deno.test("waitForMissingHashes: rejects a snapshot without labels instead of throwing", async () => { + for ( + const data of [{ steps: [], status: "building" }, { labels: null }, { + labels: [], + status: "building", + }, null] + ) { + const { client, state } = fakeWatchClient(); + const wait = waitForMissingHashes(client, INPUT); + state.observer!.onData(data); + const f = await failure(wait); + assertEquals(f.reason, "malformed_label"); + assertEquals(state.unsubscribed, 1); + } +}); + +Deno.test("waitForMissingHashes: gives up after the timeout", async () => { + const { client, state } = fakeWatchClient(); + const wait = waitForMissingHashes(client, INPUT, { timeoutMs: 10 }); + state.observer!.onData(building()); + const f = await failure(wait); + assertEquals(f.reason, "timeout"); + assert(f.reason === "timeout" && f.timeoutMs === 10); + assertEquals(f.revision?.status, "building"); + assertEquals(state.unsubscribed, 1); + // A snapshot after the deadline is ignored. + state.observer!.onData(building({ [MISSING_HASHES_LABEL]: "[]" })); + assertEquals(state.unsubscribed, 1); +}); + +Deno.test("describeMissingHashesFailure: a failed revision reports its failure and the build URL", () => { + const described = describeMissingHashesFailure({ + reason: "revision_terminal", + revision: { + labels: {}, + steps: [], + status: "failed", + build_failure: { kind: "process_exit", message: "git clone failed" }, + }, + }, URL); + assertEquals(described.errorCode, "REVISION_FAILED"); + assertEquals(described.code, 1); + assertStringIncludes(described.message, "The revision failed"); + assertStringIncludes(described.message, "git clone failed"); + assertStringIncludes(described.hint!, URL); +}); + +Deno.test("describeMissingHashesFailure: a cancelled revision without classification", () => { + const described = describeMissingHashesFailure({ + reason: "revision_terminal", + revision: { labels: {}, steps: [], status: "cancelled" }, + }, URL); + assertEquals(described.errorCode, "REVISION_CANCELLED"); + assertEquals( + described.message, + "The revision was cancelled before the upload could start.", + ); + assertStringIncludes(described.hint!, URL); +}); + +Deno.test("describeMissingHashesFailure: stream end, timeout and errors are actionable", () => { + const ended = describeMissingHashesFailure( + { reason: "stream_ended", revision: undefined }, + URL, + ); + assertEquals(ended.errorCode, "DIFFSYNC_STREAM_ENDED"); + assertEquals(ended.code, 6); + assertStringIncludes(ended.hint!, URL); + assertStringIncludes(ended.hint!, "retry"); + + const timedOut = describeMissingHashesFailure( + { reason: "timeout", revision: undefined, timeoutMs: 10 * 60 * 1000 }, + URL, + ); + assertEquals(timedOut.errorCode, "DIFFSYNC_TIMEOUT"); + assertStringIncludes(timedOut.message, "10 minutes"); + assertStringIncludes(timedOut.hint!, URL); + + const malformed = describeMissingHashesFailure( + { + reason: "malformed_label", + revision: { labels: {}, steps: [], status: "building" }, + value: "{}", + }, + URL, + ); + assertEquals(malformed.errorCode, "DIFFSYNC_MALFORMED_RESPONSE"); + assertStringIncludes(malformed.message, "{}"); + + const errored = describeMissingHashesFailure( + { + reason: "subscription_error", + revision: undefined, + error: new Error("connection reset"), + }, + URL, + ); + assertStringIncludes(errored.message, "connection reset"); + assertEquals(errored.errorCode, "DIFFSYNC_SUBSCRIPTION_ERROR"); + assertEquals(errored.code, 6); + assertStringIncludes(errored.hint!, URL); + assertStringIncludes(errored.hint!, "retry"); +});