feat: run summary in the workflow, and keep raw results when the agent fails

.github/workflows/run.yaml

@@ -123,6 +123,17 @@ jobs:
           path: .perfkit/${{ needs.start.outputs.run-id }}/raw/
           if-no-files-found: ignore
           overwrite: true
+      - name: Write the run summary
+        if: failure()
+        env:
+          RUN_ID: ${{ needs.start.outputs.run-id }}
+        run: |
+          {
+            echo "## perfkit run $RUN_ID"
+            echo
+            echo "The run failed before its report was written; see this job's log."
+            echo "The raw results downloaded so far are in the artifact perfkit-raw-$RUN_ID."
+          } >> "$GITHUB_STEP_SUMMARY"
 
   wait-2:
     name: Wait for the run (next 5 h 50 min)
@@ -160,6 +171,17 @@ jobs:
           path: .perfkit/${{ needs.start.outputs.run-id }}/raw/
           if-no-files-found: ignore
           overwrite: true
+      - name: Write the run summary
+        if: failure()
+        env:
+          RUN_ID: ${{ needs.start.outputs.run-id }}
+        run: |
+          {
+            echo "## perfkit run $RUN_ID"
+            echo
+            echo "The run failed before its report was written; see this job's log."
+            echo "The raw results downloaded so far are in the artifact perfkit-raw-$RUN_ID."
+          } >> "$GITHUB_STEP_SUMMARY"
 
   finish:
     name: Collect the results and delete the droplets
@@ -202,10 +224,28 @@ jobs:
           name: perfkit-report-${{ needs.start.outputs.run-id }}
           path: .perfkit/${{ needs.start.outputs.run-id }}/
           if-no-files-found: ignore
+      - name: Write the run summary
+        if: always()
+        env:
+          RUN_ID: ${{ needs.start.outputs.run-id }}
+        run: |
+          report=".perfkit/$RUN_ID/report.json"
+          if [ -f "$report" ]; then
+            npx -y "$PERFKIT" summary --report "$report" >> "$GITHUB_STEP_SUMMARY"
+          else
+            {
+              echo "## perfkit run $RUN_ID"
+              echo
+              echo "No report was written; see this job's log."
+              echo "The raw results are in the artifact perfkit-report-$RUN_ID."
+            } >> "$GITHUB_STEP_SUMMARY"
+          fi
 
+  # Every job above uploads the raw results it downloaded before it ends, so
+  # this job, which deletes the droplets, always runs after those uploads.
   cleanup:
     name: Delete anything the run left behind
-    needs: [start, finish]
+    needs: [start, wait-1, wait-2, finish]
     if: always() && needs.start.outputs.run-id != ''
     runs-on: ubuntu-latest
     permissions:

README.md

@@ -301,6 +301,10 @@ $ echo $?
 - **Cancelled.** SIGINT or SIGTERM stops the agent, marks every unfinished benchmark `aborted`
   (`the run was cancelled`), writes the report if one benchmark finished, deletes everything and
   exits 130. During `--wait` it only exits 130.
+- **The agent fails.** Before it exits 3, `collect` downloads the raw results of every benchmark
+  that did not finish, as they are, into `<out>/<run id>/raw/<id>/` (without a `done` marker) and
+  prints `[<id>] downloaded the partial raw results to <dir>`, so hours of a soak survive the
+  teardown. It does the same for a cancelled run before it deletes everything.
 
 | Exit code | Meaning |
 | :--- | :--- |
@@ -314,6 +318,26 @@ $ echo $?
 For GitHub, [docs/github-workflow.md](docs/github-workflow.md) has the reusable workflow that
 splits a run across jobs.
 
+## perfkit summary
+
+```sh
+perfkit summary --report <report.json>
+```
+
+Prints a report as GitHub Markdown, for a workflow's run summary: the run id, the target version
+and commit, the overall verdict, and per benchmark its verdict (with its reason when it is
+inconclusive, aborted or not run). A capacity benchmark adds its capacity limit, where the ramp
+stopped and a table of the steps (offered and placed rate, p99, failed %, ok or why it stopped); a
+soak adds its rules against their limits (memory growth per process, failed calls, restarts) and
+p99 in the first and the last hour. The reusable workflow appends it to `$GITHUB_STEP_SUMMARY`:
+
+```sh
+perfkit summary --report .perfkit/<run id>/report.json >> "$GITHUB_STEP_SUMMARY"
+```
+
+It exits 0 when it printed the summary, 1 for a usage error, and 2 when the report is invalid,
+unreadable or of another schema.
+
 ## perfkit sweep
 
 ```sh

docs/github-workflow.md

@@ -14,12 +14,18 @@ the load on its own, and later jobs only follow it:
 1. `start` runs `perfkit run --detach` and outputs the run id.
 2. `wait-1` and `wait-2` each run `perfkit collect <id> --wait --timeout 5h50m`. Exit 4 (the run
    goes on) lets the next job wait; exit 0 skips the rest of the waiting. Each uploads the raw
-   results so far as the artifact `perfkit-raw-<id>`, overwriting it.
+   results so far as the artifact `perfkit-raw-<id>`, overwriting it. When the agent fails
+   (exit 3), `collect` first downloads the raw results of the unfinished benchmarks as they are,
+   so the upload keeps them.
 3. `finish` downloads that artifact and runs `perfkit collect <id>`: it writes `report.json`,
    deletes the droplets, and uploads `.perfkit/<id>/` as `perfkit-report-<id>`. Exit 1 (a
-   benchmark did not pass) still uploads.
+   benchmark did not pass) still uploads. It then appends `perfkit summary` of the report to the
+   workflow run's summary page: the verdicts, the capacity steps and the soak rules.
 4. `cleanup` always runs `perfkit sweep --run <id>`, so a cancelled or failed workflow leaves
-   nothing billing.
+   nothing billing. It needs every job before it, so it starts only after their uploads.
+
+Where a run failed before its report, the failed job's summary says so and names the artifact
+that holds the raw results.
 
 Every job has `permissions: { contents: read }`. Each job runs on a new machine, so `collect`
 adds that machine's IP to the run's firewall before it logs in, with `PERFKIT_SSH_KEY`.

src/cli/runCli.ts

@@ -3,6 +3,7 @@
  */
 import { Command, CommanderError } from "commander";
 import { createRenderSite } from "../render/createRenderSite.js";
+import { createSummarizeReport } from "../summary/createSummarizeReport.js";
 import { createValidateFile } from "../validate/createValidateFile.js";
 import {
   DEFAULT_COLLECT_TIMEOUT,
@@ -88,7 +89,8 @@ Exit codes:
   0    the run is done (with --wait), or every benchmark passed
   1    report written, at least one benchmark not passed
   2    bad flags, a missing token or no SSH key for the run
-  3    the agent failed, or no benchmark produced results
+  3    the agent failed (the unfinished benchmarks' raw results are still
+       downloaded), or no benchmark produced results
   4    the timeout passed while the run goes on; nothing deleted
   130  cancelled; with --wait nothing deleted, otherwise everything`;
 const RENDER_HELP = `
@@ -104,13 +106,23 @@ Exit codes:
   2  the report or the history is invalid, unreadable or of another schema
   3  the history URL could not be fetched (a 404 means no history)`;
 
+const SUMMARY_HELP = `
+Prints the report as GitHub Markdown, for a workflow's run summary:
+  perfkit summary --report .perfkit/<run id>/report.json >> "$GITHUB_STEP_SUMMARY"
+
+Exit codes:
+  0  the summary was printed
+  1  usage error
+  2  the report is invalid, unreadable or of another schema`;
+
 /**
  * Runs the perfkit CLI with the given argv (as in process.argv) and returns
  * the exit code. Output goes through `deps.stdout` and `deps.stderr`.
  */
 export async function runCli(argv: string[], deps: CliDeps): Promise<number> {
   const validateFile = createValidateFile(deps);
   const renderSite = createRenderSite(deps);
+  const summarizeReport = createSummarizeReport(deps);
   let exitCode = EXIT_VALID;
   let usageExitCode = EXIT_USAGE;
 
@@ -295,6 +307,24 @@ export async function runCli(argv: string[], deps: CliDeps): Promise<number> {
       }
     );
 
+  program
+    .command("summary")
+    .description(
+      "Print a report.json as Markdown, for a GitHub workflow's run summary."
+    )
+    .requiredOption("--report <file>", "the run's report.json")
+    .addHelpText("after", SUMMARY_HELP)
+    .action(async (options: { report: string }) => {
+      const result = await summarizeReport(options);
+      if (!result.ok) {
+        result.errors.forEach((line) => deps.stderr(`${line}\n`));
+        exitCode = EXIT_INVALID;
+        return;
+      }
+      deps.stdout(result.markdown);
+      exitCode = EXIT_VALID;
+    });
+
   try {
     await program.parseAsync(argv);
     return exitCode;

src/orchestrate/createCollect.ts

@@ -18,6 +18,7 @@ import {
   MAX_FAILED_CHECKS,
   nameOf
 } from "./constants.js";
+import { downloadPartialRaw } from "./downloadPartialRaw.js";
 import { formatResultLine, formatSpan } from "./formatResultLine.js";
 import { runPaths } from "./paths.js";
 import {
@@ -55,7 +56,9 @@ type Ending = "done" | "failed" | "timeout" | "unreachable" | "cancelled";
  * agent's progress lines and downloading each benchmark's raw results as
  * soon as its `done` marker exists. With `wait` it only waits: exit 0 when
  * the run is done, 4 on timeout, 130 on a signal, 3 when the agent failed
- * or the load generator stopped answering; it never deletes. Otherwise,
+ * or the load generator stopped answering; it never deletes. When the agent
+ * failed (or, without `wait`, the run was cancelled) it first downloads the
+ * raw results of the benchmarks that did not finish, as they are. Otherwise,
  * once the run ends it writes report.json, deletes everything and exits 0
  * (every selected benchmark passed) or 1. A signal stops the agent, marks
  * unfinished benchmarks `aborted` (the run was cancelled), and still writes
@@ -188,8 +191,25 @@ export function createCollect(deps: CollectDeps) {
       );
       return EXIT_RUNNING;
     }
+    // Before giving up on a failed run, keep what its benchmarks wrote so
+    // far: the workflow uploads raw/ and then deletes the droplets.
+    const salvage = async () => {
+      const ids = await downloadPartialRaw({
+        remote: deps.remote,
+        root: deps.root,
+        runDir,
+        downloaded
+      });
+      ids.forEach((id) =>
+        deps.stdout(
+          `[${id}] downloaded the partial raw results to ${join(runDir, "raw", id)}\n`
+        )
+      );
+    };
+
     if (input.wait) {
       if (ending === "done") return EXIT_OK;
+      if (ending === "failed") await salvage();
       if (ending === "cancelled") return EXIT_CANCELLED;
       deps.stderr(
         `perfkit: ${ending === "failed" ? current?.failed : LOADGEN_UNRESPONSIVE}; nothing was deleted\n`
@@ -207,6 +227,7 @@ export function createCollect(deps: CollectDeps) {
         .catch(() => undefined);
       await read().catch(() => false);
     }
+    if (ending === "failed" || ending === "cancelled") await salvage();
 
     const finish = async (code: number): Promise<number> => {
       const { summary, failures } = await deps.teardown();

src/orchestrate/downloadPartialRaw.ts

@@ -0,0 +1,56 @@
+/**
+ * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License.
+ */
+import { mkdir, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { benchmarkIdSchema } from "../schemas/primitives.schema.js";
+import { runPaths } from "./paths.js";
+import type { Remote } from "./types.js";
+
+/**
+ * Downloads the raw results of every benchmark on the load generator that
+ * collect has not downloaded yet, finished or not, into `<runDir>/raw/<id>`.
+ * Collect calls it before it gives up on a failed or cancelled run, so the
+ * workflow's raw artifact keeps hours of results that the run's teardown
+ * would delete. A partial directory gets no `done` marker, so it never
+ * counts as finished. It never throws: a benchmark it cannot download is
+ * left out.
+ *
+ * @returns The ids it downloaded, in the load generator's order
+ */
+export async function downloadPartialRaw(input: {
+  remote: Remote;
+  root: string;
+  runDir: string;
+  downloaded: ReadonlySet<string>;
+}): Promise<string[]> {
+  const paths = runPaths(input.root);
+  const listed = await input.remote
+    .exec("loadgen", `ls -1 ${paths.raw} 2>/dev/null; true`, {
+      timeoutMs: 60 * 1000
+    })
+    .catch(() => null);
+  if (listed === null || listed.code !== 0) return [];
+  const ids = listed.stdout
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(
+      (id) =>
+        benchmarkIdSchema.safeParse(id).success && !input.downloaded.has(id)
+    );
+  return ids.reduce<Promise<string[]>>(
+    (previous, id) =>
+      previous.then(async (done) => {
+        const target = join(input.runDir, "raw", id);
+        try {
+          await rm(target, { recursive: true, force: true });
+          await mkdir(join(input.runDir, "raw"), { recursive: true });
+          await input.remote.download("loadgen", paths.rawOf(id), target);
+          return [...done, id];
+        } catch {
+          return done;
+        }
+      }),
+    Promise.resolve([])
+  );
+}

src/summary/createSummarizeReport.ts

@@ -0,0 +1,56 @@
+/**
+ * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License.
+ */
+import { withErrorHandlingAndValidation } from "../errors/withErrorHandlingAndValidation.js";
+import { reportSchema, SCHEMA_IDS } from "../schemas/index.js";
+import { createValidateFile } from "../validate/createValidateFile.js";
+import { parseDocument } from "../validate/parseDocument.js";
+import type { ValidateFileDeps } from "../validate/types.js";
+import { formatReportSummary } from "./formatReportSummary.js";
+import {
+  type SummarizeReportInput,
+  summarizeReportInputSchema
+} from "./summarizeReport.schema.js";
+
+export type SummarizeReportResult =
+  { ok: true; markdown: string } | { ok: false; errors: string[] };
+
+/**
+ * Creates the function behind `perfkit summary`: it validates a report.json
+ * and returns it as GitHub Markdown, for $GITHUB_STEP_SUMMARY.
+ *
+ * @param deps - File system access (injected)
+ * @returns A validated function that returns the Markdown, or the error
+ *   lines when the file is not a valid report
+ */
+export function createSummarizeReport(deps: ValidateFileDeps) {
+  const validateFile = createValidateFile(deps);
+
+  const fn = async ({
+    report: file
+  }: SummarizeReportInput): Promise<SummarizeReportResult> => {
+    const checked = await validateFile({ file });
+    if (!checked.valid) {
+      return {
+        ok: false,
+        errors: checked.errors.map(({ path, message }) =>
+          path === null ? `${file}: ${message}` : `${file}: ${path}: ${message}`
+        )
+      };
+    }
+    if (checked.format !== SCHEMA_IDS.report) {
+      return {
+        ok: false,
+        errors: [
+          `${file}: expected a ${SCHEMA_IDS.report} report; got ${checked.format}`
+        ]
+      };
+    }
+    const report = reportSchema.parse(
+      parseDocument(await deps.readFile(file), file)
+    );
+    return { ok: true, markdown: formatReportSummary(report) };
+  };
+
+  return withErrorHandlingAndValidation(fn, summarizeReportInputSchema);
+}