diff --git a/.github/workflows/run.yaml b/.github/workflows/run.yaml index 028a20b..6aa61d4 100644 --- a/.github/workflows/run.yaml +++ b/.github/workflows/run.yaml @@ -123,6 +123,17 @@ jobs: path: .perfkit/${{ needs.start.outputs.run-id }}/raw/ if-no-files-found: ignore overwrite: true + - name: Write the run summary + if: failure() + env: + RUN_ID: ${{ needs.start.outputs.run-id }} + run: | + { + echo "## perfkit run $RUN_ID" + echo + echo "The run failed before its report was written; see this job's log." + echo "The raw results downloaded so far are in the artifact perfkit-raw-$RUN_ID." + } >> "$GITHUB_STEP_SUMMARY" wait-2: name: Wait for the run (next 5 h 50 min) @@ -160,6 +171,17 @@ jobs: path: .perfkit/${{ needs.start.outputs.run-id }}/raw/ if-no-files-found: ignore overwrite: true + - name: Write the run summary + if: failure() + env: + RUN_ID: ${{ needs.start.outputs.run-id }} + run: | + { + echo "## perfkit run $RUN_ID" + echo + echo "The run failed before its report was written; see this job's log." + echo "The raw results downloaded so far are in the artifact perfkit-raw-$RUN_ID." + } >> "$GITHUB_STEP_SUMMARY" finish: name: Collect the results and delete the droplets @@ -202,10 +224,28 @@ jobs: name: perfkit-report-${{ needs.start.outputs.run-id }} path: .perfkit/${{ needs.start.outputs.run-id }}/ if-no-files-found: ignore + - name: Write the run summary + if: always() + env: + RUN_ID: ${{ needs.start.outputs.run-id }} + run: | + report=".perfkit/$RUN_ID/report.json" + if [ -f "$report" ]; then + npx -y "$PERFKIT" summary --report "$report" >> "$GITHUB_STEP_SUMMARY" + else + { + echo "## perfkit run $RUN_ID" + echo + echo "No report was written; see this job's log." + echo "The raw results are in the artifact perfkit-report-$RUN_ID." + } >> "$GITHUB_STEP_SUMMARY" + fi + # Every job above uploads the raw results it downloaded before it ends, so + # this job, which deletes the droplets, always runs after those uploads. cleanup: name: Delete anything the run left behind - needs: [start, finish] + needs: [start, wait-1, wait-2, finish] if: always() && needs.start.outputs.run-id != '' runs-on: ubuntu-latest permissions: diff --git a/README.md b/README.md index 87ac7a9..f9ec332 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,10 @@ $ echo $? - **Cancelled.** SIGINT or SIGTERM stops the agent, marks every unfinished benchmark `aborted` (`the run was cancelled`), writes the report if one benchmark finished, deletes everything and exits 130. During `--wait` it only exits 130. +- **The agent fails.** Before it exits 3, `collect` downloads the raw results of every benchmark + that did not finish, as they are, into `//raw//` (without a `done` marker) and + prints `[] downloaded the partial raw results to `, so hours of a soak survive the + teardown. It does the same for a cancelled run before it deletes everything. | Exit code | Meaning | | :--- | :--- | @@ -314,6 +318,26 @@ $ echo $? For GitHub, [docs/github-workflow.md](docs/github-workflow.md) has the reusable workflow that splits a run across jobs. +## perfkit summary + +```sh +perfkit summary --report +``` + +Prints a report as GitHub Markdown, for a workflow's run summary: the run id, the target version +and commit, the overall verdict, and per benchmark its verdict (with its reason when it is +inconclusive, aborted or not run). A capacity benchmark adds its capacity limit, where the ramp +stopped and a table of the steps (offered and placed rate, p99, failed %, ok or why it stopped); a +soak adds its rules against their limits (memory growth per process, failed calls, restarts) and +p99 in the first and the last hour. The reusable workflow appends it to `$GITHUB_STEP_SUMMARY`: + +```sh +perfkit summary --report .perfkit//report.json >> "$GITHUB_STEP_SUMMARY" +``` + +It exits 0 when it printed the summary, 1 for a usage error, and 2 when the report is invalid, +unreadable or of another schema. + ## perfkit sweep ```sh diff --git a/docs/github-workflow.md b/docs/github-workflow.md index 78ae2d0..2436e2b 100644 --- a/docs/github-workflow.md +++ b/docs/github-workflow.md @@ -14,12 +14,18 @@ the load on its own, and later jobs only follow it: 1. `start` runs `perfkit run --detach` and outputs the run id. 2. `wait-1` and `wait-2` each run `perfkit collect --wait --timeout 5h50m`. Exit 4 (the run goes on) lets the next job wait; exit 0 skips the rest of the waiting. Each uploads the raw - results so far as the artifact `perfkit-raw-`, overwriting it. + results so far as the artifact `perfkit-raw-`, overwriting it. When the agent fails + (exit 3), `collect` first downloads the raw results of the unfinished benchmarks as they are, + so the upload keeps them. 3. `finish` downloads that artifact and runs `perfkit collect `: it writes `report.json`, deletes the droplets, and uploads `.perfkit//` as `perfkit-report-`. Exit 1 (a - benchmark did not pass) still uploads. + benchmark did not pass) still uploads. It then appends `perfkit summary` of the report to the + workflow run's summary page: the verdicts, the capacity steps and the soak rules. 4. `cleanup` always runs `perfkit sweep --run `, so a cancelled or failed workflow leaves - nothing billing. + nothing billing. It needs every job before it, so it starts only after their uploads. + +Where a run failed before its report, the failed job's summary says so and names the artifact +that holds the raw results. Every job has `permissions: { contents: read }`. Each job runs on a new machine, so `collect` adds that machine's IP to the run's firewall before it logs in, with `PERFKIT_SSH_KEY`. diff --git a/src/cli/runCli.ts b/src/cli/runCli.ts index 976be25..735bd2d 100644 --- a/src/cli/runCli.ts +++ b/src/cli/runCli.ts @@ -3,6 +3,7 @@ */ import { Command, CommanderError } from "commander"; import { createRenderSite } from "../render/createRenderSite.js"; +import { createSummarizeReport } from "../summary/createSummarizeReport.js"; import { createValidateFile } from "../validate/createValidateFile.js"; import { DEFAULT_COLLECT_TIMEOUT, @@ -88,7 +89,8 @@ Exit codes: 0 the run is done (with --wait), or every benchmark passed 1 report written, at least one benchmark not passed 2 bad flags, a missing token or no SSH key for the run - 3 the agent failed, or no benchmark produced results + 3 the agent failed (the unfinished benchmarks' raw results are still + downloaded), or no benchmark produced results 4 the timeout passed while the run goes on; nothing deleted 130 cancelled; with --wait nothing deleted, otherwise everything`; const RENDER_HELP = ` @@ -104,6 +106,15 @@ Exit codes: 2 the report or the history is invalid, unreadable or of another schema 3 the history URL could not be fetched (a 404 means no history)`; +const SUMMARY_HELP = ` +Prints the report as GitHub Markdown, for a workflow's run summary: + perfkit summary --report .perfkit//report.json >> "$GITHUB_STEP_SUMMARY" + +Exit codes: + 0 the summary was printed + 1 usage error + 2 the report is invalid, unreadable or of another schema`; + /** * Runs the perfkit CLI with the given argv (as in process.argv) and returns * the exit code. Output goes through `deps.stdout` and `deps.stderr`. @@ -111,6 +122,7 @@ Exit codes: export async function runCli(argv: string[], deps: CliDeps): Promise { const validateFile = createValidateFile(deps); const renderSite = createRenderSite(deps); + const summarizeReport = createSummarizeReport(deps); let exitCode = EXIT_VALID; let usageExitCode = EXIT_USAGE; @@ -295,6 +307,24 @@ export async function runCli(argv: string[], deps: CliDeps): Promise { } ); + program + .command("summary") + .description( + "Print a report.json as Markdown, for a GitHub workflow's run summary." + ) + .requiredOption("--report ", "the run's report.json") + .addHelpText("after", SUMMARY_HELP) + .action(async (options: { report: string }) => { + const result = await summarizeReport(options); + if (!result.ok) { + result.errors.forEach((line) => deps.stderr(`${line}\n`)); + exitCode = EXIT_INVALID; + return; + } + deps.stdout(result.markdown); + exitCode = EXIT_VALID; + }); + try { await program.parseAsync(argv); return exitCode; diff --git a/src/orchestrate/createCollect.ts b/src/orchestrate/createCollect.ts index d1a5dd1..e4ac493 100644 --- a/src/orchestrate/createCollect.ts +++ b/src/orchestrate/createCollect.ts @@ -18,6 +18,7 @@ import { MAX_FAILED_CHECKS, nameOf } from "./constants.js"; +import { downloadPartialRaw } from "./downloadPartialRaw.js"; import { formatResultLine, formatSpan } from "./formatResultLine.js"; import { runPaths } from "./paths.js"; import { @@ -55,7 +56,9 @@ type Ending = "done" | "failed" | "timeout" | "unreachable" | "cancelled"; * agent's progress lines and downloading each benchmark's raw results as * soon as its `done` marker exists. With `wait` it only waits: exit 0 when * the run is done, 4 on timeout, 130 on a signal, 3 when the agent failed - * or the load generator stopped answering; it never deletes. Otherwise, + * or the load generator stopped answering; it never deletes. When the agent + * failed (or, without `wait`, the run was cancelled) it first downloads the + * raw results of the benchmarks that did not finish, as they are. Otherwise, * once the run ends it writes report.json, deletes everything and exits 0 * (every selected benchmark passed) or 1. A signal stops the agent, marks * unfinished benchmarks `aborted` (the run was cancelled), and still writes @@ -188,8 +191,25 @@ export function createCollect(deps: CollectDeps) { ); return EXIT_RUNNING; } + // Before giving up on a failed run, keep what its benchmarks wrote so + // far: the workflow uploads raw/ and then deletes the droplets. + const salvage = async () => { + const ids = await downloadPartialRaw({ + remote: deps.remote, + root: deps.root, + runDir, + downloaded + }); + ids.forEach((id) => + deps.stdout( + `[${id}] downloaded the partial raw results to ${join(runDir, "raw", id)}\n` + ) + ); + }; + if (input.wait) { if (ending === "done") return EXIT_OK; + if (ending === "failed") await salvage(); if (ending === "cancelled") return EXIT_CANCELLED; deps.stderr( `perfkit: ${ending === "failed" ? current?.failed : LOADGEN_UNRESPONSIVE}; nothing was deleted\n` @@ -207,6 +227,7 @@ export function createCollect(deps: CollectDeps) { .catch(() => undefined); await read().catch(() => false); } + if (ending === "failed" || ending === "cancelled") await salvage(); const finish = async (code: number): Promise => { const { summary, failures } = await deps.teardown(); diff --git a/src/orchestrate/downloadPartialRaw.ts b/src/orchestrate/downloadPartialRaw.ts new file mode 100644 index 0000000..ef418b8 --- /dev/null +++ b/src/orchestrate/downloadPartialRaw.ts @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { mkdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { benchmarkIdSchema } from "../schemas/primitives.schema.js"; +import { runPaths } from "./paths.js"; +import type { Remote } from "./types.js"; + +/** + * Downloads the raw results of every benchmark on the load generator that + * collect has not downloaded yet, finished or not, into `/raw/`. + * Collect calls it before it gives up on a failed or cancelled run, so the + * workflow's raw artifact keeps hours of results that the run's teardown + * would delete. A partial directory gets no `done` marker, so it never + * counts as finished. It never throws: a benchmark it cannot download is + * left out. + * + * @returns The ids it downloaded, in the load generator's order + */ +export async function downloadPartialRaw(input: { + remote: Remote; + root: string; + runDir: string; + downloaded: ReadonlySet; +}): Promise { + const paths = runPaths(input.root); + const listed = await input.remote + .exec("loadgen", `ls -1 ${paths.raw} 2>/dev/null; true`, { + timeoutMs: 60 * 1000 + }) + .catch(() => null); + if (listed === null || listed.code !== 0) return []; + const ids = listed.stdout + .split("\n") + .map((line) => line.trim()) + .filter( + (id) => + benchmarkIdSchema.safeParse(id).success && !input.downloaded.has(id) + ); + return ids.reduce>( + (previous, id) => + previous.then(async (done) => { + const target = join(input.runDir, "raw", id); + try { + await rm(target, { recursive: true, force: true }); + await mkdir(join(input.runDir, "raw"), { recursive: true }); + await input.remote.download("loadgen", paths.rawOf(id), target); + return [...done, id]; + } catch { + return done; + } + }), + Promise.resolve([]) + ); +} diff --git a/src/summary/createSummarizeReport.ts b/src/summary/createSummarizeReport.ts new file mode 100644 index 0000000..4dd10af --- /dev/null +++ b/src/summary/createSummarizeReport.ts @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { withErrorHandlingAndValidation } from "../errors/withErrorHandlingAndValidation.js"; +import { reportSchema, SCHEMA_IDS } from "../schemas/index.js"; +import { createValidateFile } from "../validate/createValidateFile.js"; +import { parseDocument } from "../validate/parseDocument.js"; +import type { ValidateFileDeps } from "../validate/types.js"; +import { formatReportSummary } from "./formatReportSummary.js"; +import { + type SummarizeReportInput, + summarizeReportInputSchema +} from "./summarizeReport.schema.js"; + +export type SummarizeReportResult = + { ok: true; markdown: string } | { ok: false; errors: string[] }; + +/** + * Creates the function behind `perfkit summary`: it validates a report.json + * and returns it as GitHub Markdown, for $GITHUB_STEP_SUMMARY. + * + * @param deps - File system access (injected) + * @returns A validated function that returns the Markdown, or the error + * lines when the file is not a valid report + */ +export function createSummarizeReport(deps: ValidateFileDeps) { + const validateFile = createValidateFile(deps); + + const fn = async ({ + report: file + }: SummarizeReportInput): Promise => { + const checked = await validateFile({ file }); + if (!checked.valid) { + return { + ok: false, + errors: checked.errors.map(({ path, message }) => + path === null ? `${file}: ${message}` : `${file}: ${path}: ${message}` + ) + }; + } + if (checked.format !== SCHEMA_IDS.report) { + return { + ok: false, + errors: [ + `${file}: expected a ${SCHEMA_IDS.report} report; got ${checked.format}` + ] + }; + } + const report = reportSchema.parse( + parseDocument(await deps.readFile(file), file) + ); + return { ok: true, markdown: formatReportSummary(report) }; + }; + + return withErrorHandlingAndValidation(fn, summarizeReportInputSchema); +} diff --git a/src/summary/formatCapacitySummary.ts b/src/summary/formatCapacitySummary.ts new file mode 100644 index 0000000..6680a21 --- /dev/null +++ b/src/summary/formatCapacitySummary.ts @@ -0,0 +1,89 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { CapacityBenchmarkResult, Step } from "../schemas/index.js"; +import { cell, num, pct, table } from "./markdown.js"; + +/** + * The body of one capacity benchmark in the run summary: the capacity limit + * and the step table (offered and placed rate, p99, failed %, and whether + * the step was ok or where the ramp stopped). The heading and the verdict + * line are the caller's. + */ +export function formatCapacitySummary( + benchmark: CapacityBenchmarkResult +): string { + const unit = benchmark.unit; + const rate = (value: number) => `${num(value)} ${unit}/s`; + const parts: string[] = []; + + if (benchmark.verdict === "passed" || benchmark.verdict === "failed") { + const { headline, stoppedBy } = benchmark; + parts.push( + headline.limitCps === null + ? "**Capacity limit:** none, the first step already broke a rule." + : `**Capacity limit:** ${headline.limitIsLowerBound ? "at least " : ""}${rate(headline.limitCps)}, p99 ${num(headline.p99AtLimitMs ?? 0)} ms, ${pct(headline.failedAtLimitPct ?? 0)} failed${headline.limitIsLowerBound ? " (no step broke a rule)" : ""}.` + ); + if (stoppedBy) { + parts.push( + `**Stopped at** ${rate(stoppedBy.offeredCps)}: ${ruleText(stoppedBy.rule, stoppedBy.measured, stoppedBy.limit)}.` + ); + } + } + + const steps = benchmark.verdict === "not-run" ? [] : benchmark.series?.steps; + if (steps && steps.length > 0) { + const limits = + benchmark.verdict === "passed" || benchmark.verdict === "failed" + ? { + p99: benchmark.rules.find(({ id }) => id === "p99LatencyMs")?.limit, + failed: benchmark.rules.find(({ id }) => id === "failedCallsPct") + ?.limit + } + : { p99: undefined, failed: undefined }; + parts.push( + table( + [ + `Offered (${unit}/s)`, + `Placed (${unit}/s)`, + "p99", + "Failed", + "Result" + ], + steps.map((step) => [ + num(step.offeredCps), + num(step.achievedCps), + `${num(step.p99Ms)} ms`, + pct(failedPct(step)), + cell(resultOf(step, limits)) + ]), + [true, true, true, true, false] + ) + ); + } + return parts.join("\n\n"); +} + +function failedPct(step: Step): number { + return step.calls > 0 ? (step.failed / step.calls) * 100 : 0; +} + +function resultOf( + step: Step, + limits: { p99: number | undefined; failed: number | undefined } +): string { + if (!step.valid) return "invalid; stopped"; + if (limits.p99 !== undefined && step.p99Ms >= limits.p99) { + return `stopped: p99 over ${num(limits.p99)} ms`; + } + if (limits.failed !== undefined && failedPct(step) >= limits.failed) { + return `stopped: failed over ${pct(limits.failed)}`; + } + return "ok"; +} + +function ruleText(rule: string, measured: number, limit: number): string { + return rule === "p99LatencyMs" + ? `p99 ${num(measured)} ms, limit ${num(limit)} ms` + : `${pct(measured)} failed, limit ${pct(limit)}`; +} diff --git a/src/summary/formatReportSummary.ts b/src/summary/formatReportSummary.ts new file mode 100644 index 0000000..0f224a0 --- /dev/null +++ b/src/summary/formatReportSummary.ts @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { BenchmarkResult, Report } from "../schemas/index.js"; +import { formatCapacitySummary } from "./formatCapacitySummary.js"; +import { formatSoakSummary } from "./formatSoakSummary.js"; +import { capitalized, cell, duration, verdictWord } from "./markdown.js"; + +/** + * A report.json as GitHub Markdown, for a workflow's run summary: the run + * (id, target version, commit, machines), the overall verdict, and per + * benchmark its verdict (with the reason when it has one), then the + * capacity limit and steps or the soak rules. + */ +export function formatReportSummary(report: Report): string { + const { run, verdict } = report; + const others = ( + [ + ["failed", verdict.failed], + ["inconclusive", verdict.inconclusive], + ["aborted", verdict.aborted], + ["not run", verdict.notRun] + ] as const + ) + .filter(([, n]) => n > 0) + .map(([word, n]) => `${n} ${word}`); + const header = [ + `## perfkit run ${run.id}`, + [ + `**${cell(report.project.name)} ${cell(run.version)}**, commit \`${run.commit}\`.`, + `**${verdict.passed} of ${verdict.total} benchmark${verdict.total === 1 ? "" : "s"} passed**${others.length > 0 ? ` (${others.join(", ")})` : ""}.` + ].join(" "), + `Started ${run.startedAt}, ended ${run.endedAt}; ${run.infra.provider} ${run.infra.region}, server under test ${run.infra.uut.size}, load generator ${run.infra.loadgen.size}; perfkit ${run.tools.perfkit}.` + ]; + return `${[...header, ...report.benchmarks.map(formatBenchmark)].join("\n\n")}\n`; +} + +function formatBenchmark(benchmark: BenchmarkResult): string { + const heading = `### ${cell(benchmark.title)} (\`${benchmark.id}\`): ${verdictWord(benchmark.verdict)}`; + const about = + benchmark.verdict === "not-run" + ? `${benchmark.kind === "capacity" ? "Capacity" : "Soak"} benchmark.` + : `${benchmark.kind === "capacity" ? "Capacity" : "Soak"} benchmark, ${duration(benchmark.durationS)}, ${cell(benchmark.tool)}.`; + const reason = + benchmark.reason === null + ? [] + : [ + `**${capitalized(verdictWord(benchmark.verdict))}:** ${cell(benchmark.reason)}.` + ]; + const body = + benchmark.kind === "capacity" + ? formatCapacitySummary(benchmark) + : formatSoakSummary(benchmark); + return [heading, about, ...reason, ...(body === "" ? [] : [body])].join( + "\n\n" + ); +} diff --git a/src/summary/formatSoakSummary.ts b/src/summary/formatSoakSummary.ts new file mode 100644 index 0000000..9e98676 --- /dev/null +++ b/src/summary/formatSoakSummary.ts @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { SoakBenchmarkResult, SoakRule } from "../schemas/index.js"; +import { cell, growth, num, pct, table } from "./markdown.js"; + +/** + * The body of one soak benchmark in the run summary: its rules (memory + * growth per process, failed calls, restarts) against their limits, and p99 + * in the first and the last hour. The heading and the verdict line are the + * caller's. + */ +export function formatSoakSummary(benchmark: SoakBenchmarkResult): string { + if (benchmark.verdict !== "passed" && benchmark.verdict !== "failed") { + return ""; + } + const unit = benchmark.unit; + const rules = table( + ["Rule", "Measured", "Limit", "Result"], + benchmark.rules.map((rule) => [ + cell(ruleName(rule, unit)), + measuredOf(rule), + limitOf(rule), + rule.passed ? "passed" : "failed" + ]), + [false, true, true, false] + ); + const { reported, headline, load } = benchmark; + return [ + `**Load:** ${num(load.cps)} ${unit}/s, each held ${num(load.holdS)} s; ${num(headline.calls, 0)} ${unit} in all.`, + rules, + `**p99:** ${num(reported.p99FirstHourMs)} ms in the first hour, ${num(reported.p99LastHourMs)} ms in the last hour.` + ].join("\n\n"); +} + +function ruleName(rule: SoakRule, unit: string): string { + if (rule.id === "memoryGrowthMBPerHour") { + return `Memory growth, ${rule.process}`; + } + return rule.id === "failedCallsPct" ? `Failed ${unit}` : "Restarts"; +} + +function measuredOf(rule: SoakRule): string { + if (rule.id === "memoryGrowthMBPerHour") return growth(rule.measured); + return rule.id === "failedCallsPct" + ? pct(rule.measured) + : num(rule.measured, 0); +} + +function limitOf(rule: SoakRule): string { + if (rule.id === "memoryGrowthMBPerHour") return `${num(rule.limit)} MB/h`; + return rule.id === "failedCallsPct" ? pct(rule.limit) : num(rule.limit, 0); +} diff --git a/src/summary/markdown.ts b/src/summary/markdown.ts new file mode 100644 index 0000000..dd44e99 --- /dev/null +++ b/src/summary/markdown.ts @@ -0,0 +1,73 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +/** + * How the run summary writes Markdown and numbers. It is plain GitHub + * Markdown, for $GITHUB_STEP_SUMMARY, so it shares nothing with the HTML + * page's formatting. + */ +import type { Verdict } from "../schemas/index.js"; + +const LABELS: Record = { + passed: "passed", + failed: "failed", + inconclusive: "inconclusive", + aborted: "aborted", + "not-run": "not run" +}; + +/** `passed`, `not run` */ +export function verdictWord(verdict: Verdict): string { + return LABELS[verdict]; +} + +/** `Not run` */ +export function capitalized(text: string): string { + return `${text.charAt(0).toUpperCase()}${text.slice(1)}`; +} + +/** Text safe inside a table cell or a heading: no pipes, no line breaks. */ +export function cell(text: string): string { + return text.replace(/\|/g, "\\|").replace(/\s*\n\s*/g, " "); +} + +/** `1,240`, `0.03`; at most `digits` decimals. */ +export function num(value: number, digits = 1): string { + return value.toLocaleString("en-US", { + maximumFractionDigits: digits, + minimumFractionDigits: 0 + }); +} + +/** `0.03%` */ +export function pct(value: number): string { + return `${num(value, 2)}%`; +} + +/** `+7.4 MB/h` */ +export function growth(value: number): string { + return `${value < 0 ? "-" : "+"}${num(Math.abs(value))} MB/h`; +} + +/** `45 s`, `26 min`, `6 h 30 min` */ +export function duration(seconds: number): string { + if (seconds < 60) return `${Math.round(seconds)} s`; + const minutes = Math.round(seconds / 60); + const hours = Math.floor(minutes / 60); + if (hours === 0) return `${minutes} min`; + return minutes % 60 === 0 ? `${hours} h` : `${hours} h ${minutes % 60} min`; +} + +/** A GitHub Markdown table; numeric columns are right-aligned. */ +export function table( + header: string[], + rows: string[][], + right: boolean[] +): string { + const line = (cells: string[]) => `| ${cells.join(" | ")} |`; + return [ + line(header), + line(right.map((r) => (r ? "---:" : ":---"))), + ...rows.map(line) + ].join("\n"); +} diff --git a/src/summary/summarizeReport.schema.ts b/src/summary/summarizeReport.schema.ts new file mode 100644 index 0000000..f785e6d --- /dev/null +++ b/src/summary/summarizeReport.schema.ts @@ -0,0 +1,10 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { z } from "zod/v4"; + +export const summarizeReportInputSchema = z.strictObject({ + report: z.string().min(1, { error: "report must not be empty" }) +}); + +export type SummarizeReportInput = z.infer; diff --git a/test/cli/runCli.unit.test.ts b/test/cli/runCli.unit.test.ts index 8861b07..335707c 100644 --- a/test/cli/runCli.unit.test.ts +++ b/test/cli/runCli.unit.test.ts @@ -298,3 +298,45 @@ describe("perfkit validate", () => { }); }); }); + +describe("perfkit summary", () => { + it("should print the report as Markdown with exit 0", async () => { + // Act + const result = await run(["summary", "--report", "report.json"], { + "report.json": loadReport("failed") + }); + + // Assert + expect(result.code).to.equal(0); + expect(result.err).to.equal(""); + expect(result.out).to.match(/^## perfkit run 20261020T100005Z-c0b54ab\n/); + expect(result.out).to.include( + "### EdgePort + Echo (`edgeport-echo`): failed" + ); + }); + + it("should print the errors of an invalid report and exit 2", async () => { + // Arrange + const report = loadReport("failed"); + report.verdict.passed = 3; + + // Act + const result = await run(["summary", "--report", "report.json"], { + "report.json": report + }); + + // Assert + expect(result.code).to.equal(2); + expect(result.out).to.equal(""); + expect(result.err).to.match(/^report\.json: verdict/); + }); + + it("should exit 1 without --report", async () => { + // Act + const result = await run(["summary"]); + + // Assert + expect(result.code).to.equal(1); + expect(result.err).to.include("--report "); + }); +}); diff --git a/test/orchestrate/createCollect.unit.test.ts b/test/orchestrate/createCollect.unit.test.ts index 88be2d0..6c087a0 100644 --- a/test/orchestrate/createCollect.unit.test.ts +++ b/test/orchestrate/createCollect.unit.test.ts @@ -3,7 +3,7 @@ */ import { expect } from "chai"; import { EventEmitter } from "node:events"; -import { readFile, writeFile } from "node:fs/promises"; +import { readdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import sinon from "sinon"; import { createNodeAnalyzeDeps } from "../../src/analyze/index.js"; @@ -98,6 +98,37 @@ async function reportOf(dir: string, runId: string) { ); } +const AGENT_FAILED = "the agent failed: Maximum call stack size exceeded"; + +/** + * The half-done run, after its agent failed in the soak: the soak's raw + * directory holds hours of results but no `done` marker. + */ +async function agentFailed() { + const t = await halfDone(); + const root = t.setup.hosts.roots.loadgen; + await writeRaw(join(root, "raw", "one-soak"), { + "sampler-uut.jsonl": '{"t":"2026-11-03T14:20:00Z"}\n', + "uac_stat.csv": "StartTime;\n" + }); + await writeFile( + join(root, "status.json"), + JSON.stringify({ + benchmarks: { + "edgeport-echo": { state: "done" }, + "one-soak": { + state: "running", + startedAt: "2026-11-03T14:20:00Z", + plannedS: 23400 + } + }, + done: false, + failed: AGENT_FAILED + }) + ); + return t; +} + describe("createCollect", () => { afterEach(() => { sinon.restore(); @@ -212,6 +243,89 @@ describe("createCollect", () => { expect(t.teardown.callCount).to.equal(0); }); + it("should download the soak's partial raw results, delete nothing and exit 3 when the agent failed during --wait", async () => { + // Arrange + const t = await agentFailed(); + + // Act + const code = await t.collect()({ + runId: t.runId, + wait: true, + out: t.setup.out, + timeoutMs: null + }); + + // Assert + expect(code).to.equal(3); + const raw = join(t.setup.out, t.runId, "raw"); + expect((await readdir(raw)).sort()).to.deep.equal([ + "edgeport-echo", + "one-soak" + ]); + expect((await readdir(join(raw, "one-soak"))).sort()).to.deep.equal([ + "sampler-uut.jsonl", + "uac_stat.csv" + ]); + expect(t.out).to.include( + `[one-soak] downloaded the partial raw results to ${join(raw, "one-soak")}\n` + ); + expect(t.err).to.deep.equal([ + `perfkit: ${AGENT_FAILED}; nothing was deleted\n` + ]); + expect(t.teardown.callCount).to.equal(0); + }); + + it("should download the partial raw results before it deletes everything when the agent failed", async () => { + // Arrange + const t = await agentFailed(); + const raw = join(t.setup.out, t.runId, "raw"); + let atTeardown: string[] = []; + t.teardown.callsFake(async () => { + atTeardown = (await readdir(raw)).sort(); + return { summary: "deleted 2 droplets", failures: [] }; + }); + + // Act + const code = await t.collect()({ + runId: t.runId, + out: t.setup.out, + timeoutMs: null + }); + + // Assert + expect(code).to.equal(3); + expect(atTeardown).to.deep.equal(["edgeport-echo", "one-soak"]); + expect(t.err).to.deep.equal([ + `perfkit: ${AGENT_FAILED}; no report was written\n` + ]); + expect(t.teardown.callCount).to.equal(1); + }); + + it("should still exit 3 when the partial raw results cannot be downloaded", async () => { + // Arrange + const t = await agentFailed(); + const download = t.setup.hosts.download; + t.setup.hosts.download = async (host, remotePath, localPath) => { + if (remotePath.endsWith("/one-soak")) throw new Error("scp failed"); + return download(host, remotePath, localPath); + }; + + // Act + const code = await t.collect()({ + runId: t.runId, + wait: true, + out: t.setup.out, + timeoutMs: null + }); + + // Assert + expect(code).to.equal(3); + expect( + t.out.some((line) => line.includes("downloaded the partial raw")) + ).to.equal(false); + expect(t.teardown.callCount).to.equal(0); + }); + it("should throw ValidationError for a bad run id and read nothing", async () => { // Arrange const t = await halfDone(); diff --git a/test/orchestrate/downloadPartialRaw.unit.test.ts b/test/orchestrate/downloadPartialRaw.unit.test.ts new file mode 100644 index 0000000..ebca485 --- /dev/null +++ b/test/orchestrate/downloadPartialRaw.unit.test.ts @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { mkdtemp, readdir, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { REMOTE_ROOT } from "../../src/orchestrate/constants.js"; +import { downloadPartialRaw } from "../../src/orchestrate/downloadPartialRaw.js"; +import type { Remote } from "../../src/orchestrate/types.js"; +import { FakeHosts, writeRaw } from "./harness.js"; + +async function setup() { + const hosts = await FakeHosts.create(); + const raw = join(hosts.roots.loadgen, "raw"); + await writeRaw(join(raw, "edgeport-echo"), { done: "x\n" }); + await writeRaw(join(raw, "one-soak"), { "uac_stat.csv": "StartTime;\n" }); + const remote: Remote = { + exec: (host, command) => hosts.exec(host, command, "cli"), + upload: hosts.upload, + download: hosts.download + }; + const runDir = await mkdtemp(join(tmpdir(), "perfkit-run-")); + return { hosts, remote, runDir }; +} + +describe("downloadPartialRaw", () => { + it("should download every benchmark not downloaded yet, finished or not", async () => { + // Arrange + const { remote, runDir } = await setup(); + + // Act + const ids = await downloadPartialRaw({ + remote, + root: REMOTE_ROOT, + runDir, + downloaded: new Set(["edgeport-echo"]) + }); + + // Assert + expect(ids).to.deep.equal(["one-soak"]); + expect(await readdir(join(runDir, "raw"))).to.deep.equal(["one-soak"]); + expect( + await readFile(join(runDir, "raw", "one-soak", "uac_stat.csv"), "utf8") + ).to.equal("StartTime;\n"); + }); + + it("should return nothing when the load generator does not answer", async () => { + // Arrange + const { hosts, remote, runDir } = await setup(); + hosts.down.add("loadgen"); + + // Act + const ids = await downloadPartialRaw({ + remote, + root: REMOTE_ROOT, + runDir, + downloaded: new Set() + }); + + // Assert + expect(ids).to.deep.equal([]); + }); + + it("should skip a benchmark whose download fails and keep the others", async () => { + // Arrange + const { remote, runDir } = await setup(); + const failing: Remote = { + ...remote, + download: async (host, remotePath, localPath) => { + if (remotePath.endsWith("/edgeport-echo")) throw new Error("scp"); + return remote.download(host, remotePath, localPath); + } + }; + + // Act + const ids = await downloadPartialRaw({ + remote: failing, + root: REMOTE_ROOT, + runDir, + downloaded: new Set() + }); + + // Assert + expect(ids).to.deep.equal(["one-soak"]); + }); + + it("should ignore names that are not benchmark ids", async () => { + // Arrange + const { remote, runDir } = await setup(); + const listing: Remote = { + ...remote, + exec: async () => ({ code: 0, stdout: "../etc\nNot An Id\n", stderr: "" }) + }; + + // Act + const ids = await downloadPartialRaw({ + remote: listing, + root: REMOTE_ROOT, + runDir, + downloaded: new Set() + }); + + // Assert + expect(ids).to.deep.equal([]); + }); +}); diff --git a/test/summary/createSummarizeReport.unit.test.ts b/test/summary/createSummarizeReport.unit.test.ts new file mode 100644 index 0000000..3531ec0 --- /dev/null +++ b/test/summary/createSummarizeReport.unit.test.ts @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { ValidationError } from "../../src/errors/ValidationError.js"; +import { createSummarizeReport } from "../../src/summary/createSummarizeReport.js"; +import { type Json, loadReport, loadRuns } from "../fixtures.js"; + +function summarize(memory: Record) { + return createSummarizeReport({ + readFile: async (path) => { + if (!(path in memory)) { + throw Object.assign(new Error("nope"), { code: "ENOENT" }); + } + return JSON.stringify(memory[path]); + }, + isFile: async (path) => path in memory + }); +} + +describe("createSummarizeReport", () => { + it("should return the report as Markdown", async () => { + // Act + const result = await summarize({ "report.json": loadReport("passed") })({ + report: "report.json" + }); + + // Assert + expect(result.ok).to.equal(true); + if (!result.ok) return; + expect(result.markdown).to.match( + /^## perfkit run 20261103T141210Z-bd3254a\n/ + ); + }); + + it("should return the errors of an invalid report", async () => { + // Arrange + const report = loadReport("passed"); + delete report.run.commit; + + // Act + const result = await summarize({ "report.json": report })({ + report: "report.json" + }); + + // Assert + expect(result).to.deep.equal({ + ok: false, + errors: ["report.json: run.commit: required"] + }); + }); + + it("should refuse a runs.json", async () => { + // Act + const result = await summarize({ "runs.json": loadRuns() })({ + report: "runs.json" + }); + + // Assert + expect(result).to.deep.equal({ + ok: false, + errors: [ + "runs.json: expected a perfkit/report/v1 report; got perfkit/runs/v1" + ] + }); + }); + + it("should report a missing file", async () => { + // Act + const result = await summarize({})({ report: "missing.json" }); + + // Assert + expect(result).to.deep.equal({ + ok: false, + errors: ["missing.json: cannot read: no such file"] + }); + }); + + it("should throw ValidationError for an empty path", async () => { + // Act + const error = await summarize({})({ report: "" }).catch((e: unknown) => e); + + // Assert + expect(error).to.be.instanceOf(ValidationError); + }); +}); diff --git a/test/summary/formatCapacitySummary.unit.test.ts b/test/summary/formatCapacitySummary.unit.test.ts new file mode 100644 index 0000000..3767075 --- /dev/null +++ b/test/summary/formatCapacitySummary.unit.test.ts @@ -0,0 +1,116 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { + type CapacityBenchmarkResult, + reportSchema +} from "../../src/schemas/index.js"; +import { formatCapacitySummary } from "../../src/summary/formatCapacitySummary.js"; +import { loadReport, REPORT_FORMS } from "../fixtures.js"; + +function capacityOf( + form: (typeof REPORT_FORMS)[number] +): CapacityBenchmarkResult { + const benchmark = reportSchema + .parse(loadReport(form)) + .benchmarks.find(({ kind }) => kind === "capacity"); + return benchmark as CapacityBenchmarkResult; +} + +describe("formatCapacitySummary", () => { + it("should give the limit, where the ramp stopped and every step", () => { + // Arrange + const benchmark = capacityOf("passed"); + + // Act + const text = formatCapacitySummary(benchmark); + + // Assert + const lines = text.split("\n"); + expect(lines[0]).to.equal( + "**Capacity limit:** 1,240 messages/s, p99 30.9 ms, 0.04% failed." + ); + expect(lines[2]).to.equal( + "**Stopped at** 1,300 messages/s: p99 62 ms, limit 50 ms." + ); + expect(lines[4]).to.equal( + "| Offered (messages/s) | Placed (messages/s) | p99 | Failed | Result |" + ); + expect(lines[6]).to.equal("| 50 | 49.6 | 4 ms | 0% | ok |"); + expect(lines.at(-1)).to.equal( + "| 1,300 | 1,289.6 | 62 ms | 0.06% | stopped: p99 over 50 ms |" + ); + expect(lines.length).to.equal( + 6 + (benchmark.verdict === "passed" ? benchmark.series.steps.length : 0) + ); + }); + + it("should say the limit is a lower bound when no step broke a rule", () => { + // Act + const text = formatCapacitySummary(capacityOf("lower-bound")); + + // Assert + expect(text.split("\n")[0]).to.equal( + "**Capacity limit:** at least 5,000 messages/s, p99 18.4 ms, 0.02% failed (no step broke a rule)." + ); + expect(text).to.not.include("Stopped at"); + }); + + it("should say there is no limit when the first step broke a rule", () => { + // Act + const text = formatCapacitySummary(capacityOf("failed")); + + // Assert + expect(text).to.include( + "**Capacity limit:** none, the first step already broke a rule." + ); + expect(text).to.include( + "| 50 | 49.6 | 72 ms | 0.03% | stopped: p99 over 50 ms |" + ); + }); + + it("should mark a step over the failed limit and an invalid step", () => { + // Arrange + const benchmark = capacityOf("passed"); + if (benchmark.verdict !== "passed") throw new Error("expected passed"); + const [first, second] = benchmark.series.steps; + const steps = [ + { ...first!, calls: 1000, failed: 5 }, + { ...second!, valid: false } + ]; + + // Act + const text = formatCapacitySummary({ + ...benchmark, + series: { steps } + }); + + // Assert + expect(text).to.include("| 0.5% | stopped: failed over 0.1% |"); + expect(text).to.include("| invalid; stopped |"); + }); + + it("should list a partial benchmark's steps without limits", () => { + // Act + const text = formatCapacitySummary(capacityOf("not-run")); + + // Assert + expect(text).to.not.include("Capacity limit"); + expect(text.split("\n").at(-1)).to.equal( + "| 200 | 198.4 | 7 ms | 0.03% | ok |" + ); + }); + + it("should be empty for a benchmark that did not run", () => { + // Act + const text = formatCapacitySummary({ + ...capacityOf("passed"), + verdict: "not-run", + reason: "the run was aborted before this benchmark started" + } as CapacityBenchmarkResult); + + // Assert + expect(text).to.equal(""); + }); +}); diff --git a/test/summary/formatReportSummary.unit.test.ts b/test/summary/formatReportSummary.unit.test.ts new file mode 100644 index 0000000..ed05085 --- /dev/null +++ b/test/summary/formatReportSummary.unit.test.ts @@ -0,0 +1,75 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { reportSchema } from "../../src/schemas/index.js"; +import { formatReportSummary } from "../../src/summary/formatReportSummary.js"; +import { loadReport, REPORT_FORMS } from "../fixtures.js"; + +const parsed = (form: (typeof REPORT_FORMS)[number]) => + reportSchema.parse(loadReport(form)); + +describe("formatReportSummary", () => { + it("should open with the run id, target version, commit and verdict", () => { + // Act + const text = formatReportSummary(parsed("failed")); + + // Assert + expect(text.split("\n").slice(0, 5)).to.deep.equal([ + "## perfkit run 20261020T100005Z-c0b54ab", + "", + "**Routr 2.31.3**, commit `c0b54abe09d1`. **0 of 2 benchmarks passed** (2 failed).", + "", + "Started 2026-10-20T10:00:05Z, ended 2026-10-20T16:41:30Z; digitalocean nyc3, server under test c-8, load generator c-16; perfkit 0.1.0." + ]); + expect(text.endsWith("\n")).to.equal(true); + }); + + it("should give each benchmark a heading with its verdict", () => { + // Act + const text = formatReportSummary(parsed("failed")); + + // Assert + expect(text).to.include( + "### EdgePort + Echo (`edgeport-echo`): failed\n\nCapacity benchmark, 1 min, SIPp 3.7.3, UDP." + ); + expect(text).to.include( + "### RoutrOne, six hours (`one-soak`): failed\n\nSoak benchmark, 6 h 30 min, SIPp 3.7.3, UDP; OpenTelemetry." + ); + expect(text).to.include( + "| Memory growth, one | +7.4 MB/h | 5 MB/h | failed |" + ); + }); + + it("should give the reason of an aborted and a not-run benchmark", () => { + // Act + const text = formatReportSummary(parsed("not-run")); + + // Assert + expect(text).to.include("(1 aborted, 1 not run)"); + expect(text).to.include( + "### EdgePort + Echo (`edgeport-echo`): aborted\n\nCapacity benchmark, 4 min, SIPp 3.7.3, UDP.\n\n**Aborted:** the server under test stopped responding." + ); + expect(text).to.include( + "### RoutrOne, six hours (`one-soak`): not run\n\nSoak benchmark.\n\n**Not run:** the run was aborted before this benchmark started.\n" + ); + }); + + it("should give the reason of an inconclusive benchmark", () => { + // Act + const text = formatReportSummary(parsed("inconclusive")); + + // Assert + expect(text).to.include( + "**Inconclusive:** load generator saturated at 1,300 messages/s (SIPp at 93% of one core)." + ); + }); + + it("should not list other counts when every benchmark passed", () => { + // Act + const text = formatReportSummary(parsed("passed")); + + // Assert + expect(text).to.include("**2 of 2 benchmarks passed**."); + }); +}); diff --git a/test/summary/formatSoakSummary.unit.test.ts b/test/summary/formatSoakSummary.unit.test.ts new file mode 100644 index 0000000..69235b9 --- /dev/null +++ b/test/summary/formatSoakSummary.unit.test.ts @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { + reportSchema, + type SoakBenchmarkResult +} from "../../src/schemas/index.js"; +import { formatSoakSummary } from "../../src/summary/formatSoakSummary.js"; +import { loadReport, REPORT_FORMS } from "../fixtures.js"; + +function soakOf(form: (typeof REPORT_FORMS)[number]): SoakBenchmarkResult { + const benchmark = reportSchema + .parse(loadReport(form)) + .benchmarks.find(({ kind }) => kind === "soak"); + return benchmark as SoakBenchmarkResult; +} + +describe("formatSoakSummary", () => { + it("should list the load, every rule against its limit and p99 by hour", () => { + // Act + const text = formatSoakSummary(soakOf("failed")); + + // Assert + expect(text).to.equal( + [ + "**Load:** 100 calls/s, each held 30 s; 2,160,000 calls in all.", + "", + "| Rule | Measured | Limit | Result |", + "| :--- | ---: | ---: | :--- |", + "| Memory growth, one | +7.4 MB/h | 5 MB/h | failed |", + "| Memory growth, edgeport | +0.3 MB/h | 5 MB/h | passed |", + "| Memory growth, postgres | +0.1 MB/h | 5 MB/h | passed |", + "| Failed calls | 0.01% | 0.1% | passed |", + "| Restarts | 0 | 0 | passed |", + "", + "**p99:** 21 ms in the first hour, 34 ms in the last hour." + ].join("\n") + ); + }); + + it("should be empty for an aborted soak", () => { + // Act / Assert + expect(formatSoakSummary(soakOf("aborted-soak"))).to.equal(""); + }); +}); diff --git a/test/summary/markdown.unit.test.ts b/test/summary/markdown.unit.test.ts new file mode 100644 index 0000000..742f2eb --- /dev/null +++ b/test/summary/markdown.unit.test.ts @@ -0,0 +1,75 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { + capitalized, + cell, + duration, + growth, + num, + pct, + table, + verdictWord +} from "../../src/summary/markdown.js"; + +describe("summary markdown helpers", () => { + it("should name each verdict in lowercase words", () => { + // Act + const words = ( + ["passed", "failed", "inconclusive", "aborted", "not-run"] as const + ).map(verdictWord); + + // Assert + expect(words).to.deep.equal([ + "passed", + "failed", + "inconclusive", + "aborted", + "not run" + ]); + }); + + it("should capitalize the first letter only", () => { + // Act / Assert + expect(capitalized("not run")).to.equal("Not run"); + expect(capitalized("")).to.equal(""); + }); + + it("should escape pipes and join lines in a cell", () => { + // Act / Assert + expect(cell("a | b\n c")).to.equal("a \\| b c"); + }); + + it("should write numbers, percentages, growth and durations", () => { + // Act / Assert + expect(num(1240.84)).to.equal("1,240.8"); + expect(num(2160000, 0)).to.equal("2,160,000"); + expect(pct(0.034)).to.equal("0.03%"); + expect(growth(7.44)).to.equal("+7.4 MB/h"); + expect(growth(-0.1)).to.equal("-0.1 MB/h"); + expect(duration(45)).to.equal("45 s"); + expect(duration(1560)).to.equal("26 min"); + expect(duration(21600)).to.equal("6 h"); + expect(duration(23400)).to.equal("6 h 30 min"); + }); + + it("should write a table with right-aligned numeric columns", () => { + // Act + const text = table( + ["Name", "Value"], + [ + ["a", "1"], + ["b", "2"] + ], + [false, true] + ); + + // Assert + expect(text).to.equal( + ["| Name | Value |", "| :--- | ---: |", "| a | 1 |", "| b | 2 |"].join( + "\n" + ) + ); + }); +});