From 0c9f1de7f831d93fcb996ded3d7c713ca758c673 Mon Sep 17 00:00:00 2001 From: Pedro Sanders Date: Thu, 5 Nov 2026 07:45:38 -0400 Subject: [PATCH 1/2] fix(orchestrate): keep the raw results of unfinished benchmarks when the agent fails collect now downloads the raw directory of every benchmark it has not downloaded yet before it exits on an agent failure (and before it tears down a cancelled run), so the workflow's raw artifact keeps them. The cleanup job of run.yaml now needs every job before it. --- .github/workflows/run.yaml | 42 ++++++- docs/github-workflow.md | 12 +- src/orchestrate/createCollect.ts | 23 +++- src/orchestrate/downloadPartialRaw.ts | 56 +++++++++ test/orchestrate/createCollect.unit.test.ts | 116 +++++++++++++++++- .../downloadPartialRaw.unit.test.ts | 107 ++++++++++++++++ 6 files changed, 350 insertions(+), 6 deletions(-) create mode 100644 src/orchestrate/downloadPartialRaw.ts create mode 100644 test/orchestrate/downloadPartialRaw.unit.test.ts diff --git a/.github/workflows/run.yaml b/.github/workflows/run.yaml index 028a20b..6aa61d4 100644 --- a/.github/workflows/run.yaml +++ b/.github/workflows/run.yaml @@ -123,6 +123,17 @@ jobs: path: .perfkit/${{ needs.start.outputs.run-id }}/raw/ if-no-files-found: ignore overwrite: true + - name: Write the run summary + if: failure() + env: + RUN_ID: ${{ needs.start.outputs.run-id }} + run: | + { + echo "## perfkit run $RUN_ID" + echo + echo "The run failed before its report was written; see this job's log." + echo "The raw results downloaded so far are in the artifact perfkit-raw-$RUN_ID." + } >> "$GITHUB_STEP_SUMMARY" wait-2: name: Wait for the run (next 5 h 50 min) @@ -160,6 +171,17 @@ jobs: path: .perfkit/${{ needs.start.outputs.run-id }}/raw/ if-no-files-found: ignore overwrite: true + - name: Write the run summary + if: failure() + env: + RUN_ID: ${{ needs.start.outputs.run-id }} + run: | + { + echo "## perfkit run $RUN_ID" + echo + echo "The run failed before its report was written; see this job's log." + echo "The raw results downloaded so far are in the artifact perfkit-raw-$RUN_ID." + } >> "$GITHUB_STEP_SUMMARY" finish: name: Collect the results and delete the droplets @@ -202,10 +224,28 @@ jobs: name: perfkit-report-${{ needs.start.outputs.run-id }} path: .perfkit/${{ needs.start.outputs.run-id }}/ if-no-files-found: ignore + - name: Write the run summary + if: always() + env: + RUN_ID: ${{ needs.start.outputs.run-id }} + run: | + report=".perfkit/$RUN_ID/report.json" + if [ -f "$report" ]; then + npx -y "$PERFKIT" summary --report "$report" >> "$GITHUB_STEP_SUMMARY" + else + { + echo "## perfkit run $RUN_ID" + echo + echo "No report was written; see this job's log." + echo "The raw results are in the artifact perfkit-report-$RUN_ID." + } >> "$GITHUB_STEP_SUMMARY" + fi + # Every job above uploads the raw results it downloaded before it ends, so + # this job, which deletes the droplets, always runs after those uploads. cleanup: name: Delete anything the run left behind - needs: [start, finish] + needs: [start, wait-1, wait-2, finish] if: always() && needs.start.outputs.run-id != '' runs-on: ubuntu-latest permissions: diff --git a/docs/github-workflow.md b/docs/github-workflow.md index 78ae2d0..2436e2b 100644 --- a/docs/github-workflow.md +++ b/docs/github-workflow.md @@ -14,12 +14,18 @@ the load on its own, and later jobs only follow it: 1. `start` runs `perfkit run --detach` and outputs the run id. 2. `wait-1` and `wait-2` each run `perfkit collect --wait --timeout 5h50m`. Exit 4 (the run goes on) lets the next job wait; exit 0 skips the rest of the waiting. Each uploads the raw - results so far as the artifact `perfkit-raw-`, overwriting it. + results so far as the artifact `perfkit-raw-`, overwriting it. When the agent fails + (exit 3), `collect` first downloads the raw results of the unfinished benchmarks as they are, + so the upload keeps them. 3. `finish` downloads that artifact and runs `perfkit collect `: it writes `report.json`, deletes the droplets, and uploads `.perfkit//` as `perfkit-report-`. Exit 1 (a - benchmark did not pass) still uploads. + benchmark did not pass) still uploads. It then appends `perfkit summary` of the report to the + workflow run's summary page: the verdicts, the capacity steps and the soak rules. 4. `cleanup` always runs `perfkit sweep --run `, so a cancelled or failed workflow leaves - nothing billing. + nothing billing. It needs every job before it, so it starts only after their uploads. + +Where a run failed before its report, the failed job's summary says so and names the artifact +that holds the raw results. Every job has `permissions: { contents: read }`. Each job runs on a new machine, so `collect` adds that machine's IP to the run's firewall before it logs in, with `PERFKIT_SSH_KEY`. diff --git a/src/orchestrate/createCollect.ts b/src/orchestrate/createCollect.ts index d1a5dd1..e4ac493 100644 --- a/src/orchestrate/createCollect.ts +++ b/src/orchestrate/createCollect.ts @@ -18,6 +18,7 @@ import { MAX_FAILED_CHECKS, nameOf } from "./constants.js"; +import { downloadPartialRaw } from "./downloadPartialRaw.js"; import { formatResultLine, formatSpan } from "./formatResultLine.js"; import { runPaths } from "./paths.js"; import { @@ -55,7 +56,9 @@ type Ending = "done" | "failed" | "timeout" | "unreachable" | "cancelled"; * agent's progress lines and downloading each benchmark's raw results as * soon as its `done` marker exists. With `wait` it only waits: exit 0 when * the run is done, 4 on timeout, 130 on a signal, 3 when the agent failed - * or the load generator stopped answering; it never deletes. Otherwise, + * or the load generator stopped answering; it never deletes. When the agent + * failed (or, without `wait`, the run was cancelled) it first downloads the + * raw results of the benchmarks that did not finish, as they are. Otherwise, * once the run ends it writes report.json, deletes everything and exits 0 * (every selected benchmark passed) or 1. A signal stops the agent, marks * unfinished benchmarks `aborted` (the run was cancelled), and still writes @@ -188,8 +191,25 @@ export function createCollect(deps: CollectDeps) { ); return EXIT_RUNNING; } + // Before giving up on a failed run, keep what its benchmarks wrote so + // far: the workflow uploads raw/ and then deletes the droplets. + const salvage = async () => { + const ids = await downloadPartialRaw({ + remote: deps.remote, + root: deps.root, + runDir, + downloaded + }); + ids.forEach((id) => + deps.stdout( + `[${id}] downloaded the partial raw results to ${join(runDir, "raw", id)}\n` + ) + ); + }; + if (input.wait) { if (ending === "done") return EXIT_OK; + if (ending === "failed") await salvage(); if (ending === "cancelled") return EXIT_CANCELLED; deps.stderr( `perfkit: ${ending === "failed" ? current?.failed : LOADGEN_UNRESPONSIVE}; nothing was deleted\n` @@ -207,6 +227,7 @@ export function createCollect(deps: CollectDeps) { .catch(() => undefined); await read().catch(() => false); } + if (ending === "failed" || ending === "cancelled") await salvage(); const finish = async (code: number): Promise => { const { summary, failures } = await deps.teardown(); diff --git a/src/orchestrate/downloadPartialRaw.ts b/src/orchestrate/downloadPartialRaw.ts new file mode 100644 index 0000000..ef418b8 --- /dev/null +++ b/src/orchestrate/downloadPartialRaw.ts @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { mkdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { benchmarkIdSchema } from "../schemas/primitives.schema.js"; +import { runPaths } from "./paths.js"; +import type { Remote } from "./types.js"; + +/** + * Downloads the raw results of every benchmark on the load generator that + * collect has not downloaded yet, finished or not, into `/raw/`. + * Collect calls it before it gives up on a failed or cancelled run, so the + * workflow's raw artifact keeps hours of results that the run's teardown + * would delete. A partial directory gets no `done` marker, so it never + * counts as finished. It never throws: a benchmark it cannot download is + * left out. + * + * @returns The ids it downloaded, in the load generator's order + */ +export async function downloadPartialRaw(input: { + remote: Remote; + root: string; + runDir: string; + downloaded: ReadonlySet; +}): Promise { + const paths = runPaths(input.root); + const listed = await input.remote + .exec("loadgen", `ls -1 ${paths.raw} 2>/dev/null; true`, { + timeoutMs: 60 * 1000 + }) + .catch(() => null); + if (listed === null || listed.code !== 0) return []; + const ids = listed.stdout + .split("\n") + .map((line) => line.trim()) + .filter( + (id) => + benchmarkIdSchema.safeParse(id).success && !input.downloaded.has(id) + ); + return ids.reduce>( + (previous, id) => + previous.then(async (done) => { + const target = join(input.runDir, "raw", id); + try { + await rm(target, { recursive: true, force: true }); + await mkdir(join(input.runDir, "raw"), { recursive: true }); + await input.remote.download("loadgen", paths.rawOf(id), target); + return [...done, id]; + } catch { + return done; + } + }), + Promise.resolve([]) + ); +} diff --git a/test/orchestrate/createCollect.unit.test.ts b/test/orchestrate/createCollect.unit.test.ts index 88be2d0..6c087a0 100644 --- a/test/orchestrate/createCollect.unit.test.ts +++ b/test/orchestrate/createCollect.unit.test.ts @@ -3,7 +3,7 @@ */ import { expect } from "chai"; import { EventEmitter } from "node:events"; -import { readFile, writeFile } from "node:fs/promises"; +import { readdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import sinon from "sinon"; import { createNodeAnalyzeDeps } from "../../src/analyze/index.js"; @@ -98,6 +98,37 @@ async function reportOf(dir: string, runId: string) { ); } +const AGENT_FAILED = "the agent failed: Maximum call stack size exceeded"; + +/** + * The half-done run, after its agent failed in the soak: the soak's raw + * directory holds hours of results but no `done` marker. + */ +async function agentFailed() { + const t = await halfDone(); + const root = t.setup.hosts.roots.loadgen; + await writeRaw(join(root, "raw", "one-soak"), { + "sampler-uut.jsonl": '{"t":"2026-11-03T14:20:00Z"}\n', + "uac_stat.csv": "StartTime;\n" + }); + await writeFile( + join(root, "status.json"), + JSON.stringify({ + benchmarks: { + "edgeport-echo": { state: "done" }, + "one-soak": { + state: "running", + startedAt: "2026-11-03T14:20:00Z", + plannedS: 23400 + } + }, + done: false, + failed: AGENT_FAILED + }) + ); + return t; +} + describe("createCollect", () => { afterEach(() => { sinon.restore(); @@ -212,6 +243,89 @@ describe("createCollect", () => { expect(t.teardown.callCount).to.equal(0); }); + it("should download the soak's partial raw results, delete nothing and exit 3 when the agent failed during --wait", async () => { + // Arrange + const t = await agentFailed(); + + // Act + const code = await t.collect()({ + runId: t.runId, + wait: true, + out: t.setup.out, + timeoutMs: null + }); + + // Assert + expect(code).to.equal(3); + const raw = join(t.setup.out, t.runId, "raw"); + expect((await readdir(raw)).sort()).to.deep.equal([ + "edgeport-echo", + "one-soak" + ]); + expect((await readdir(join(raw, "one-soak"))).sort()).to.deep.equal([ + "sampler-uut.jsonl", + "uac_stat.csv" + ]); + expect(t.out).to.include( + `[one-soak] downloaded the partial raw results to ${join(raw, "one-soak")}\n` + ); + expect(t.err).to.deep.equal([ + `perfkit: ${AGENT_FAILED}; nothing was deleted\n` + ]); + expect(t.teardown.callCount).to.equal(0); + }); + + it("should download the partial raw results before it deletes everything when the agent failed", async () => { + // Arrange + const t = await agentFailed(); + const raw = join(t.setup.out, t.runId, "raw"); + let atTeardown: string[] = []; + t.teardown.callsFake(async () => { + atTeardown = (await readdir(raw)).sort(); + return { summary: "deleted 2 droplets", failures: [] }; + }); + + // Act + const code = await t.collect()({ + runId: t.runId, + out: t.setup.out, + timeoutMs: null + }); + + // Assert + expect(code).to.equal(3); + expect(atTeardown).to.deep.equal(["edgeport-echo", "one-soak"]); + expect(t.err).to.deep.equal([ + `perfkit: ${AGENT_FAILED}; no report was written\n` + ]); + expect(t.teardown.callCount).to.equal(1); + }); + + it("should still exit 3 when the partial raw results cannot be downloaded", async () => { + // Arrange + const t = await agentFailed(); + const download = t.setup.hosts.download; + t.setup.hosts.download = async (host, remotePath, localPath) => { + if (remotePath.endsWith("/one-soak")) throw new Error("scp failed"); + return download(host, remotePath, localPath); + }; + + // Act + const code = await t.collect()({ + runId: t.runId, + wait: true, + out: t.setup.out, + timeoutMs: null + }); + + // Assert + expect(code).to.equal(3); + expect( + t.out.some((line) => line.includes("downloaded the partial raw")) + ).to.equal(false); + expect(t.teardown.callCount).to.equal(0); + }); + it("should throw ValidationError for a bad run id and read nothing", async () => { // Arrange const t = await halfDone(); diff --git a/test/orchestrate/downloadPartialRaw.unit.test.ts b/test/orchestrate/downloadPartialRaw.unit.test.ts new file mode 100644 index 0000000..ebca485 --- /dev/null +++ b/test/orchestrate/downloadPartialRaw.unit.test.ts @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { mkdtemp, readdir, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { REMOTE_ROOT } from "../../src/orchestrate/constants.js"; +import { downloadPartialRaw } from "../../src/orchestrate/downloadPartialRaw.js"; +import type { Remote } from "../../src/orchestrate/types.js"; +import { FakeHosts, writeRaw } from "./harness.js"; + +async function setup() { + const hosts = await FakeHosts.create(); + const raw = join(hosts.roots.loadgen, "raw"); + await writeRaw(join(raw, "edgeport-echo"), { done: "x\n" }); + await writeRaw(join(raw, "one-soak"), { "uac_stat.csv": "StartTime;\n" }); + const remote: Remote = { + exec: (host, command) => hosts.exec(host, command, "cli"), + upload: hosts.upload, + download: hosts.download + }; + const runDir = await mkdtemp(join(tmpdir(), "perfkit-run-")); + return { hosts, remote, runDir }; +} + +describe("downloadPartialRaw", () => { + it("should download every benchmark not downloaded yet, finished or not", async () => { + // Arrange + const { remote, runDir } = await setup(); + + // Act + const ids = await downloadPartialRaw({ + remote, + root: REMOTE_ROOT, + runDir, + downloaded: new Set(["edgeport-echo"]) + }); + + // Assert + expect(ids).to.deep.equal(["one-soak"]); + expect(await readdir(join(runDir, "raw"))).to.deep.equal(["one-soak"]); + expect( + await readFile(join(runDir, "raw", "one-soak", "uac_stat.csv"), "utf8") + ).to.equal("StartTime;\n"); + }); + + it("should return nothing when the load generator does not answer", async () => { + // Arrange + const { hosts, remote, runDir } = await setup(); + hosts.down.add("loadgen"); + + // Act + const ids = await downloadPartialRaw({ + remote, + root: REMOTE_ROOT, + runDir, + downloaded: new Set() + }); + + // Assert + expect(ids).to.deep.equal([]); + }); + + it("should skip a benchmark whose download fails and keep the others", async () => { + // Arrange + const { remote, runDir } = await setup(); + const failing: Remote = { + ...remote, + download: async (host, remotePath, localPath) => { + if (remotePath.endsWith("/edgeport-echo")) throw new Error("scp"); + return remote.download(host, remotePath, localPath); + } + }; + + // Act + const ids = await downloadPartialRaw({ + remote: failing, + root: REMOTE_ROOT, + runDir, + downloaded: new Set() + }); + + // Assert + expect(ids).to.deep.equal(["one-soak"]); + }); + + it("should ignore names that are not benchmark ids", async () => { + // Arrange + const { remote, runDir } = await setup(); + const listing: Remote = { + ...remote, + exec: async () => ({ code: 0, stdout: "../etc\nNot An Id\n", stderr: "" }) + }; + + // Act + const ids = await downloadPartialRaw({ + remote: listing, + root: REMOTE_ROOT, + runDir, + downloaded: new Set() + }); + + // Assert + expect(ids).to.deep.equal([]); + }); +}); From 185e3c3d41dbec8d36a6c0254664f8fdde6cfee9 Mon Sep 17 00:00:00 2001 From: Pedro Sanders Date: Thu, 5 Nov 2026 07:45:38 -0400 Subject: [PATCH 2/2] feat(cli): add perfkit summary and write it to the workflow run summary perfkit summary --report prints a report.json as GitHub Markdown: the run, each benchmark's verdict and reason, the capacity limit and steps, and the soak rules and p99 by hour. run.yaml appends it to $GITHUB_STEP_SUMMARY, or a note naming the raw artifact when no report was written. --- README.md | 24 ++++ src/cli/runCli.ts | 32 ++++- src/summary/createSummarizeReport.ts | 56 +++++++++ src/summary/formatCapacitySummary.ts | 89 ++++++++++++++ src/summary/formatReportSummary.ts | 57 +++++++++ src/summary/formatSoakSummary.ts | 53 ++++++++ src/summary/markdown.ts | 73 +++++++++++ src/summary/summarizeReport.schema.ts | 10 ++ test/cli/runCli.unit.test.ts | 42 +++++++ .../createSummarizeReport.unit.test.ts | 86 +++++++++++++ .../formatCapacitySummary.unit.test.ts | 116 ++++++++++++++++++ test/summary/formatReportSummary.unit.test.ts | 75 +++++++++++ test/summary/formatSoakSummary.unit.test.ts | 46 +++++++ test/summary/markdown.unit.test.ts | 75 +++++++++++ 14 files changed, 833 insertions(+), 1 deletion(-) create mode 100644 src/summary/createSummarizeReport.ts create mode 100644 src/summary/formatCapacitySummary.ts create mode 100644 src/summary/formatReportSummary.ts create mode 100644 src/summary/formatSoakSummary.ts create mode 100644 src/summary/markdown.ts create mode 100644 src/summary/summarizeReport.schema.ts create mode 100644 test/summary/createSummarizeReport.unit.test.ts create mode 100644 test/summary/formatCapacitySummary.unit.test.ts create mode 100644 test/summary/formatReportSummary.unit.test.ts create mode 100644 test/summary/formatSoakSummary.unit.test.ts create mode 100644 test/summary/markdown.unit.test.ts diff --git a/README.md b/README.md index 87ac7a9..f9ec332 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,10 @@ $ echo $? - **Cancelled.** SIGINT or SIGTERM stops the agent, marks every unfinished benchmark `aborted` (`the run was cancelled`), writes the report if one benchmark finished, deletes everything and exits 130. During `--wait` it only exits 130. +- **The agent fails.** Before it exits 3, `collect` downloads the raw results of every benchmark + that did not finish, as they are, into `//raw//` (without a `done` marker) and + prints `[] downloaded the partial raw results to `, so hours of a soak survive the + teardown. It does the same for a cancelled run before it deletes everything. | Exit code | Meaning | | :--- | :--- | @@ -314,6 +318,26 @@ $ echo $? For GitHub, [docs/github-workflow.md](docs/github-workflow.md) has the reusable workflow that splits a run across jobs. +## perfkit summary + +```sh +perfkit summary --report +``` + +Prints a report as GitHub Markdown, for a workflow's run summary: the run id, the target version +and commit, the overall verdict, and per benchmark its verdict (with its reason when it is +inconclusive, aborted or not run). A capacity benchmark adds its capacity limit, where the ramp +stopped and a table of the steps (offered and placed rate, p99, failed %, ok or why it stopped); a +soak adds its rules against their limits (memory growth per process, failed calls, restarts) and +p99 in the first and the last hour. The reusable workflow appends it to `$GITHUB_STEP_SUMMARY`: + +```sh +perfkit summary --report .perfkit//report.json >> "$GITHUB_STEP_SUMMARY" +``` + +It exits 0 when it printed the summary, 1 for a usage error, and 2 when the report is invalid, +unreadable or of another schema. + ## perfkit sweep ```sh diff --git a/src/cli/runCli.ts b/src/cli/runCli.ts index 976be25..735bd2d 100644 --- a/src/cli/runCli.ts +++ b/src/cli/runCli.ts @@ -3,6 +3,7 @@ */ import { Command, CommanderError } from "commander"; import { createRenderSite } from "../render/createRenderSite.js"; +import { createSummarizeReport } from "../summary/createSummarizeReport.js"; import { createValidateFile } from "../validate/createValidateFile.js"; import { DEFAULT_COLLECT_TIMEOUT, @@ -88,7 +89,8 @@ Exit codes: 0 the run is done (with --wait), or every benchmark passed 1 report written, at least one benchmark not passed 2 bad flags, a missing token or no SSH key for the run - 3 the agent failed, or no benchmark produced results + 3 the agent failed (the unfinished benchmarks' raw results are still + downloaded), or no benchmark produced results 4 the timeout passed while the run goes on; nothing deleted 130 cancelled; with --wait nothing deleted, otherwise everything`; const RENDER_HELP = ` @@ -104,6 +106,15 @@ Exit codes: 2 the report or the history is invalid, unreadable or of another schema 3 the history URL could not be fetched (a 404 means no history)`; +const SUMMARY_HELP = ` +Prints the report as GitHub Markdown, for a workflow's run summary: + perfkit summary --report .perfkit//report.json >> "$GITHUB_STEP_SUMMARY" + +Exit codes: + 0 the summary was printed + 1 usage error + 2 the report is invalid, unreadable or of another schema`; + /** * Runs the perfkit CLI with the given argv (as in process.argv) and returns * the exit code. Output goes through `deps.stdout` and `deps.stderr`. @@ -111,6 +122,7 @@ Exit codes: export async function runCli(argv: string[], deps: CliDeps): Promise { const validateFile = createValidateFile(deps); const renderSite = createRenderSite(deps); + const summarizeReport = createSummarizeReport(deps); let exitCode = EXIT_VALID; let usageExitCode = EXIT_USAGE; @@ -295,6 +307,24 @@ export async function runCli(argv: string[], deps: CliDeps): Promise { } ); + program + .command("summary") + .description( + "Print a report.json as Markdown, for a GitHub workflow's run summary." + ) + .requiredOption("--report ", "the run's report.json") + .addHelpText("after", SUMMARY_HELP) + .action(async (options: { report: string }) => { + const result = await summarizeReport(options); + if (!result.ok) { + result.errors.forEach((line) => deps.stderr(`${line}\n`)); + exitCode = EXIT_INVALID; + return; + } + deps.stdout(result.markdown); + exitCode = EXIT_VALID; + }); + try { await program.parseAsync(argv); return exitCode; diff --git a/src/summary/createSummarizeReport.ts b/src/summary/createSummarizeReport.ts new file mode 100644 index 0000000..4dd10af --- /dev/null +++ b/src/summary/createSummarizeReport.ts @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { withErrorHandlingAndValidation } from "../errors/withErrorHandlingAndValidation.js"; +import { reportSchema, SCHEMA_IDS } from "../schemas/index.js"; +import { createValidateFile } from "../validate/createValidateFile.js"; +import { parseDocument } from "../validate/parseDocument.js"; +import type { ValidateFileDeps } from "../validate/types.js"; +import { formatReportSummary } from "./formatReportSummary.js"; +import { + type SummarizeReportInput, + summarizeReportInputSchema +} from "./summarizeReport.schema.js"; + +export type SummarizeReportResult = + { ok: true; markdown: string } | { ok: false; errors: string[] }; + +/** + * Creates the function behind `perfkit summary`: it validates a report.json + * and returns it as GitHub Markdown, for $GITHUB_STEP_SUMMARY. + * + * @param deps - File system access (injected) + * @returns A validated function that returns the Markdown, or the error + * lines when the file is not a valid report + */ +export function createSummarizeReport(deps: ValidateFileDeps) { + const validateFile = createValidateFile(deps); + + const fn = async ({ + report: file + }: SummarizeReportInput): Promise => { + const checked = await validateFile({ file }); + if (!checked.valid) { + return { + ok: false, + errors: checked.errors.map(({ path, message }) => + path === null ? `${file}: ${message}` : `${file}: ${path}: ${message}` + ) + }; + } + if (checked.format !== SCHEMA_IDS.report) { + return { + ok: false, + errors: [ + `${file}: expected a ${SCHEMA_IDS.report} report; got ${checked.format}` + ] + }; + } + const report = reportSchema.parse( + parseDocument(await deps.readFile(file), file) + ); + return { ok: true, markdown: formatReportSummary(report) }; + }; + + return withErrorHandlingAndValidation(fn, summarizeReportInputSchema); +} diff --git a/src/summary/formatCapacitySummary.ts b/src/summary/formatCapacitySummary.ts new file mode 100644 index 0000000..6680a21 --- /dev/null +++ b/src/summary/formatCapacitySummary.ts @@ -0,0 +1,89 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { CapacityBenchmarkResult, Step } from "../schemas/index.js"; +import { cell, num, pct, table } from "./markdown.js"; + +/** + * The body of one capacity benchmark in the run summary: the capacity limit + * and the step table (offered and placed rate, p99, failed %, and whether + * the step was ok or where the ramp stopped). The heading and the verdict + * line are the caller's. + */ +export function formatCapacitySummary( + benchmark: CapacityBenchmarkResult +): string { + const unit = benchmark.unit; + const rate = (value: number) => `${num(value)} ${unit}/s`; + const parts: string[] = []; + + if (benchmark.verdict === "passed" || benchmark.verdict === "failed") { + const { headline, stoppedBy } = benchmark; + parts.push( + headline.limitCps === null + ? "**Capacity limit:** none, the first step already broke a rule." + : `**Capacity limit:** ${headline.limitIsLowerBound ? "at least " : ""}${rate(headline.limitCps)}, p99 ${num(headline.p99AtLimitMs ?? 0)} ms, ${pct(headline.failedAtLimitPct ?? 0)} failed${headline.limitIsLowerBound ? " (no step broke a rule)" : ""}.` + ); + if (stoppedBy) { + parts.push( + `**Stopped at** ${rate(stoppedBy.offeredCps)}: ${ruleText(stoppedBy.rule, stoppedBy.measured, stoppedBy.limit)}.` + ); + } + } + + const steps = benchmark.verdict === "not-run" ? [] : benchmark.series?.steps; + if (steps && steps.length > 0) { + const limits = + benchmark.verdict === "passed" || benchmark.verdict === "failed" + ? { + p99: benchmark.rules.find(({ id }) => id === "p99LatencyMs")?.limit, + failed: benchmark.rules.find(({ id }) => id === "failedCallsPct") + ?.limit + } + : { p99: undefined, failed: undefined }; + parts.push( + table( + [ + `Offered (${unit}/s)`, + `Placed (${unit}/s)`, + "p99", + "Failed", + "Result" + ], + steps.map((step) => [ + num(step.offeredCps), + num(step.achievedCps), + `${num(step.p99Ms)} ms`, + pct(failedPct(step)), + cell(resultOf(step, limits)) + ]), + [true, true, true, true, false] + ) + ); + } + return parts.join("\n\n"); +} + +function failedPct(step: Step): number { + return step.calls > 0 ? (step.failed / step.calls) * 100 : 0; +} + +function resultOf( + step: Step, + limits: { p99: number | undefined; failed: number | undefined } +): string { + if (!step.valid) return "invalid; stopped"; + if (limits.p99 !== undefined && step.p99Ms >= limits.p99) { + return `stopped: p99 over ${num(limits.p99)} ms`; + } + if (limits.failed !== undefined && failedPct(step) >= limits.failed) { + return `stopped: failed over ${pct(limits.failed)}`; + } + return "ok"; +} + +function ruleText(rule: string, measured: number, limit: number): string { + return rule === "p99LatencyMs" + ? `p99 ${num(measured)} ms, limit ${num(limit)} ms` + : `${pct(measured)} failed, limit ${pct(limit)}`; +} diff --git a/src/summary/formatReportSummary.ts b/src/summary/formatReportSummary.ts new file mode 100644 index 0000000..0f224a0 --- /dev/null +++ b/src/summary/formatReportSummary.ts @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { BenchmarkResult, Report } from "../schemas/index.js"; +import { formatCapacitySummary } from "./formatCapacitySummary.js"; +import { formatSoakSummary } from "./formatSoakSummary.js"; +import { capitalized, cell, duration, verdictWord } from "./markdown.js"; + +/** + * A report.json as GitHub Markdown, for a workflow's run summary: the run + * (id, target version, commit, machines), the overall verdict, and per + * benchmark its verdict (with the reason when it has one), then the + * capacity limit and steps or the soak rules. + */ +export function formatReportSummary(report: Report): string { + const { run, verdict } = report; + const others = ( + [ + ["failed", verdict.failed], + ["inconclusive", verdict.inconclusive], + ["aborted", verdict.aborted], + ["not run", verdict.notRun] + ] as const + ) + .filter(([, n]) => n > 0) + .map(([word, n]) => `${n} ${word}`); + const header = [ + `## perfkit run ${run.id}`, + [ + `**${cell(report.project.name)} ${cell(run.version)}**, commit \`${run.commit}\`.`, + `**${verdict.passed} of ${verdict.total} benchmark${verdict.total === 1 ? "" : "s"} passed**${others.length > 0 ? ` (${others.join(", ")})` : ""}.` + ].join(" "), + `Started ${run.startedAt}, ended ${run.endedAt}; ${run.infra.provider} ${run.infra.region}, server under test ${run.infra.uut.size}, load generator ${run.infra.loadgen.size}; perfkit ${run.tools.perfkit}.` + ]; + return `${[...header, ...report.benchmarks.map(formatBenchmark)].join("\n\n")}\n`; +} + +function formatBenchmark(benchmark: BenchmarkResult): string { + const heading = `### ${cell(benchmark.title)} (\`${benchmark.id}\`): ${verdictWord(benchmark.verdict)}`; + const about = + benchmark.verdict === "not-run" + ? `${benchmark.kind === "capacity" ? "Capacity" : "Soak"} benchmark.` + : `${benchmark.kind === "capacity" ? "Capacity" : "Soak"} benchmark, ${duration(benchmark.durationS)}, ${cell(benchmark.tool)}.`; + const reason = + benchmark.reason === null + ? [] + : [ + `**${capitalized(verdictWord(benchmark.verdict))}:** ${cell(benchmark.reason)}.` + ]; + const body = + benchmark.kind === "capacity" + ? formatCapacitySummary(benchmark) + : formatSoakSummary(benchmark); + return [heading, about, ...reason, ...(body === "" ? [] : [body])].join( + "\n\n" + ); +} diff --git a/src/summary/formatSoakSummary.ts b/src/summary/formatSoakSummary.ts new file mode 100644 index 0000000..9e98676 --- /dev/null +++ b/src/summary/formatSoakSummary.ts @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { SoakBenchmarkResult, SoakRule } from "../schemas/index.js"; +import { cell, growth, num, pct, table } from "./markdown.js"; + +/** + * The body of one soak benchmark in the run summary: its rules (memory + * growth per process, failed calls, restarts) against their limits, and p99 + * in the first and the last hour. The heading and the verdict line are the + * caller's. + */ +export function formatSoakSummary(benchmark: SoakBenchmarkResult): string { + if (benchmark.verdict !== "passed" && benchmark.verdict !== "failed") { + return ""; + } + const unit = benchmark.unit; + const rules = table( + ["Rule", "Measured", "Limit", "Result"], + benchmark.rules.map((rule) => [ + cell(ruleName(rule, unit)), + measuredOf(rule), + limitOf(rule), + rule.passed ? "passed" : "failed" + ]), + [false, true, true, false] + ); + const { reported, headline, load } = benchmark; + return [ + `**Load:** ${num(load.cps)} ${unit}/s, each held ${num(load.holdS)} s; ${num(headline.calls, 0)} ${unit} in all.`, + rules, + `**p99:** ${num(reported.p99FirstHourMs)} ms in the first hour, ${num(reported.p99LastHourMs)} ms in the last hour.` + ].join("\n\n"); +} + +function ruleName(rule: SoakRule, unit: string): string { + if (rule.id === "memoryGrowthMBPerHour") { + return `Memory growth, ${rule.process}`; + } + return rule.id === "failedCallsPct" ? `Failed ${unit}` : "Restarts"; +} + +function measuredOf(rule: SoakRule): string { + if (rule.id === "memoryGrowthMBPerHour") return growth(rule.measured); + return rule.id === "failedCallsPct" + ? pct(rule.measured) + : num(rule.measured, 0); +} + +function limitOf(rule: SoakRule): string { + if (rule.id === "memoryGrowthMBPerHour") return `${num(rule.limit)} MB/h`; + return rule.id === "failedCallsPct" ? pct(rule.limit) : num(rule.limit, 0); +} diff --git a/src/summary/markdown.ts b/src/summary/markdown.ts new file mode 100644 index 0000000..dd44e99 --- /dev/null +++ b/src/summary/markdown.ts @@ -0,0 +1,73 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +/** + * How the run summary writes Markdown and numbers. It is plain GitHub + * Markdown, for $GITHUB_STEP_SUMMARY, so it shares nothing with the HTML + * page's formatting. + */ +import type { Verdict } from "../schemas/index.js"; + +const LABELS: Record = { + passed: "passed", + failed: "failed", + inconclusive: "inconclusive", + aborted: "aborted", + "not-run": "not run" +}; + +/** `passed`, `not run` */ +export function verdictWord(verdict: Verdict): string { + return LABELS[verdict]; +} + +/** `Not run` */ +export function capitalized(text: string): string { + return `${text.charAt(0).toUpperCase()}${text.slice(1)}`; +} + +/** Text safe inside a table cell or a heading: no pipes, no line breaks. */ +export function cell(text: string): string { + return text.replace(/\|/g, "\\|").replace(/\s*\n\s*/g, " "); +} + +/** `1,240`, `0.03`; at most `digits` decimals. */ +export function num(value: number, digits = 1): string { + return value.toLocaleString("en-US", { + maximumFractionDigits: digits, + minimumFractionDigits: 0 + }); +} + +/** `0.03%` */ +export function pct(value: number): string { + return `${num(value, 2)}%`; +} + +/** `+7.4 MB/h` */ +export function growth(value: number): string { + return `${value < 0 ? "-" : "+"}${num(Math.abs(value))} MB/h`; +} + +/** `45 s`, `26 min`, `6 h 30 min` */ +export function duration(seconds: number): string { + if (seconds < 60) return `${Math.round(seconds)} s`; + const minutes = Math.round(seconds / 60); + const hours = Math.floor(minutes / 60); + if (hours === 0) return `${minutes} min`; + return minutes % 60 === 0 ? `${hours} h` : `${hours} h ${minutes % 60} min`; +} + +/** A GitHub Markdown table; numeric columns are right-aligned. */ +export function table( + header: string[], + rows: string[][], + right: boolean[] +): string { + const line = (cells: string[]) => `| ${cells.join(" | ")} |`; + return [ + line(header), + line(right.map((r) => (r ? "---:" : ":---"))), + ...rows.map(line) + ].join("\n"); +} diff --git a/src/summary/summarizeReport.schema.ts b/src/summary/summarizeReport.schema.ts new file mode 100644 index 0000000..f785e6d --- /dev/null +++ b/src/summary/summarizeReport.schema.ts @@ -0,0 +1,10 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { z } from "zod/v4"; + +export const summarizeReportInputSchema = z.strictObject({ + report: z.string().min(1, { error: "report must not be empty" }) +}); + +export type SummarizeReportInput = z.infer; diff --git a/test/cli/runCli.unit.test.ts b/test/cli/runCli.unit.test.ts index 8861b07..335707c 100644 --- a/test/cli/runCli.unit.test.ts +++ b/test/cli/runCli.unit.test.ts @@ -298,3 +298,45 @@ describe("perfkit validate", () => { }); }); }); + +describe("perfkit summary", () => { + it("should print the report as Markdown with exit 0", async () => { + // Act + const result = await run(["summary", "--report", "report.json"], { + "report.json": loadReport("failed") + }); + + // Assert + expect(result.code).to.equal(0); + expect(result.err).to.equal(""); + expect(result.out).to.match(/^## perfkit run 20261020T100005Z-c0b54ab\n/); + expect(result.out).to.include( + "### EdgePort + Echo (`edgeport-echo`): failed" + ); + }); + + it("should print the errors of an invalid report and exit 2", async () => { + // Arrange + const report = loadReport("failed"); + report.verdict.passed = 3; + + // Act + const result = await run(["summary", "--report", "report.json"], { + "report.json": report + }); + + // Assert + expect(result.code).to.equal(2); + expect(result.out).to.equal(""); + expect(result.err).to.match(/^report\.json: verdict/); + }); + + it("should exit 1 without --report", async () => { + // Act + const result = await run(["summary"]); + + // Assert + expect(result.code).to.equal(1); + expect(result.err).to.include("--report "); + }); +}); diff --git a/test/summary/createSummarizeReport.unit.test.ts b/test/summary/createSummarizeReport.unit.test.ts new file mode 100644 index 0000000..3531ec0 --- /dev/null +++ b/test/summary/createSummarizeReport.unit.test.ts @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { ValidationError } from "../../src/errors/ValidationError.js"; +import { createSummarizeReport } from "../../src/summary/createSummarizeReport.js"; +import { type Json, loadReport, loadRuns } from "../fixtures.js"; + +function summarize(memory: Record) { + return createSummarizeReport({ + readFile: async (path) => { + if (!(path in memory)) { + throw Object.assign(new Error("nope"), { code: "ENOENT" }); + } + return JSON.stringify(memory[path]); + }, + isFile: async (path) => path in memory + }); +} + +describe("createSummarizeReport", () => { + it("should return the report as Markdown", async () => { + // Act + const result = await summarize({ "report.json": loadReport("passed") })({ + report: "report.json" + }); + + // Assert + expect(result.ok).to.equal(true); + if (!result.ok) return; + expect(result.markdown).to.match( + /^## perfkit run 20261103T141210Z-bd3254a\n/ + ); + }); + + it("should return the errors of an invalid report", async () => { + // Arrange + const report = loadReport("passed"); + delete report.run.commit; + + // Act + const result = await summarize({ "report.json": report })({ + report: "report.json" + }); + + // Assert + expect(result).to.deep.equal({ + ok: false, + errors: ["report.json: run.commit: required"] + }); + }); + + it("should refuse a runs.json", async () => { + // Act + const result = await summarize({ "runs.json": loadRuns() })({ + report: "runs.json" + }); + + // Assert + expect(result).to.deep.equal({ + ok: false, + errors: [ + "runs.json: expected a perfkit/report/v1 report; got perfkit/runs/v1" + ] + }); + }); + + it("should report a missing file", async () => { + // Act + const result = await summarize({})({ report: "missing.json" }); + + // Assert + expect(result).to.deep.equal({ + ok: false, + errors: ["missing.json: cannot read: no such file"] + }); + }); + + it("should throw ValidationError for an empty path", async () => { + // Act + const error = await summarize({})({ report: "" }).catch((e: unknown) => e); + + // Assert + expect(error).to.be.instanceOf(ValidationError); + }); +}); diff --git a/test/summary/formatCapacitySummary.unit.test.ts b/test/summary/formatCapacitySummary.unit.test.ts new file mode 100644 index 0000000..3767075 --- /dev/null +++ b/test/summary/formatCapacitySummary.unit.test.ts @@ -0,0 +1,116 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { + type CapacityBenchmarkResult, + reportSchema +} from "../../src/schemas/index.js"; +import { formatCapacitySummary } from "../../src/summary/formatCapacitySummary.js"; +import { loadReport, REPORT_FORMS } from "../fixtures.js"; + +function capacityOf( + form: (typeof REPORT_FORMS)[number] +): CapacityBenchmarkResult { + const benchmark = reportSchema + .parse(loadReport(form)) + .benchmarks.find(({ kind }) => kind === "capacity"); + return benchmark as CapacityBenchmarkResult; +} + +describe("formatCapacitySummary", () => { + it("should give the limit, where the ramp stopped and every step", () => { + // Arrange + const benchmark = capacityOf("passed"); + + // Act + const text = formatCapacitySummary(benchmark); + + // Assert + const lines = text.split("\n"); + expect(lines[0]).to.equal( + "**Capacity limit:** 1,240 messages/s, p99 30.9 ms, 0.04% failed." + ); + expect(lines[2]).to.equal( + "**Stopped at** 1,300 messages/s: p99 62 ms, limit 50 ms." + ); + expect(lines[4]).to.equal( + "| Offered (messages/s) | Placed (messages/s) | p99 | Failed | Result |" + ); + expect(lines[6]).to.equal("| 50 | 49.6 | 4 ms | 0% | ok |"); + expect(lines.at(-1)).to.equal( + "| 1,300 | 1,289.6 | 62 ms | 0.06% | stopped: p99 over 50 ms |" + ); + expect(lines.length).to.equal( + 6 + (benchmark.verdict === "passed" ? benchmark.series.steps.length : 0) + ); + }); + + it("should say the limit is a lower bound when no step broke a rule", () => { + // Act + const text = formatCapacitySummary(capacityOf("lower-bound")); + + // Assert + expect(text.split("\n")[0]).to.equal( + "**Capacity limit:** at least 5,000 messages/s, p99 18.4 ms, 0.02% failed (no step broke a rule)." + ); + expect(text).to.not.include("Stopped at"); + }); + + it("should say there is no limit when the first step broke a rule", () => { + // Act + const text = formatCapacitySummary(capacityOf("failed")); + + // Assert + expect(text).to.include( + "**Capacity limit:** none, the first step already broke a rule." + ); + expect(text).to.include( + "| 50 | 49.6 | 72 ms | 0.03% | stopped: p99 over 50 ms |" + ); + }); + + it("should mark a step over the failed limit and an invalid step", () => { + // Arrange + const benchmark = capacityOf("passed"); + if (benchmark.verdict !== "passed") throw new Error("expected passed"); + const [first, second] = benchmark.series.steps; + const steps = [ + { ...first!, calls: 1000, failed: 5 }, + { ...second!, valid: false } + ]; + + // Act + const text = formatCapacitySummary({ + ...benchmark, + series: { steps } + }); + + // Assert + expect(text).to.include("| 0.5% | stopped: failed over 0.1% |"); + expect(text).to.include("| invalid; stopped |"); + }); + + it("should list a partial benchmark's steps without limits", () => { + // Act + const text = formatCapacitySummary(capacityOf("not-run")); + + // Assert + expect(text).to.not.include("Capacity limit"); + expect(text.split("\n").at(-1)).to.equal( + "| 200 | 198.4 | 7 ms | 0.03% | ok |" + ); + }); + + it("should be empty for a benchmark that did not run", () => { + // Act + const text = formatCapacitySummary({ + ...capacityOf("passed"), + verdict: "not-run", + reason: "the run was aborted before this benchmark started" + } as CapacityBenchmarkResult); + + // Assert + expect(text).to.equal(""); + }); +}); diff --git a/test/summary/formatReportSummary.unit.test.ts b/test/summary/formatReportSummary.unit.test.ts new file mode 100644 index 0000000..ed05085 --- /dev/null +++ b/test/summary/formatReportSummary.unit.test.ts @@ -0,0 +1,75 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { reportSchema } from "../../src/schemas/index.js"; +import { formatReportSummary } from "../../src/summary/formatReportSummary.js"; +import { loadReport, REPORT_FORMS } from "../fixtures.js"; + +const parsed = (form: (typeof REPORT_FORMS)[number]) => + reportSchema.parse(loadReport(form)); + +describe("formatReportSummary", () => { + it("should open with the run id, target version, commit and verdict", () => { + // Act + const text = formatReportSummary(parsed("failed")); + + // Assert + expect(text.split("\n").slice(0, 5)).to.deep.equal([ + "## perfkit run 20261020T100005Z-c0b54ab", + "", + "**Routr 2.31.3**, commit `c0b54abe09d1`. **0 of 2 benchmarks passed** (2 failed).", + "", + "Started 2026-10-20T10:00:05Z, ended 2026-10-20T16:41:30Z; digitalocean nyc3, server under test c-8, load generator c-16; perfkit 0.1.0." + ]); + expect(text.endsWith("\n")).to.equal(true); + }); + + it("should give each benchmark a heading with its verdict", () => { + // Act + const text = formatReportSummary(parsed("failed")); + + // Assert + expect(text).to.include( + "### EdgePort + Echo (`edgeport-echo`): failed\n\nCapacity benchmark, 1 min, SIPp 3.7.3, UDP." + ); + expect(text).to.include( + "### RoutrOne, six hours (`one-soak`): failed\n\nSoak benchmark, 6 h 30 min, SIPp 3.7.3, UDP; OpenTelemetry." + ); + expect(text).to.include( + "| Memory growth, one | +7.4 MB/h | 5 MB/h | failed |" + ); + }); + + it("should give the reason of an aborted and a not-run benchmark", () => { + // Act + const text = formatReportSummary(parsed("not-run")); + + // Assert + expect(text).to.include("(1 aborted, 1 not run)"); + expect(text).to.include( + "### EdgePort + Echo (`edgeport-echo`): aborted\n\nCapacity benchmark, 4 min, SIPp 3.7.3, UDP.\n\n**Aborted:** the server under test stopped responding." + ); + expect(text).to.include( + "### RoutrOne, six hours (`one-soak`): not run\n\nSoak benchmark.\n\n**Not run:** the run was aborted before this benchmark started.\n" + ); + }); + + it("should give the reason of an inconclusive benchmark", () => { + // Act + const text = formatReportSummary(parsed("inconclusive")); + + // Assert + expect(text).to.include( + "**Inconclusive:** load generator saturated at 1,300 messages/s (SIPp at 93% of one core)." + ); + }); + + it("should not list other counts when every benchmark passed", () => { + // Act + const text = formatReportSummary(parsed("passed")); + + // Assert + expect(text).to.include("**2 of 2 benchmarks passed**."); + }); +}); diff --git a/test/summary/formatSoakSummary.unit.test.ts b/test/summary/formatSoakSummary.unit.test.ts new file mode 100644 index 0000000..69235b9 --- /dev/null +++ b/test/summary/formatSoakSummary.unit.test.ts @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { + reportSchema, + type SoakBenchmarkResult +} from "../../src/schemas/index.js"; +import { formatSoakSummary } from "../../src/summary/formatSoakSummary.js"; +import { loadReport, REPORT_FORMS } from "../fixtures.js"; + +function soakOf(form: (typeof REPORT_FORMS)[number]): SoakBenchmarkResult { + const benchmark = reportSchema + .parse(loadReport(form)) + .benchmarks.find(({ kind }) => kind === "soak"); + return benchmark as SoakBenchmarkResult; +} + +describe("formatSoakSummary", () => { + it("should list the load, every rule against its limit and p99 by hour", () => { + // Act + const text = formatSoakSummary(soakOf("failed")); + + // Assert + expect(text).to.equal( + [ + "**Load:** 100 calls/s, each held 30 s; 2,160,000 calls in all.", + "", + "| Rule | Measured | Limit | Result |", + "| :--- | ---: | ---: | :--- |", + "| Memory growth, one | +7.4 MB/h | 5 MB/h | failed |", + "| Memory growth, edgeport | +0.3 MB/h | 5 MB/h | passed |", + "| Memory growth, postgres | +0.1 MB/h | 5 MB/h | passed |", + "| Failed calls | 0.01% | 0.1% | passed |", + "| Restarts | 0 | 0 | passed |", + "", + "**p99:** 21 ms in the first hour, 34 ms in the last hour." + ].join("\n") + ); + }); + + it("should be empty for an aborted soak", () => { + // Act / Assert + expect(formatSoakSummary(soakOf("aborted-soak"))).to.equal(""); + }); +}); diff --git a/test/summary/markdown.unit.test.ts b/test/summary/markdown.unit.test.ts new file mode 100644 index 0000000..742f2eb --- /dev/null +++ b/test/summary/markdown.unit.test.ts @@ -0,0 +1,75 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { + capitalized, + cell, + duration, + growth, + num, + pct, + table, + verdictWord +} from "../../src/summary/markdown.js"; + +describe("summary markdown helpers", () => { + it("should name each verdict in lowercase words", () => { + // Act + const words = ( + ["passed", "failed", "inconclusive", "aborted", "not-run"] as const + ).map(verdictWord); + + // Assert + expect(words).to.deep.equal([ + "passed", + "failed", + "inconclusive", + "aborted", + "not run" + ]); + }); + + it("should capitalize the first letter only", () => { + // Act / Assert + expect(capitalized("not run")).to.equal("Not run"); + expect(capitalized("")).to.equal(""); + }); + + it("should escape pipes and join lines in a cell", () => { + // Act / Assert + expect(cell("a | b\n c")).to.equal("a \\| b c"); + }); + + it("should write numbers, percentages, growth and durations", () => { + // Act / Assert + expect(num(1240.84)).to.equal("1,240.8"); + expect(num(2160000, 0)).to.equal("2,160,000"); + expect(pct(0.034)).to.equal("0.03%"); + expect(growth(7.44)).to.equal("+7.4 MB/h"); + expect(growth(-0.1)).to.equal("-0.1 MB/h"); + expect(duration(45)).to.equal("45 s"); + expect(duration(1560)).to.equal("26 min"); + expect(duration(21600)).to.equal("6 h"); + expect(duration(23400)).to.equal("6 h 30 min"); + }); + + it("should write a table with right-aligned numeric columns", () => { + // Act + const text = table( + ["Name", "Value"], + [ + ["a", "1"], + ["b", "2"] + ], + [false, true] + ); + + // Assert + expect(text).to.equal( + ["| Name | Value |", "| :--- | ---: |", "| a | 1 |", "| b | 2 |"].join( + "\n" + ) + ); + }); +});