From 1743c400d5962600ba0dbdd49cae4dda0805a537 Mon Sep 17 00:00:00 2001 From: Alex Langenfeld Date: Mon, 17 Aug 2026 11:57:38 -0500 Subject: [PATCH] bench: consolidate stream throughput columns Signed-off-by: Alex Langenfeld --- .changeset/calm-geese-share.md | 4 ++++ .github/scripts/render-benchmark-comment.mjs | 13 +++++----- .../scripts/render-benchmark-comment.test.js | 24 +++++++------------ 3 files changed, 19 insertions(+), 22 deletions(-) create mode 100644 .changeset/calm-geese-share.md diff --git a/.changeset/calm-geese-share.md b/.changeset/calm-geese-share.md new file mode 100644 index 0000000000..bac882809b --- /dev/null +++ b/.changeset/calm-geese-share.md @@ -0,0 +1,4 @@ +--- +--- + +Remove low-value throughput columns from the CI stream benchmark report. diff --git a/.github/scripts/render-benchmark-comment.mjs b/.github/scripts/render-benchmark-comment.mjs index 24ee66f42c..b944a3068a 100644 --- a/.github/scripts/render-benchmark-comment.mjs +++ b/.github/scripts/render-benchmark-comment.mjs @@ -787,9 +787,8 @@ function pairedSortKey(a, b) { /** * The stream-scenario table: one row per stream scenario, with the columns - * streams actually want — writer-side achieved and reader-side delivered - * sustained rates (steady window: first/last 10% of chunks trimmed), CRTT - * percentiles, and the median worst delivery stall (CDV max positive). + * streams actually want — CRTT percentiles and the median worst delivery + * stall (CDV max positive). * Deltas vs main are plain percentages; deliberately NO 🔴/🟢 marks — targets * attach in a later PR once a baseline exists. Rates read higher-is-better, * latencies lower-is-better, so directional marks would need per-column @@ -813,14 +812,14 @@ function renderStreamTable(result) { const lines = [ '**Streams**', '', - '| Scenario | wr c/s | rd c/s | wr KiB/s | rd KiB/s | CRTT 1st | p75 | p90 | p99 | CDV max | iters |', - '|----------|-------:|-------:|---------:|---------:|---------:|----:|----:|----:|--------:|------:|', + '| Scenario | CRTT 1st | p75 | p90 | p99 | CDV max | iters |', + '|----------|---------:|----:|----:|----:|--------:|------:|', ]; for (const row of rows) { const s = row.stream; const b = row.baselineStream ?? {}; lines.push( - `| ${row.scenario} | ${cell(s.wrCps, b.wrCps)} | ${cell(s.rdCps, b.rdCps)} | ${cell(s.wrKiBps, b.wrKiBps)} | ${cell(s.rdKiBps, b.rdKiBps)} | ${cell(s.firstMs, b.firstMs)} | ${cell(row.p75, row.baselineP75)} | ${cell(row.p90, row.baselineP90)} | ${cell(row.p99, row.baselineP99)} | ${cell(s.cdvMaxMs, b.cdvMaxMs)} | ${s.iterations} |` + `| ${row.scenario} | ${cell(s.firstMs, b.firstMs)} | ${cell(row.p75, row.baselineP75)} | ${cell(row.p90, row.baselineP90)} | ${cell(row.p99, row.baselineP99)} | ${cell(s.cdvMaxMs, b.cdvMaxMs)} | ${s.iterations} |` ); } return lines.join('\n'); @@ -995,7 +994,7 @@ function renderFooter(entries) { const smallprint = [ ...(hasStreamTable ? [ - '**Streams**: writer/reader sustained rates (steady window, 10% trimmed each side), first-chunk RTT (the stream-open path, before any buffering/backpressure), CRTT percentiles, and worst delivery stall (CDV max). Cells are medians across iterations; per-run values in the artifacts. No \ud83d\udd34/\ud83d\udfe2 marks until targets attach.', + '**Streams**: first-chunk RTT (the stream-open path, before any buffering/backpressure), CRTT percentiles, and worst delivery stall (CDV max). Cells are medians across iterations; per-run values in the artifacts. No \ud83d\udd34/\ud83d\udfe2 marks until targets attach.', '', ] : []), diff --git a/.github/scripts/render-benchmark-comment.test.js b/.github/scripts/render-benchmark-comment.test.js index c00a4323ed..e9284fe046 100644 --- a/.github/scripts/render-benchmark-comment.test.js +++ b/.github/scripts/render-benchmark-comment.test.js @@ -706,7 +706,7 @@ function crttResult({ avg = 120, hist }) { }); } -test('renders stream scenarios in their own table with rate columns', async () => { +test('renders stream scenarios in their own table without rate columns', async () => { const { renderComment, extractHistory } = await loadModule(); const hist = crttHist([ [59, 1400], @@ -728,18 +728,12 @@ test('renders stream scenarios in their own table with rate columns', async () = assert.doesNotMatch(body, /\| \*\*stream\*\* \|/); assert.match( body, - /\| Scenario \| wr c\/s \| rd c\/s \| wr KiB\/s \| rd KiB\/s \| CRTT 1st \| p75 \| p90 \| p99 \| CDV max \| iters \|/ - ); - // Rate cells with plain vs-main deltas, latency cells from percentile - // baselines, and NO red/green marks anywhere in the stream table. - assert.match( - body, - /\| chunk RTT \(llm\) \| 100 \(\u00b10%\) \| 99\.4 \(\+10%\) \| 6\.1 \(\u00b10%\) \|/ - ); - assert.match( - body, - /\| replay eve-test \(2x\) \| 297 \(\u00b10%\) \| 288 \(\u00b10%\) \| 742 \(\u00b10%\) \| 719 \(\u00b10%\) \|/ + /\| Scenario \| CRTT 1st \| p75 \| p90 \| p99 \| CDV max \| iters \|/ ); + // Latency cells retain their vs-main deltas, and the stream table has no + // red/green marks. + assert.match(body, /\| chunk RTT \(llm\) \| 96 \(\u00b10%\) \|/); + assert.match(body, /\| replay eve-test \(2x\) \| 118 \(\u00b10%\) \|/); assert.match(body, /\| 141 \(\u00b10%\) \| 10 \|/); const streamsSection = body.slice( body.indexOf('**Streams**'), @@ -763,7 +757,7 @@ test('renders stream scenarios in their own table with rate columns', async () = // the internal 'stream' id never leaks into it. assert.match(body, /\*\*CRTT\*\*: chunk round-trip time/); assert.match(body, /\*\*CDV\*\*: chunk delay variation/); - assert.match(body, /\*\*Streams\*\*: writer\/reader sustained rates/); + assert.match(body, /\*\*Streams\*\*: first-chunk RTT/); // History block: per-run arrays and sparkline payloads stripped, medians // and baseline annotations kept. const history = extractHistory(body); @@ -780,7 +774,7 @@ test('renders stream scenarios in their own table with rate columns', async () = history, commit: 'ffffff1234567890', }); - assert.match(rerendered, /\| chunk RTT \(llm\) \| 100/); + assert.match(rerendered, /\| chunk RTT \(llm\) \| 96/); assert.doesNotMatch(rerendered, /CRTT drill-down/); }); @@ -794,7 +788,7 @@ test('renders the stream table without deltas when main has no baseline', async }); assert.match( body, - /\| chunk RTT \(llm\) \| 100 \| 99\.4 \| 6\.1 \| 6 \| 96 \| 188 \| 438 \| 1229 \| 141 \| 10 \|/ + /\| chunk RTT \(llm\) \| 96 \| 188 \| 438 \| 1229 \| 141 \| 10 \|/ ); assert.doesNotMatch(body, /%\)/); assert.match(body, /No `main` baseline yet/);