Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .changeset/calm-geese-share.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
---

Remove low-value throughput columns from the CI stream benchmark report.
13 changes: 6 additions & 7 deletions .github/scripts/render-benchmark-comment.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -787,9 +787,8 @@ function pairedSortKey(a, b) {

/**
* The stream-scenario table: one row per stream scenario, with the columns
* streams actually want — writer-side achieved and reader-side delivered
* sustained rates (steady window: first/last 10% of chunks trimmed), CRTT
* percentiles, and the median worst delivery stall (CDV max positive).
* streams actually want — CRTT percentiles and the median worst delivery
* stall (CDV max positive).
* Deltas vs main are plain percentages; deliberately NO 🔴/🟢 marks — targets
* attach in a later PR once a baseline exists. Rates read higher-is-better,
* latencies lower-is-better, so directional marks would need per-column
Expand All @@ -813,14 +812,14 @@ function renderStreamTable(result) {
const lines = [
'**Streams**',
'',
'| Scenario | wr c/s | rd c/s | wr KiB/s | rd KiB/s | CRTT 1st | p75 | p90 | p99 | CDV max | iters |',
'|----------|-------:|-------:|---------:|---------:|---------:|----:|----:|----:|--------:|------:|',
'| Scenario | CRTT 1st | p75 | p90 | p99 | CDV max | iters |',
'|----------|---------:|----:|----:|----:|--------:|------:|',
];
for (const row of rows) {
const s = row.stream;
const b = row.baselineStream ?? {};
lines.push(
`| ${row.scenario} | ${cell(s.wrCps, b.wrCps)} | ${cell(s.rdCps, b.rdCps)} | ${cell(s.wrKiBps, b.wrKiBps)} | ${cell(s.rdKiBps, b.rdKiBps)} | ${cell(s.firstMs, b.firstMs)} | ${cell(row.p75, row.baselineP75)} | ${cell(row.p90, row.baselineP90)} | ${cell(row.p99, row.baselineP99)} | ${cell(s.cdvMaxMs, b.cdvMaxMs)} | ${s.iterations} |`
`| ${row.scenario} | ${cell(s.firstMs, b.firstMs)} | ${cell(row.p75, row.baselineP75)} | ${cell(row.p90, row.baselineP90)} | ${cell(row.p99, row.baselineP99)} | ${cell(s.cdvMaxMs, b.cdvMaxMs)} | ${s.iterations} |`
);
}
return lines.join('\n');
Expand Down Expand Up @@ -995,7 +994,7 @@ function renderFooter(entries) {
const smallprint = [
...(hasStreamTable
? [
'<sub>**Streams**: writer/reader sustained rates (steady window, 10% trimmed each side), first-chunk RTT (the stream-open path, before any buffering/backpressure), CRTT percentiles, and worst delivery stall (CDV max). Cells are medians across iterations; per-run values in the artifacts. No \ud83d\udd34/\ud83d\udfe2 marks until targets attach.</sub>',
'<sub>**Streams**: first-chunk RTT (the stream-open path, before any buffering/backpressure), CRTT percentiles, and worst delivery stall (CDV max). Cells are medians across iterations; per-run values in the artifacts. No \ud83d\udd34/\ud83d\udfe2 marks until targets attach.</sub>',
'',
]
: []),
Expand Down
24 changes: 9 additions & 15 deletions .github/scripts/render-benchmark-comment.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -706,7 +706,7 @@ function crttResult({ avg = 120, hist }) {
});
}

test('renders stream scenarios in their own table with rate columns', async () => {
test('renders stream scenarios in their own table without rate columns', async () => {
const { renderComment, extractHistory } = await loadModule();
const hist = crttHist([
[59, 1400],
Expand All @@ -728,18 +728,12 @@ test('renders stream scenarios in their own table with rate columns', async () =
assert.doesNotMatch(body, /\| \*\*stream\*\* \|/);
assert.match(
body,
/\| Scenario \| wr c\/s \| rd c\/s \| wr KiB\/s \| rd KiB\/s \| CRTT 1st \| p75 \| p90 \| p99 \| CDV max \| iters \|/
);
// Rate cells with plain vs-main deltas, latency cells from percentile
// baselines, and NO red/green marks anywhere in the stream table.
assert.match(
body,
/\| chunk RTT \(llm\) \| 100 \(\u00b10%\) \| 99\.4 \(\+10%\) \| 6\.1 \(\u00b10%\) \|/
);
assert.match(
body,
/\| replay eve-test \(2x\) \| 297 \(\u00b10%\) \| 288 \(\u00b10%\) \| 742 \(\u00b10%\) \| 719 \(\u00b10%\) \|/
/\| Scenario \| CRTT 1st \| p75 \| p90 \| p99 \| CDV max \| iters \|/
);
// Latency cells retain their vs-main deltas, and the stream table has no
// red/green marks.
assert.match(body, /\| chunk RTT \(llm\) \| 96 \(\u00b10%\) \|/);
assert.match(body, /\| replay eve-test \(2x\) \| 118 \(\u00b10%\) \|/);
assert.match(body, /\| 141 \(\u00b10%\) \| 10 \|/);
const streamsSection = body.slice(
body.indexOf('**Streams**'),
Expand All @@ -763,7 +757,7 @@ test('renders stream scenarios in their own table with rate columns', async () =
// the internal 'stream' id never leaks into it.
assert.match(body, /\*\*CRTT\*\*: chunk round-trip time/);
assert.match(body, /\*\*CDV\*\*: chunk delay variation/);
assert.match(body, /\*\*Streams\*\*: writer\/reader sustained rates/);
assert.match(body, /\*\*Streams\*\*: first-chunk RTT/);
// History block: per-run arrays and sparkline payloads stripped, medians
// and baseline annotations kept.
const history = extractHistory(body);
Expand All @@ -780,7 +774,7 @@ test('renders stream scenarios in their own table with rate columns', async () =
history,
commit: 'ffffff1234567890',
});
assert.match(rerendered, /\| chunk RTT \(llm\) \| 100/);
assert.match(rerendered, /\| chunk RTT \(llm\) \| 96/);
assert.doesNotMatch(rerendered, /CRTT drill-down/);
});

Expand All @@ -794,7 +788,7 @@ test('renders the stream table without deltas when main has no baseline', async
});
assert.match(
body,
/\| chunk RTT \(llm\) \| 100 \| 99\.4 \| 6\.1 \| 6 \| 96 \| 188 \| 438 \| 1229 \| 141 \| 10 \|/
/\| chunk RTT \(llm\) \| 96 \| 188 \| 438 \| 1229 \| 141 \| 10 \|/
);
assert.doesNotMatch(body, /%\)/);
assert.match(body, /No `main` baseline yet/);
Expand Down
Loading