diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index dbee3ba545..5c5eac8945 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -322,6 +322,7 @@ jobs: const path = require('path') // vlt must be more than 10% behind the fastest registry before a // difference counts as noticeably slower. + const { benchmarkStatistics } = require(`${process.env.GITHUB_WORKSPACE}/scripts/benchmark-statistics.js`) const THRESHOLD = 1.1 const slower = [] @@ -339,14 +340,15 @@ jobs: // not establish a fair speed comparison. Failures are already // reported by the raw scan before processing. const ok = (JSON.parse(fs.readFileSync(path.join(dir, entry), 'utf8')).results ?? []) - .filter((r) => r.mean > 0 && r.status !== 'partial' && !(r.dropped_runs > 0) && (r.exit_codes ?? []).every((c) => c === 0)) + .map((r) => ({ command: r.command, ...benchmarkStatistics(r) })) + .filter((r) => !r.failed && !r.partial && r.statistic === 'median' && r.value > 0) const vlt = ok.find((r) => r.command === 'vlt') if (!vlt) continue - const best = ok.reduce((a, b) => (b.mean < a.mean ? b : a)) - if (best.command === 'vlt' || vlt.mean <= best.mean * THRESHOLD) continue + const best = ok.reduce((a, b) => (b.value < a.value ? b : a)) + if (best.command === 'vlt' || vlt.value <= best.value * THRESHOLD) continue slower.push( - `- **${label}**: ${best.command} ${best.mean.toFixed(1)}s vs vlt ${vlt.mean.toFixed(1)}s (${(vlt.mean / best.mean).toFixed(2)}x slower)` + `- **${label}**: ${best.command} ${best.value.toFixed(1)}s median vs vlt ${vlt.value.toFixed(1)}s (${(vlt.value / best.value).toFixed(2)}x slower)` ) } diff --git a/README.md b/README.md index b4c9d24826..cf5c539dc8 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,8 @@ aggregate CPU values, which cannot be recalculated for the surviving sample. Chart data adds optional `_partial`, `_attempted_runs`, `_successful_runs`, and `_dropped_runs` fields in both total and per-package datasets. Tables and chart notices label partial results with their -success counts. Synthetic averages retain that warning. Commands with any +success counts. Synthetic averages retain that warning and omit the affected +command's value. Commands with any partial result in the selected comparisons are excluded from rankings; history omits that command's affected daily variation and category average. @@ -227,6 +228,41 @@ recover counts when those artifacts are still available. CI checks that inspect raw `exit_codes` must run before `./bench process`; the registry failure scan already does so. Persisted original codes also remain available in dated result files on gh-pages. +### Timing statistic and sample quality + +The canonical timing is the **median of successful measured runs** for each +fixture, variation, and tool/registry. For `[3, 4, 294]` seconds, the chart shows +4 seconds instead of the 100.3-second mean. No successful outlier is discarded: +the table reports the successful/attempted sample count and full observed range +(3–294 seconds here). The range is descriptive, not a confidence interval; +three samples provide limited evidence, even when their median looks stable. + +The first **measured** run (run 0) is intentionally included. We do not assume +that it is a warmup or steady-state run: each variation controls its cache and +lockfile preparation. Explicit Hyperfine warmups are excluded from `times` and +therefore from the median and sample count. A one-run result is labeled 1/1; +its equal min/max does not imply certainty. + +Dated and latest raw results retain mean, median, standard deviation, and +individual successful timings. Chart data uses the median (seconds), or the +same median multiplied by 1000 and divided by package count (ms/package). +Failed attempts are excluded from timing statistics but retained in completeness +metadata. Partial survivor samples are labeled and excluded from averages, +history, leaderboard timing/win calculations, and registry speed alerts. +All-failed results remain DNF; the existing leaderboard DNF penalty uses the +slowest successful median for that fixture. + +Average views and leaderboard timing values are arithmetic averages of +per-benchmark medians, **not pooled medians**. History averages those medians +across available fixtures, and category history additionally averages across +variations. Coverage may vary by date; neither an averaged standard deviation +nor an invented confidence interval is shown for aggregates. + +Older chart files without statistic metadata remain readable in the latest +view and are labeled **legacy mean**. They are omitted from median history to +avoid a false trend at the methodology change. Reprocessing the original dated +results regenerates median points from `times` or stored `median`; a mean-only +raw result stays labeled `legacy-mean` and cannot be converted to a median. ### Console Output @@ -234,7 +270,7 @@ Each benchmark run provides a summary in the console: ``` === Project Name (cache type) === -package-manager: X.XXs (stddev: X.XXs) +package-manager: 4s (median; 3/3 successful runs; range 3s–294s) ... ``` diff --git a/app/src/components/history-chart.tsx b/app/src/components/history-chart.tsx index 1e462433b1..309f831af6 100644 --- a/app/src/components/history-chart.tsx +++ b/app/src/components/history-chart.tsx @@ -236,6 +236,11 @@ export const HistoryChart = ({ +

+ Arithmetic averages of complete benchmark medians across available fixtures. + Legacy mean-only dates and incomplete samples are omitted. The average view + also averages across variations; coverage may differ by date. +

Days with partial results are omitted for that command and variation. diff --git a/app/src/components/variation/chart.tsx b/app/src/components/variation/chart.tsx index e57a12c486..a3de50f14d 100644 --- a/app/src/components/variation/chart.tsx +++ b/app/src/components/variation/chart.tsx @@ -408,7 +408,7 @@ export const VariationChart = ({ const isActive = variationActivePackageManagers.has(pm); const hasNumber = typeof value === "number"; const shouldFallback = - isActive && !hasNumber && typeof slowest === "number"; + isActive && !hasNumber && item[`${pm}_partial`] !== true && typeof slowest === "number"; const isDnf = item[dnfKey] === true || shouldFallback; if (hasNumber) { @@ -499,10 +499,10 @@ export const VariationChart = ({ const value = fixtureResult[pm]; const hasNumber = typeof value === "number"; const shouldFallback = - !hasNumber && typeof slowest === "number"; + !hasNumber && fixtureResult[`${pm}_partial`] !== true && typeof slowest === "number"; const isDnf = fixtureResult[dnfKey] === true || shouldFallback; - const resolvedValue = hasNumber ? value : slowest; + const resolvedValue = hasNumber ? value : shouldFallback ? slowest : undefined; if (typeof resolvedValue !== "number") return null; @@ -762,10 +762,10 @@ export const VariationChart = ({ const value = fixtureResult[pm]; const hasNumber = typeof value === "number"; const shouldFallback = - !hasNumber && typeof slowest === "number"; + !hasNumber && fixtureResult[`${pm}_partial`] !== true && typeof slowest === "number"; const isDnf = fixtureResult[dnfKey] === true || shouldFallback; - const resolvedValue = hasNumber ? value : slowest; + const resolvedValue = hasNumber ? value : shouldFallback ? slowest : undefined; if (typeof resolvedValue !== "number") return null; @@ -1016,9 +1016,9 @@ export const VariationChart = ({ const dnfKey = `${pm}_dnf` as keyof FixtureResult; const value = fixtureResult[pm]; const hasNumber = typeof value === "number"; - const shouldFallback = !hasNumber && typeof slowest === "number"; + const shouldFallback = !hasNumber && fixtureResult[`${pm}_partial`] !== true && typeof slowest === "number"; const isDnf = fixtureResult[dnfKey] === true || shouldFallback; - const resolvedValue = hasNumber ? value : slowest; + const resolvedValue = hasNumber ? value : shouldFallback ? slowest : undefined; if (typeof resolvedValue !== "number") { return null; diff --git a/app/src/components/variation/index.tsx b/app/src/components/variation/index.tsx index 3c5766e812..f50120d05a 100644 --- a/app/src/components/variation/index.tsx +++ b/app/src/components/variation/index.tsx @@ -219,6 +219,14 @@ export const VariationPage = () => {

)} +

+ Times use the median of measured runs, including the first measured run; + explicit warmups are excluded. Tables show successful/attempted runs and + the observed range, which is not a confidence interval. Averages combine + per-benchmark medians and exclude partial samples. Older results are + labeled as legacy means. +

+ {/* History chart - performance over time */} {historyData && ( >; fixtureDnf: Partial>; fixturePartial: Partial>; + fixtureSamples: Partial>; } const columnHelper = createColumnHelper(); @@ -94,6 +95,7 @@ export const VariationTable = ({ const fixtureValues: Partial> = {}; const fixtureDnf: Partial> = {}; const fixturePartial: Partial> = {}; + const fixtureSamples: Partial> = {}; variationData.forEach((fixtureResult) => { const fixture = fixtureResult.fixture; @@ -104,6 +106,18 @@ export const VariationTable = ({ { ...fixtureResult }, packageManager, ); + const stat = fixtureResult[`${packageManager}_statistic`]; + const count = fixtureResult[`${packageManager}_sample_count`]; + const attempted = fixtureResult[`${packageManager}_attempted_runs`]; + const min = fixtureResult[`${packageManager}_min`]; + const max = fixtureResult[`${packageManager}_max`]; + const unit = isPerPackage ? "ms/pkg" : "s"; + const range = typeof min === "number" && typeof max === "number" + ? ` · range ${min.toFixed(2)}–${max.toFixed(2)} ${unit}` : ""; + const partial = fixtureResult[`${packageManager}_partial`] === true ? " · partial" : ""; + fixtureSamples[fixture] = stat === "average-of-medians" || stat === "legacy-average" + ? `${stat === "average-of-medians" ? "Average of medians" : "Legacy average"} · ${fixtureResult[`${packageManager}_variation_count`]} variations` + : `${stat === "median" ? "Median" : "Legacy mean"} · ${count === undefined ? "sample count unknown" : `${count}/${attempted ?? count} runs`}${range}${partial}`; if (isDnf) { fixtureDnf[fixture] = true; @@ -118,9 +132,10 @@ export const VariationTable = ({ fixtureValues, fixtureDnf, fixturePartial, + fixtureSamples, }; }), - [filteredPackageManagers, variationData], + [filteredPackageManagers, variationData, isPerPackage], ); const columns = useMemo( @@ -199,12 +214,19 @@ export const VariationTable = ({ {partial} )} +
+ {info.row.original.fixtureSamples[fixture]} +
); } return (
- - + {partial ? ( + + {partial} · excluded from average + + ) : -}
); }, diff --git a/app/src/hooks/use-history-data.ts b/app/src/hooks/use-history-data.ts index 0a715fd6fb..0578ea372e 100644 --- a/app/src/hooks/use-history-data.ts +++ b/app/src/hooks/use-history-data.ts @@ -150,7 +150,15 @@ function extractFromDataSet( for (const fixture of fixtures) { for (const pm of PACKAGE_MANAGERS) { - if (partialPMs.has(pm) || fixture[`${pm}_dnf`] === true) continue; + // Never splice legacy mean points into a median series. Reprocessing + // dated raw files adds median metadata and restores those dates. + if ( + partialPMs.has(pm) || + fixture[`${pm}_statistic`] !== "median" || + fixture[`${pm}_dnf`] === true || + fixture[`${pm}_partial`] === true + ) + continue; const val = fixture[pm]; if (typeof val === "number" && Number.isFinite(val)) { if (!pmTotals[pm]) pmTotals[pm] = { sum: 0, count: 0 }; diff --git a/app/src/lib/utils.ts b/app/src/lib/utils.ts index c90fd7b90f..4a193f748d 100644 --- a/app/src/lib/utils.ts +++ b/app/src/lib/utils.ts @@ -42,10 +42,6 @@ export const calculateAverageVariationData = ( }, ): FixtureResult[] => { type FillKey = Extract; - type StddevKey = Extract< - keyof PackageManagerData, - `${PackageManager}_stddev` - >; type CountKey = Extract; type DnfKey = Extract; @@ -107,9 +103,12 @@ export const calculateAverageVariationData = ( averagedResult[key] = results.reduce((sum, r) => sum + (r[key] ?? 0), 0); } } + // Keep the warning visible without publishing an average based on + // surviving runs or quietly omitting this command's slow cases. + return; } const values = results - .filter((r) => r[dnfKey] !== true) + .filter((r) => r[dnfKey] !== true && r[`${pm}_partial`] !== true) .map((r) => r[pm]) .filter((val): val is number => typeof val === "number" && val > 0); @@ -125,25 +124,20 @@ export const calculateAverageVariationData = ( averagedResult[fillKey] = firstFill; } - // Calculate average standard deviation if available - const stddevKey: StddevKey = `${pm}_stddev`; - const stddevValues = results - .filter((r) => r[dnfKey] !== true) - .map((r) => r[stddevKey]) - .filter((val): val is number => typeof val === "number" && val > 0); - - if (stddevValues.length > 0) { - const avgStddev = - stddevValues.reduce((sum, val) => sum + val, 0) / - stddevValues.length; - averagedResult[stddevKey] = avgStddev; - } + // This is an arithmetic average of benchmark medians, not a pooled + // sample. Averaging standard deviations would misrepresent its spread. + const contributing = results.filter((r) => + r[dnfKey] !== true && r[`${pm}_partial`] !== true && + typeof r[pm] === "number" && r[pm]! > 0); + averagedResult[`${pm}_statistic`] = contributing.every((r) => r[`${pm}_statistic`] === "median") + ? "average-of-medians" : "legacy-average"; + averagedResult[`${pm}_variation_count`] = values.length; // For per-package data, also average the count if available if (isPerPackage) { const countKey: CountKey = `${pm}_count`; const countValues = results - .filter((r) => r[dnfKey] !== true) + .filter((r) => r[dnfKey] !== true && r[`${pm}_partial`] !== true) .map((r) => r[countKey]) .filter((val): val is number => typeof val === "number" && val > 0); @@ -326,7 +320,7 @@ export const calculateLeaderboard = ( } // Calculate performance — DNF runs are imputed as the slowest successful - // time for that fixture, matching the "Performance Over Time" chart data + // median for that fixture; history omits failures. variationsToUse.forEach((variation) => { const dataSource = usePerPackageData ? chartData.perPackageCountChartData.data @@ -668,7 +662,7 @@ export const getAvailablePackageManagers = ( allPackageManagers.forEach((pm) => { const value = fixtureResult[pm]; const dnfKey = `${pm}_dnf` as keyof FixtureResult; - if (fixtureResult[dnfKey] === true) { + if (fixtureResult[dnfKey] === true || fixtureResult[`${pm}_partial`] === true) { availablePackageManagers.add(pm); return; } diff --git a/app/src/types/chart-data.ts b/app/src/types/chart-data.ts index db8867934f..29b3398f3e 100644 --- a/app/src/types/chart-data.ts +++ b/app/src/types/chart-data.ts @@ -65,15 +65,21 @@ export interface BaseFixtureResult { fixture: Fixture; } -type RunCompletenessData = Partial< - Record<`${PackageManager}_partial`, boolean> & +type SampleMetadata = Partial< + Record< + `${PackageManager}_${"sample_count" | "attempted_runs" | "successful_runs" | "dropped_runs" | "min" | "max" | "variation_count"}`, + number + > +> & + Partial< Record< - `${PackageManager}_${"attempted_runs" | "successful_runs" | "dropped_runs"}`, - number + `${PackageManager}_statistic`, + "median" | "legacy-mean" | "average-of-medians" | "legacy-average" > ->; + > & + Partial>; -export interface PackageManagerData extends RunCompletenessData { +export interface PackageManagerData extends SampleMetadata { npm?: number; yarn?: number; pnpm?: number; diff --git a/app/tests/partial-results.test.js b/app/tests/partial-results.test.js index fa54fd5bda..fb04e5a317 100644 --- a/app/tests/partial-results.test.js +++ b/app/tests/partial-results.test.js @@ -10,8 +10,15 @@ import { hasPartialResult, } from "../src/hooks/use-history-data.ts"; -const complete = { fixture: "next", npm: 5, vlt: 4 }; +const complete = { + fixture: "next", + npm: 5, + vlt: 4, + npm_statistic: "median", + vlt_statistic: "median", +}; const partial = { + ...complete, fixture: "astro", npm: 0.1, vlt: 3, @@ -84,7 +91,7 @@ test("daily history omits the entire partial command instead of averaging only s assert.equal(hasPartialResult(chartData, ["clean", "cache"], "vlt"), false); }); -test("complete and historical daily results remain usable and DNF placeholders are excluded", () => { +test("complete median results remain usable and DNF placeholders are excluded", () => { const response = { ...chartData, perPackageCountChartData: { @@ -92,7 +99,7 @@ test("complete and historical daily results remain usable and DNF placeholders a data: { clean: [ complete, - { fixture: "astro", npm: 100, npm_dnf: true, vlt: 6 }, + { ...complete, fixture: "astro", npm: 100, npm_dnf: true, vlt: 6 }, ], }, }, diff --git a/app/tests/statistics.test.ts b/app/tests/statistics.test.ts new file mode 100644 index 0000000000..94da043ca1 --- /dev/null +++ b/app/tests/statistics.test.ts @@ -0,0 +1,93 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { extractDayData } from "../src/hooks/use-history-data.ts"; +import { + calculateAverageVariationData, + calculateLeaderboard, +} from "../src/lib/utils.ts"; +import type { + BenchmarkChartData, + FixtureResult, +} from "../src/types/chart-data.ts"; + +const row: FixtureResult = { + fixture: "next", + vlt: 4, + npm: 5, + vlt_statistic: "median", + npm_statistic: "median", + vlt_stddev: 167, + npm_stddev: 1, +}; +const chart = (data: Record) => + ({ + chartData: { + data, + variations: Object.keys(data), + packageManagers: ["vlt", "npm"], + colors: {}, + }, + perPackageCountChartData: { + data, + variations: Object.keys(data), + packageManagers: ["vlt", "npm"], + colors: {}, + }, + versions: {}, + }) as BenchmarkChartData; + +test("aggregate uses medians, labels average-of-medians, and does not invent dispersion", () => { + const result = calculateAverageVariationData( + chart({ clean: [row], lockfile: [{ ...row, vlt: 6 }] }), + ); + assert.equal(result[0].vlt, 5); + assert.equal(result[0].vlt_statistic, "average-of-medians"); + assert.equal(result[0].vlt_variation_count, 2); + assert.equal(result[0].vlt_stddev, undefined); + const ranking = calculateLeaderboard( + chart({ clean: [row] }), + "clean", + "package-managers", + ); + assert.equal(ranking[0].packageManager, "vlt"); + assert.equal(ranking[0].averageTime, 4); + assert.equal(ranking[0].wins, 1); +}); + +test("a partial sample cannot improve a PM's aggregate or leaderboard by dropping its slow case", () => { + const data = chart({ + clean: [row], + lockfile: [{ ...row, vlt: 1, vlt_partial: true }], + }); + assert.equal(calculateAverageVariationData(data)[0].vlt, undefined); + assert.equal( + calculateLeaderboard(data, "average", "package-managers").some( + (r) => r.packageManager === "vlt", + ), + false, + ); +}); + +test("history averages complete medians and omits legacy means and partial comparison sets", () => { + const response = (rows: FixtureResult[]) => ({ + date: "2026-09-15", + chartData: { + variations: ["clean"], + data: { clean: rows }, + packageManagers: ["vlt", "npm"], + }, + }); + const day = extractDayData( + response([row, { ...row, fixture: "astro", vlt: 6 }]), + ); + assert.equal(day.clean.vlt, 5); + assert.equal(day.clean.npm, 5); + const legacy = { ...row }; + delete legacy.vlt_statistic; + assert.equal(extractDayData(response([legacy])).clean.vlt, undefined); + const partial = extractDayData( + response([row, { ...row, vlt_partial: true }]), + ); + assert.equal(partial.clean.vlt, undefined); + assert.equal(partial.clean.npm, 5); +}); diff --git a/scripts/benchmark-data.test.js b/scripts/benchmark-data.test.js index f8fc8e3ff4..71ff8ac769 100644 --- a/scripts/benchmark-data.test.js +++ b/scripts/benchmark-data.test.js @@ -8,10 +8,10 @@ const test = require("node:test"); const { normalizeTiming } = require("./generate-chart.js"); test("normalizes per-package timing and standard deviation", () => { - assert.deepEqual(normalizeTiming({ mean: 1.2, stddev: 0.12 }, 60, true), { - value: 20, - stddev: 2, - }); + const timing = normalizeTiming({ median: 1.2, stddev: 0.12 }, 60, true); + assert.equal(timing.value, 20); + assert.equal(timing.stddev, 2); + assert.equal(timing.statistic, "median"); }); test("does not mix total seconds into per-package data without a count", () => { @@ -26,13 +26,9 @@ test("does not mix total seconds into per-package data without a count", () => { }); test("preserves total timing units", () => { - assert.deepEqual( - normalizeTiming({ mean: 1.2, stddev: 0.12 }, undefined, false), - { - value: 1.2, - stddev: 0.12, - }, - ); + const timing = normalizeTiming({ median: 1.2, stddev: 0.12 }, undefined, false); + assert.equal(timing.value, 1.2); + assert.equal(timing.stddev, 0.12); }); for (const fixture of [ diff --git a/scripts/benchmark-statistics.js b/scripts/benchmark-statistics.js new file mode 100644 index 0000000000..3c154c636d --- /dev/null +++ b/scripts/benchmark-statistics.js @@ -0,0 +1,84 @@ +// Canonical timing: median of successful measured runs, including measured run 0. +// Hyperfine does not include its explicit warmup runs in `times`. +const median = (values) => { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2; +}; + +const finite = (value) => + typeof value === "number" && Number.isFinite(value) && value >= 0; + +const benchmarkStatistics = (result) => { + // Older result files can serialize summary fields as numeric strings. + const numeric = (value) => { + const parsed = + typeof value === "string" && value.trim() ? Number(value) : value; + return finite(parsed) ? parsed : undefined; + }; + const storedMedian = numeric(result.median); + const storedMean = numeric(result.mean); + const codes = Array.isArray(result.exit_codes) ? result.exit_codes : []; + const times = Array.isArray(result.times) ? result.times : undefined; + const samples = times?.filter( + (time, i) => finite(time) && (!codes.length || codes[i] === 0), + ); + const successfulRuns = samples?.length ?? result.successful_runs; + const attemptedRuns = result.attempted_runs ?? times?.length; + const droppedRuns = + result.dropped_runs ?? + (attemptedRuns !== undefined && successfulRuns !== undefined + ? attemptedRuns - successfulRuns + : codes.filter((code) => code !== 0).length); + const failed = + result.status === "failure" || + result.success === false || + result.result === "failure" || + Boolean(result.error) || + (samples !== undefined && samples.length === 0) || + (!samples && codes.some((code) => code !== 0)); + const partial = !failed && (result.status === "partial" || droppedRuns > 0); + const value = failed + ? undefined + : samples?.length + ? median(samples) + : (storedMedian ?? storedMean); + return { + value, + statistic: + samples?.length || storedMedian !== undefined ? "median" : "legacy-mean", + sampleCount: successfulRuns, + attemptedRuns, + droppedRuns, + partial, + failed: failed || !finite(value), + min: samples?.length ? Math.min(...samples) : numeric(result.min), + max: samples?.length ? Math.max(...samples) : numeric(result.max), + stddev: numeric(result.stddev), + }; +}; + +const formatBenchmarkSummary = (result) => { + const stats = benchmarkStatistics(result); + if (stats.failed) return `${result.command}: DNF`; + const sample = + stats.sampleCount === undefined + ? "sample count unknown" + : `${stats.sampleCount}/${stats.attemptedRuns ?? stats.sampleCount} successful runs`; + const range = + finite(stats.min) && finite(stats.max) + ? `; range ${stats.min}s–${stats.max}s` + : ""; + return `${result.command}: ${stats.value}s (${stats.statistic}; ${sample}${range}${stats.partial ? "; PARTIAL" : ""})`; +}; + +module.exports = { median, benchmarkStatistics, formatBenchmarkSummary }; + +if (require.main === module) { + const fs = require("node:fs"); + const data = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); + for (const result of data.results) + console.log(formatBenchmarkSummary(result)); +} diff --git a/scripts/benchmark-statistics.test.js b/scripts/benchmark-statistics.test.js new file mode 100644 index 0000000000..ba4ea62911 --- /dev/null +++ b/scripts/benchmark-statistics.test.js @@ -0,0 +1,118 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); +const test = require("node:test"); +const { + benchmarkStatistics, + formatBenchmarkSummary, +} = require("./benchmark-statistics.js"); +const { normalizeTiming } = require("./generate-chart.js"); + +const outlier = { + command: "vlt", + mean: 100.333, + median: 4, + times: [294, 3, 4], + exit_codes: [0, 0, 0], +}; + +test("median resists extreme outlier while retaining measured run 0 and full range", () => { + const stats = benchmarkStatistics(outlier); + assert.equal(stats.value, 4); + assert.equal(stats.sampleCount, 3); + assert.equal(stats.min, 3); + assert.equal(stats.max, 294); + assert.equal(stats.statistic, "median"); + assert.equal(stats.partial, false); + assert.equal(benchmarkStatistics({ times: [100, 2] }).value, 51); + assert.equal(benchmarkStatistics({ times: [9] }).value, 9); + assert.match( + formatBenchmarkSummary(outlier), + /4s \(median; 3\/3 successful runs; range 3s–294s\)/, + ); + const normalized = normalizeTiming(outlier, 100, true); + assert.equal(normalized.value, 40); + assert.equal(normalized.min, 30); + assert.equal(normalized.max, 2940); + assert.equal(normalized.sampleCount, 3); +}); + +test("partial survivors are labeled; all-failed samples stay DNF; legacy mean is explicit", () => { + const partial = benchmarkStatistics({ ...outlier, exit_codes: [124, 0, 0] }); + assert.equal(partial.value, 3.5); + assert.equal(partial.partial, true); + assert.equal(partial.sampleCount, 2); + assert.equal(partial.attemptedRuns, 3); + assert.equal( + benchmarkStatistics({ times: [0], exit_codes: [1] }).failed, + true, + ); + assert.equal(benchmarkStatistics({ mean: 100 }).statistic, "legacy-mean"); + assert.equal(benchmarkStatistics({ mean: 100, median: 4 }).value, 4); + assert.equal(benchmarkStatistics({ mean: "100", median: "4" }).value, 4); +}); + +test("processing publishes the same median and metadata to dated/latest PM and registry data", (t) => { + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "median-processing-")); + t.after(() => fs.rmSync(temp, { recursive: true, force: true })); + fs.symlinkSync(__dirname, path.join(temp, "scripts"), "dir"); + for (const variation of ["clean", "registry-clean"]) { + const dir = path.join(temp, "results", "next", variation); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, "benchmarks.json"), + JSON.stringify({ + results: [ + outlier, + { command: "npm", times: [300, 5, 6], exit_codes: [124, 0, 0] }, + ], + }), + ); + fs.writeFileSync( + path.join(dir, "package-count.json"), + JSON.stringify({ vlt: { count: 100 }, npm: { count: 100 } }), + ); + } + const run = () => { + const result = spawnSync( + "bash", + [path.join(__dirname, "process-results.sh")], + { + cwd: temp, + encoding: "utf8", + env: { ...process.env, BENCH_DATE: "2026-09-15", GITHUB_SHA: "test" }, + }, + ); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /vlt: 4s \(median; 3\/3 successful runs/); + return JSON.parse( + fs.readFileSync(path.join(temp, "results/latest/chart-data.json")), + ); + }; + const chart = run(); + assert.deepEqual(run(), chart, "reprocessing preserves partial counts"); + for (const [key, variation, value, min, max] of [ + ["chartData", "clean", 4, 3, 294], + ["perPackageCountChartData", "clean", 40, 30, 2940], + ["registryChartData", "registry-clean", 4, 3, 294], + ["registryPerPackageCountChartData", "registry-clean", 40, 30, 2940], + ]) { + const row = chart[key].data[variation][0]; + assert.equal(row.vlt, value); + assert.equal(row.vlt_statistic, "median"); + assert.equal(row.vlt_sample_count, 3); + assert.equal(row.vlt_min, min); + assert.equal(row.vlt_max, max); + assert.equal(row.npm_partial, true); + assert.equal(row.npm_successful_runs, 2); + assert.equal(row.npm_attempted_runs, 3); + } + assert.deepEqual( + chart, + JSON.parse( + fs.readFileSync(path.join(temp, "results/2026-09-15/chart-data.json")), + ), + ); +}); diff --git a/scripts/generate-chart.js b/scripts/generate-chart.js index b4eafade4d..e8b7e7a640 100644 --- a/scripts/generate-chart.js +++ b/scripts/generate-chart.js @@ -6,6 +6,7 @@ // web app folder in a `latest/` folder, e.g: app/latest/chart-data.json const fs = require("fs"); const path = require("path"); +const { benchmarkStatistics } = require("./benchmark-statistics.js"); const DATE = process.argv[2]; @@ -43,38 +44,35 @@ const REGISTRY_COLORS = { jfrog: "#40BE46", }; -const parseNumeric = (value) => { - if (typeof value === "number") { - return Number.isFinite(value) ? value : undefined; - } - if (typeof value === "string") { - const parsed = Number.parseFloat(value); - return Number.isFinite(parsed) ? parsed : undefined; - } - return undefined; -}; - const normalizeTiming = (result, count, perPackageCount) => { - if (!result || typeof result.mean !== "number") { - return undefined; - } - - if (!perPackageCount) { - return { value: result.mean, stddev: result.stddev }; - } - - if (typeof count !== "number" || count <= 0) { - return undefined; - } - - const scale = 1000 / count; + const stats = result?.statistic ? result : result && benchmarkStatistics(result); + if (!stats || stats.failed || typeof stats.value !== "number") return undefined; + if (perPackageCount && !(typeof count === "number" && count > 0)) return undefined; + const scale = perPackageCount ? 1000 / count : 1; return { - value: result.mean * scale, - stddev: - typeof result.stddev === "number" ? result.stddev * scale : undefined, + ...stats, + value: stats.value * scale, + stddev: typeof stats.stddev === "number" ? stats.stddev * scale : undefined, + min: typeof stats.min === "number" ? stats.min * scale : undefined, + max: typeof stats.max === "number" ? stats.max * scale : undefined, }; }; +const addStatistics = (row, pm, entry) => { + for (const [key, value] of Object.entries({ + statistic: entry.statistic, + sample_count: entry.sampleCount, + successful_runs: entry.sampleCount, + attempted_runs: entry.attemptedRuns, + dropped_runs: entry.droppedRuns, + min: entry.min, + max: entry.max, + partial: entry.partial || undefined, + })) { + if (value !== undefined) row[`${pm}_${key}`] = value; + } +}; + // Read and process results function readResults(file) { try { @@ -83,27 +81,7 @@ function readResults(file) { console.warn(`Warning: Invalid results format in ${file}`); return []; } - return data.results.map((r) => { - const exitCodes = Array.isArray(r.exit_codes) ? r.exit_codes : []; - return { - command: r.command, - mean: parseNumeric(r.mean), - stddev: parseNumeric(r.stddev), - exitCodes, - partial: - r.status === "partial" || - (r.dropped_runs > 0 && r.successful_runs > 0), - attempted_runs: r.attempted_runs, - successful_runs: r.successful_runs, - dropped_runs: r.dropped_runs, - failed: - exitCodes.some((code) => typeof code === "number" && code !== 0) || - r.success === false || - r.status === "failure" || - r.result === "failure" || - Boolean(r.error), - }; - }); + return data.results.map((r) => ({ command: r.command, ...benchmarkStatistics(r) })); } catch (error) { console.warn( `Warning: Could not read results from ${file}:`, @@ -113,17 +91,6 @@ function readResults(file) { } } -// Older published results have no completeness metadata. Leave it absent: -// discarded historical attempts cannot be recovered from survivor-only files. -function copyRunMetadata(target, command, result) { - if (result.partial) target[`${command}_partial`] = true; - for (const field of ["attempted_runs", "successful_runs", "dropped_runs"]) { - if (Number.isInteger(result[field]) && result[field] >= 0) { - target[`${command}_${field}`] = result[field]; - } - } -} - // Generate chart data for Recharts function generateChartData(option = {}) { const fixtures = ["next", "astro", "svelte", "vue", "large", "babylon", "run"]; @@ -192,7 +159,7 @@ function generateChartData(option = {}) { const pmResult = results.find((r) => r.command === pm); if (!pmResult) return; - const didFail = pmResult.failed || !Number.isFinite(pmResult.mean); + const didFail = pmResult.failed || !Number.isFinite(pmResult.value); const count = packageCounts[pm]; const timing = didFail ? undefined @@ -210,6 +177,7 @@ function generateChartData(option = {}) { pmEntries[pm] = { ...pmResult, + ...timing, didFail, value: timing?.value, stddev: timing?.stddev, @@ -235,8 +203,8 @@ function generateChartData(option = {}) { const fallback = entry.slowestValid ?? fallbackGlobal; Object.entries(entry.pmEntries).forEach(([pm, pmEntry]) => { - copyRunMetadata(fixtureResults, pm, pmEntry); fixtureResults[`${pm}_fill`] = COLORS[pm]; + addStatistics(fixtureResults, pm, pmEntry); if (pmEntry.count !== undefined) { fixtureResults[`${pm}_count`] = pmEntry.count; } @@ -354,7 +322,7 @@ function generateRegistryChartData(option = {}) { if (!registryResult) return; const didFail = - registryResult.failed || !Number.isFinite(registryResult.mean); + registryResult.failed || !Number.isFinite(registryResult.value); const count = packageCounts[registry]; const timing = didFail ? undefined @@ -372,6 +340,7 @@ function generateRegistryChartData(option = {}) { pmEntries[registry] = { ...registryResult, + ...timing, didFail, value: timing?.value, stddev: timing?.stddev, @@ -397,8 +366,8 @@ function generateRegistryChartData(option = {}) { const fallback = entry.slowestValid ?? fallbackGlobal; Object.entries(entry.pmEntries).forEach(([registry, regEntry]) => { - copyRunMetadata(fixtureResults, registry, regEntry); fixtureResults[`${registry}_fill`] = REGISTRY_COLORS[registry]; + addStatistics(fixtureResults, registry, regEntry); if (regEntry.count !== undefined) { fixtureResults[`${registry}_count`] = regEntry.count; } diff --git a/scripts/process-results.sh b/scripts/process-results.sh index 5949f5dfd3..f8312b349b 100644 --- a/scripts/process-results.sh +++ b/scripts/process-results.sh @@ -54,7 +54,7 @@ print_summary() { fi echo "=== RESULTS: $fixture ($variation) ===" - if ! jq -r '.results[] | "\(.command): \(.mean)s (stddev: \(.stddev)s)"' "$file"; then + if ! node ./scripts/benchmark-statistics.js "$file"; then echo "Warning: Could not parse results from $file" return 1 fi