From 673bb03afce6871a5d59bd138a467150d5f18a97 Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Tue, 15 Sep 2026 17:40:18 -0700 Subject: [PATCH] Use medians with sample quality metadata for benchmark charts --- .github/workflows/benchmark.yaml | 12 +-- .github/workflows/test.yml | 24 +++++ README.md | 38 +++++++- app/src/components/history-chart.tsx | 5 ++ app/src/components/variation/index.tsx | 8 ++ app/src/components/variation/table.tsx | 20 ++++- app/src/hooks/use-history-data.ts | 12 ++- app/src/lib/utils.ts | 46 +++++----- app/src/types/chart-data.ts | 8 +- app/tests/statistics.test.ts | 93 +++++++++++++++++++ scripts/benchmark-data.test.js | 18 ++-- scripts/benchmark-statistics.js | 84 ++++++++++++++++++ scripts/benchmark-statistics.test.js | 118 +++++++++++++++++++++++++ scripts/clean-benchmarks.js | 8 ++ scripts/generate-chart.js | 78 +++++++--------- scripts/process-results.sh | 2 +- 16 files changed, 487 insertions(+), 87 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 app/tests/statistics.test.ts create mode 100644 scripts/benchmark-statistics.js create mode 100644 scripts/benchmark-statistics.test.js diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 04b5afe803..4049bcb5a8 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -322,6 +322,7 @@ jobs: const path = require('path') // vlt must be more than 10% behind the fastest registry before a // difference counts as noticeably slower. + const { benchmarkStatistics } = require(`${process.env.GITHUB_WORKSPACE}/scripts/benchmark-statistics.js`) const THRESHOLD = 1.1 const slower = [] @@ -336,18 +337,19 @@ jobs: const label = `${match[1]} ${match[2]}` // Registries where every run failed are stamped with a non-zero - // exit code and a zero mean by clean-benchmarks and rendered as + // exit code and a zero median by clean-benchmarks and rendered as // failures on the site, so they are excluded from the speed // comparison. A failed vlt is already reported by the raw scan. const ok = (JSON.parse(fs.readFileSync(path.join(dir, entry), 'utf8')).results ?? []) - .filter((r) => r.mean > 0 && (r.exit_codes ?? []).every((c) => c === 0)) + .map((r) => ({ command: r.command, ...benchmarkStatistics(r) })) + .filter((r) => !r.failed && !r.partial && r.statistic === 'median' && r.value > 0) const vlt = ok.find((r) => r.command === 'vlt') if (!vlt) continue - const best = ok.reduce((a, b) => (b.mean < a.mean ? b : a)) - if (best.command === 'vlt' || vlt.mean <= best.mean * THRESHOLD) continue + const best = ok.reduce((a, b) => (b.value < a.value ? b : a)) + if (best.command === 'vlt' || vlt.value <= best.value * THRESHOLD) continue slower.push( - `- **${label}**: ${best.command} ${best.mean.toFixed(1)}s vs vlt ${vlt.mean.toFixed(1)}s (${(vlt.mean / best.mean).toFixed(2)}x slower)` + `- **${label}**: ${best.command} ${best.value.toFixed(1)}s median vs vlt ${vlt.value.toFixed(1)}s (${(vlt.value / best.value).toFixed(2)}x slower)` ) } diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..52c14048a0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,24 @@ +name: Data and App Tests +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: "24" + package-manager-cache: false + - run: npm install --ignore-scripts --no-package-lock + working-directory: app + - run: node --test scripts/*.test.js + - name: Test app data semantics + run: | + if [ -d app/tests ]; then node --test app/tests/*.test.*; fi + - run: npm run build + working-directory: app diff --git a/README.md b/README.md index b4dc5610e0..066773b3f8 100644 --- a/README.md +++ b/README.md @@ -199,13 +199,49 @@ The workflow: ## Results +### Timing statistic and sample quality + +The canonical timing is the **median of successful measured runs** for each +fixture, variation, and tool/registry. For `[3, 4, 294]` seconds, the chart shows +4 seconds instead of the 100.3-second mean. No successful outlier is discarded: +the table reports the successful/attempted sample count and full observed range +(3–294 seconds here). The range is descriptive, not a confidence interval; +three samples provide limited evidence, even when their median looks stable. + +The first **measured** run (run 0) is intentionally included. We do not assume +that it is a warmup or steady-state run: each variation controls its cache and +lockfile preparation. Explicit Hyperfine warmups are excluded from `times` and +therefore from the median and sample count. A one-run result is labeled 1/1; +its equal min/max does not imply certainty. + +Dated and latest raw results retain mean, median, standard deviation, and +individual successful timings. Chart data uses the median (seconds), or the +same median multiplied by 1000 and divided by package count (ms/package). +Failed attempts are excluded from timing statistics but retained in completeness +metadata. Partial survivor samples are labeled and excluded from averages, +history, leaderboard timing/win calculations, and registry speed alerts. +All-failed results remain DNF; the existing leaderboard DNF penalty uses the +slowest successful median for that fixture. + +Average views and leaderboard timing values are arithmetic averages of +per-benchmark medians, **not pooled medians**. History averages those medians +across available fixtures, and category history additionally averages across +variations. Coverage may vary by date; neither an averaged standard deviation +nor an invented confidence interval is shown for aggregates. + +Older chart files without statistic metadata remain readable in the latest +view and are labeled **legacy mean**. They are omitted from median history to +avoid a false trend at the methodology change. Reprocessing the original dated +results regenerates median points from `times` or stored `median`; a mean-only +raw result stays labeled `legacy-mean` and cannot be converted to a median. + ### Console Output Each benchmark run provides a summary in the console: ``` === Project Name (cache type) === -package-manager: X.XXs (stddev: X.XXs) +package-manager: 4s (median; 3/3 successful runs; range 3s–294s) ... ``` diff --git a/app/src/components/history-chart.tsx b/app/src/components/history-chart.tsx index 305f5c8f42..f08a06d428 100644 --- a/app/src/components/history-chart.tsx +++ b/app/src/components/history-chart.tsx @@ -236,6 +236,11 @@ export const HistoryChart = ({ +

+ Arithmetic averages of complete benchmark medians across available fixtures. + Legacy mean-only dates and incomplete samples are omitted. The average view + also averages across variations; coverage may differ by date. +

{
)} +

+ Times use the median of measured runs, including the first measured run; + explicit warmups are excluded. Tables show successful/attempted runs and + the observed range, which is not a confidence interval. Averages combine + per-benchmark medians and exclude partial samples. Older results are + labeled as legacy means. +

+ {/* History chart - performance over time */} {historyData && ( >; fixtureDnf: Partial>; + fixtureSamples: Partial>; } const columnHelper = createColumnHelper(); @@ -91,12 +92,25 @@ export const VariationTable = ({ filteredPackageManagers.map((packageManager) => { const fixtureValues: Partial> = {}; const fixtureDnf: Partial> = {}; + const fixtureSamples: Partial> = {}; variationData.forEach((fixtureResult) => { const fixture = fixtureResult.fixture; const dnfKey = `${packageManager}_dnf` as keyof FixtureResult; const value = fixtureResult[packageManager]; const isDnf = fixtureResult[dnfKey] === true; + const stat = fixtureResult[`${packageManager}_statistic`]; + const count = fixtureResult[`${packageManager}_sample_count`]; + const attempted = fixtureResult[`${packageManager}_attempted_runs`]; + const min = fixtureResult[`${packageManager}_min`]; + const max = fixtureResult[`${packageManager}_max`]; + const unit = isPerPackage ? "ms/pkg" : "s"; + const range = typeof min === "number" && typeof max === "number" + ? ` · range ${min.toFixed(2)}–${max.toFixed(2)} ${unit}` : ""; + const partial = fixtureResult[`${packageManager}_partial`] === true ? " · partial" : ""; + fixtureSamples[fixture] = stat === "average-of-medians" || stat === "legacy-average" + ? `${stat === "average-of-medians" ? "Average of medians" : "Legacy average"} · ${fixtureResult[`${packageManager}_variation_count`]} variations` + : `${stat === "median" ? "Median" : "Legacy mean"} · ${count === undefined ? "sample count unknown" : `${count}/${attempted ?? count} runs`}${range}${partial}`; if (isDnf) { fixtureDnf[fixture] = true; @@ -110,9 +124,10 @@ export const VariationTable = ({ packageManager, fixtureValues, fixtureDnf, + fixtureSamples, }; }), - [filteredPackageManagers, variationData], + [filteredPackageManagers, variationData, isPerPackage], ); const columns = useMemo( @@ -185,6 +200,9 @@ export const VariationTable = ({ {value.toFixed(decimals)} {unit} +
+ {info.row.original.fixtureSamples[fixture]} +
); } diff --git a/app/src/hooks/use-history-data.ts b/app/src/hooks/use-history-data.ts index 7a19274f68..23191bdf8a 100644 --- a/app/src/hooks/use-history-data.ts +++ b/app/src/hooks/use-history-data.ts @@ -67,7 +67,7 @@ async function parallelLimit( type FixtureDataSet = Record< string, - Array & { fixture: string }> + Array & { fixture: string }> >; interface ChartDataResponse { @@ -105,7 +105,7 @@ interface ChartDataResponse { * * Registry and task-runner variations always use total-time data. */ -function extractDayData( +export function extractDayData( response: ChartDataResponse, ): Record> { const result: Record> = {}; @@ -132,7 +132,7 @@ function extractDayData( function extractFromDataSet( data: Record< string, - Array & { fixture: string }> + Array & { fixture: string }> >, result: Record>, ): void { @@ -140,9 +140,15 @@ function extractFromDataSet( if (!Array.isArray(fixtures) || fixtures.length === 0) continue; const pmTotals: Record = {}; + const partialPMs = new Set(PACKAGE_MANAGERS.filter((pm) => + fixtures.some((fixture) => fixture[`${pm}_partial`] === true))); for (const fixture of fixtures) { for (const pm of PACKAGE_MANAGERS) { + // Never splice legacy mean points into a median series. Reprocessing + // dated raw files adds median metadata and restores those dates. + if (partialPMs.has(pm) || fixture[`${pm}_statistic`] !== "median" || + fixture[`${pm}_dnf`] === true || fixture[`${pm}_partial`] === true) continue; const val = fixture[pm]; if (typeof val === "number" && Number.isFinite(val)) { if (!pmTotals[pm]) pmTotals[pm] = { sum: 0, count: 0 }; diff --git a/app/src/lib/utils.ts b/app/src/lib/utils.ts index 1e615c0b82..b8189a19ab 100644 --- a/app/src/lib/utils.ts +++ b/app/src/lib/utils.ts @@ -42,10 +42,6 @@ export const calculateAverageVariationData = ( }, ): FixtureResult[] => { type FillKey = Extract; - type StddevKey = Extract< - keyof PackageManagerData, - `${PackageManager}_stddev` - >; type CountKey = Extract; type DnfKey = Extract; @@ -97,9 +93,10 @@ export const calculateAverageVariationData = ( const averagedResult: FixtureResult = { fixture: fixture as Fixture }; packageManagers.forEach((pm: PackageManager) => { + if (results.some((r) => r[`${pm}_partial`] === true)) return; const dnfKey: DnfKey = `${pm}_dnf`; const values = results - .filter((r) => r[dnfKey] !== true) + .filter((r) => r[dnfKey] !== true && r[`${pm}_partial`] !== true) .map((r) => r[pm]) .filter((val): val is number => typeof val === "number" && val > 0); @@ -115,25 +112,20 @@ export const calculateAverageVariationData = ( averagedResult[fillKey] = firstFill; } - // Calculate average standard deviation if available - const stddevKey: StddevKey = `${pm}_stddev`; - const stddevValues = results - .filter((r) => r[dnfKey] !== true) - .map((r) => r[stddevKey]) - .filter((val): val is number => typeof val === "number" && val > 0); - - if (stddevValues.length > 0) { - const avgStddev = - stddevValues.reduce((sum, val) => sum + val, 0) / - stddevValues.length; - averagedResult[stddevKey] = avgStddev; - } + // This is an arithmetic average of benchmark medians, not a pooled + // sample. Averaging standard deviations would misrepresent its spread. + const contributing = results.filter((r) => + r[dnfKey] !== true && r[`${pm}_partial`] !== true && + typeof r[pm] === "number" && r[pm]! > 0); + averagedResult[`${pm}_statistic`] = contributing.every((r) => r[`${pm}_statistic`] === "median") + ? "average-of-medians" : "legacy-average"; + averagedResult[`${pm}_variation_count`] = values.length; // For per-package data, also average the count if available if (isPerPackage) { const countKey: CountKey = `${pm}_count`; const countValues = results - .filter((r) => r[dnfKey] !== true) + .filter((r) => r[dnfKey] !== true && r[`${pm}_partial`] !== true) .map((r) => r[countKey]) .filter((val): val is number => typeof val === "number" && val > 0); @@ -300,8 +292,21 @@ export const calculateLeaderboard = ( ?.variations.filter((v) => v !== "average") || []; } + // A partial result invalidates this PM's selected comparison set; dropping + // only its failed/slow cases would reward survivor bias. + const partialPMs = new Set(); + for (const variation of variationsToUse) { + const source = usePerPackageData ? chartData.perPackageCountChartData.data : chartData.chartData.data; + for (const row of source[variation] ?? []) { + if (enabledFixtures && !enabledFixtures.has(row.fixture)) continue; + for (const pm of availablePackageManagers as PackageManager[]) { + if (row[`${pm}_partial`] === true) partialPMs.add(pm); + } + } + } + // Calculate performance — DNF runs are imputed as the slowest successful - // time for that fixture, matching the "Performance Over Time" chart data + // median for that fixture; history omits failures. variationsToUse.forEach((variation) => { const dataSource = usePerPackageData ? chartData.perPackageCountChartData.data @@ -320,6 +325,7 @@ export const calculateLeaderboard = ( // First pass: collect successful times and DNFs (availablePackageManagers as PackageManager[]).forEach((pm) => { + if (partialPMs.has(pm)) return; const time = fixtureResult[pm]; const dnfKey = `${pm}_dnf` as keyof FixtureResult; if (fixtureResult[dnfKey] === true) { diff --git a/app/src/types/chart-data.ts b/app/src/types/chart-data.ts index b2474f337b..810e6bfa6c 100644 --- a/app/src/types/chart-data.ts +++ b/app/src/types/chart-data.ts @@ -71,7 +71,13 @@ export interface BaseFixtureResult { fixture: Fixture; } -export interface PackageManagerData { +type SampleMetadata = Partial> & Partial> + & Partial>; + +export interface PackageManagerData extends SampleMetadata { npm?: number; yarn?: number; pnpm?: number; diff --git a/app/tests/statistics.test.ts b/app/tests/statistics.test.ts new file mode 100644 index 0000000000..94da043ca1 --- /dev/null +++ b/app/tests/statistics.test.ts @@ -0,0 +1,93 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { extractDayData } from "../src/hooks/use-history-data.ts"; +import { + calculateAverageVariationData, + calculateLeaderboard, +} from "../src/lib/utils.ts"; +import type { + BenchmarkChartData, + FixtureResult, +} from "../src/types/chart-data.ts"; + +const row: FixtureResult = { + fixture: "next", + vlt: 4, + npm: 5, + vlt_statistic: "median", + npm_statistic: "median", + vlt_stddev: 167, + npm_stddev: 1, +}; +const chart = (data: Record) => + ({ + chartData: { + data, + variations: Object.keys(data), + packageManagers: ["vlt", "npm"], + colors: {}, + }, + perPackageCountChartData: { + data, + variations: Object.keys(data), + packageManagers: ["vlt", "npm"], + colors: {}, + }, + versions: {}, + }) as BenchmarkChartData; + +test("aggregate uses medians, labels average-of-medians, and does not invent dispersion", () => { + const result = calculateAverageVariationData( + chart({ clean: [row], lockfile: [{ ...row, vlt: 6 }] }), + ); + assert.equal(result[0].vlt, 5); + assert.equal(result[0].vlt_statistic, "average-of-medians"); + assert.equal(result[0].vlt_variation_count, 2); + assert.equal(result[0].vlt_stddev, undefined); + const ranking = calculateLeaderboard( + chart({ clean: [row] }), + "clean", + "package-managers", + ); + assert.equal(ranking[0].packageManager, "vlt"); + assert.equal(ranking[0].averageTime, 4); + assert.equal(ranking[0].wins, 1); +}); + +test("a partial sample cannot improve a PM's aggregate or leaderboard by dropping its slow case", () => { + const data = chart({ + clean: [row], + lockfile: [{ ...row, vlt: 1, vlt_partial: true }], + }); + assert.equal(calculateAverageVariationData(data)[0].vlt, undefined); + assert.equal( + calculateLeaderboard(data, "average", "package-managers").some( + (r) => r.packageManager === "vlt", + ), + false, + ); +}); + +test("history averages complete medians and omits legacy means and partial comparison sets", () => { + const response = (rows: FixtureResult[]) => ({ + date: "2026-09-15", + chartData: { + variations: ["clean"], + data: { clean: rows }, + packageManagers: ["vlt", "npm"], + }, + }); + const day = extractDayData( + response([row, { ...row, fixture: "astro", vlt: 6 }]), + ); + assert.equal(day.clean.vlt, 5); + assert.equal(day.clean.npm, 5); + const legacy = { ...row }; + delete legacy.vlt_statistic; + assert.equal(extractDayData(response([legacy])).clean.vlt, undefined); + const partial = extractDayData( + response([row, { ...row, vlt_partial: true }]), + ); + assert.equal(partial.clean.vlt, undefined); + assert.equal(partial.clean.npm, 5); +}); diff --git a/scripts/benchmark-data.test.js b/scripts/benchmark-data.test.js index f8fc8e3ff4..71ff8ac769 100644 --- a/scripts/benchmark-data.test.js +++ b/scripts/benchmark-data.test.js @@ -8,10 +8,10 @@ const test = require("node:test"); const { normalizeTiming } = require("./generate-chart.js"); test("normalizes per-package timing and standard deviation", () => { - assert.deepEqual(normalizeTiming({ mean: 1.2, stddev: 0.12 }, 60, true), { - value: 20, - stddev: 2, - }); + const timing = normalizeTiming({ median: 1.2, stddev: 0.12 }, 60, true); + assert.equal(timing.value, 20); + assert.equal(timing.stddev, 2); + assert.equal(timing.statistic, "median"); }); test("does not mix total seconds into per-package data without a count", () => { @@ -26,13 +26,9 @@ test("does not mix total seconds into per-package data without a count", () => { }); test("preserves total timing units", () => { - assert.deepEqual( - normalizeTiming({ mean: 1.2, stddev: 0.12 }, undefined, false), - { - value: 1.2, - stddev: 0.12, - }, - ); + const timing = normalizeTiming({ median: 1.2, stddev: 0.12 }, undefined, false); + assert.equal(timing.value, 1.2); + assert.equal(timing.stddev, 0.12); }); for (const fixture of [ diff --git a/scripts/benchmark-statistics.js b/scripts/benchmark-statistics.js new file mode 100644 index 0000000000..3c154c636d --- /dev/null +++ b/scripts/benchmark-statistics.js @@ -0,0 +1,84 @@ +// Canonical timing: median of successful measured runs, including measured run 0. +// Hyperfine does not include its explicit warmup runs in `times`. +const median = (values) => { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2; +}; + +const finite = (value) => + typeof value === "number" && Number.isFinite(value) && value >= 0; + +const benchmarkStatistics = (result) => { + // Older result files can serialize summary fields as numeric strings. + const numeric = (value) => { + const parsed = + typeof value === "string" && value.trim() ? Number(value) : value; + return finite(parsed) ? parsed : undefined; + }; + const storedMedian = numeric(result.median); + const storedMean = numeric(result.mean); + const codes = Array.isArray(result.exit_codes) ? result.exit_codes : []; + const times = Array.isArray(result.times) ? result.times : undefined; + const samples = times?.filter( + (time, i) => finite(time) && (!codes.length || codes[i] === 0), + ); + const successfulRuns = samples?.length ?? result.successful_runs; + const attemptedRuns = result.attempted_runs ?? times?.length; + const droppedRuns = + result.dropped_runs ?? + (attemptedRuns !== undefined && successfulRuns !== undefined + ? attemptedRuns - successfulRuns + : codes.filter((code) => code !== 0).length); + const failed = + result.status === "failure" || + result.success === false || + result.result === "failure" || + Boolean(result.error) || + (samples !== undefined && samples.length === 0) || + (!samples && codes.some((code) => code !== 0)); + const partial = !failed && (result.status === "partial" || droppedRuns > 0); + const value = failed + ? undefined + : samples?.length + ? median(samples) + : (storedMedian ?? storedMean); + return { + value, + statistic: + samples?.length || storedMedian !== undefined ? "median" : "legacy-mean", + sampleCount: successfulRuns, + attemptedRuns, + droppedRuns, + partial, + failed: failed || !finite(value), + min: samples?.length ? Math.min(...samples) : numeric(result.min), + max: samples?.length ? Math.max(...samples) : numeric(result.max), + stddev: numeric(result.stddev), + }; +}; + +const formatBenchmarkSummary = (result) => { + const stats = benchmarkStatistics(result); + if (stats.failed) return `${result.command}: DNF`; + const sample = + stats.sampleCount === undefined + ? "sample count unknown" + : `${stats.sampleCount}/${stats.attemptedRuns ?? stats.sampleCount} successful runs`; + const range = + finite(stats.min) && finite(stats.max) + ? `; range ${stats.min}s–${stats.max}s` + : ""; + return `${result.command}: ${stats.value}s (${stats.statistic}; ${sample}${range}${stats.partial ? "; PARTIAL" : ""})`; +}; + +module.exports = { median, benchmarkStatistics, formatBenchmarkSummary }; + +if (require.main === module) { + const fs = require("node:fs"); + const data = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); + for (const result of data.results) + console.log(formatBenchmarkSummary(result)); +} diff --git a/scripts/benchmark-statistics.test.js b/scripts/benchmark-statistics.test.js new file mode 100644 index 0000000000..ba4ea62911 --- /dev/null +++ b/scripts/benchmark-statistics.test.js @@ -0,0 +1,118 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); +const test = require("node:test"); +const { + benchmarkStatistics, + formatBenchmarkSummary, +} = require("./benchmark-statistics.js"); +const { normalizeTiming } = require("./generate-chart.js"); + +const outlier = { + command: "vlt", + mean: 100.333, + median: 4, + times: [294, 3, 4], + exit_codes: [0, 0, 0], +}; + +test("median resists extreme outlier while retaining measured run 0 and full range", () => { + const stats = benchmarkStatistics(outlier); + assert.equal(stats.value, 4); + assert.equal(stats.sampleCount, 3); + assert.equal(stats.min, 3); + assert.equal(stats.max, 294); + assert.equal(stats.statistic, "median"); + assert.equal(stats.partial, false); + assert.equal(benchmarkStatistics({ times: [100, 2] }).value, 51); + assert.equal(benchmarkStatistics({ times: [9] }).value, 9); + assert.match( + formatBenchmarkSummary(outlier), + /4s \(median; 3\/3 successful runs; range 3s–294s\)/, + ); + const normalized = normalizeTiming(outlier, 100, true); + assert.equal(normalized.value, 40); + assert.equal(normalized.min, 30); + assert.equal(normalized.max, 2940); + assert.equal(normalized.sampleCount, 3); +}); + +test("partial survivors are labeled; all-failed samples stay DNF; legacy mean is explicit", () => { + const partial = benchmarkStatistics({ ...outlier, exit_codes: [124, 0, 0] }); + assert.equal(partial.value, 3.5); + assert.equal(partial.partial, true); + assert.equal(partial.sampleCount, 2); + assert.equal(partial.attemptedRuns, 3); + assert.equal( + benchmarkStatistics({ times: [0], exit_codes: [1] }).failed, + true, + ); + assert.equal(benchmarkStatistics({ mean: 100 }).statistic, "legacy-mean"); + assert.equal(benchmarkStatistics({ mean: 100, median: 4 }).value, 4); + assert.equal(benchmarkStatistics({ mean: "100", median: "4" }).value, 4); +}); + +test("processing publishes the same median and metadata to dated/latest PM and registry data", (t) => { + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "median-processing-")); + t.after(() => fs.rmSync(temp, { recursive: true, force: true })); + fs.symlinkSync(__dirname, path.join(temp, "scripts"), "dir"); + for (const variation of ["clean", "registry-clean"]) { + const dir = path.join(temp, "results", "next", variation); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, "benchmarks.json"), + JSON.stringify({ + results: [ + outlier, + { command: "npm", times: [300, 5, 6], exit_codes: [124, 0, 0] }, + ], + }), + ); + fs.writeFileSync( + path.join(dir, "package-count.json"), + JSON.stringify({ vlt: { count: 100 }, npm: { count: 100 } }), + ); + } + const run = () => { + const result = spawnSync( + "bash", + [path.join(__dirname, "process-results.sh")], + { + cwd: temp, + encoding: "utf8", + env: { ...process.env, BENCH_DATE: "2026-09-15", GITHUB_SHA: "test" }, + }, + ); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /vlt: 4s \(median; 3\/3 successful runs/); + return JSON.parse( + fs.readFileSync(path.join(temp, "results/latest/chart-data.json")), + ); + }; + const chart = run(); + assert.deepEqual(run(), chart, "reprocessing preserves partial counts"); + for (const [key, variation, value, min, max] of [ + ["chartData", "clean", 4, 3, 294], + ["perPackageCountChartData", "clean", 40, 30, 2940], + ["registryChartData", "registry-clean", 4, 3, 294], + ["registryPerPackageCountChartData", "registry-clean", 40, 30, 2940], + ]) { + const row = chart[key].data[variation][0]; + assert.equal(row.vlt, value); + assert.equal(row.vlt_statistic, "median"); + assert.equal(row.vlt_sample_count, 3); + assert.equal(row.vlt_min, min); + assert.equal(row.vlt_max, max); + assert.equal(row.npm_partial, true); + assert.equal(row.npm_successful_runs, 2); + assert.equal(row.npm_attempted_runs, 3); + } + assert.deepEqual( + chart, + JSON.parse( + fs.readFileSync(path.join(temp, "results/2026-09-15/chart-data.json")), + ), + ); +}); diff --git a/scripts/clean-benchmarks.js b/scripts/clean-benchmarks.js index 3cc29547f3..a2f29baa13 100644 --- a/scripts/clean-benchmarks.js +++ b/scripts/clean-benchmarks.js @@ -1,5 +1,6 @@ const fs = require("fs"); const path = require("path"); +const { benchmarkStatistics } = require("./benchmark-statistics.js"); const inputPaths = process.argv.slice(2); @@ -93,6 +94,13 @@ const cleanBenchmarkFile = (filePath) => { return; } + // Capture completeness before failures are removed; preserve it on reprocessing. + const stats = benchmarkStatistics(result); + result.attempted_runs ??= stats.attemptedRuns; + result.successful_runs ??= stats.sampleCount; + result.dropped_runs ??= stats.droppedRuns; + result.status = stats.failed ? "failure" : stats.partial ? "partial" : "success"; + const cleanTimes = times.filter((time, idx) => exitCodes[idx] === 0); const cleanExitCodes = exitCodes.filter((code) => code === 0); diff --git a/scripts/generate-chart.js b/scripts/generate-chart.js index faa4e9335c..fc7307e3e1 100644 --- a/scripts/generate-chart.js +++ b/scripts/generate-chart.js @@ -6,6 +6,7 @@ // web app folder in a `latest/` folder, e.g: app/latest/chart-data.json const fs = require("fs"); const path = require("path"); +const { benchmarkStatistics } = require("./benchmark-statistics.js"); const DATE = process.argv[2]; @@ -43,38 +44,35 @@ const REGISTRY_COLORS = { jfrog: "#40BE46", }; -const parseNumeric = (value) => { - if (typeof value === "number") { - return Number.isFinite(value) ? value : undefined; - } - if (typeof value === "string") { - const parsed = Number.parseFloat(value); - return Number.isFinite(parsed) ? parsed : undefined; - } - return undefined; -}; - const normalizeTiming = (result, count, perPackageCount) => { - if (!result || typeof result.mean !== "number") { - return undefined; - } - - if (!perPackageCount) { - return { value: result.mean, stddev: result.stddev }; - } - - if (typeof count !== "number" || count <= 0) { - return undefined; - } - - const scale = 1000 / count; + const stats = result?.statistic ? result : result && benchmarkStatistics(result); + if (!stats || stats.failed || typeof stats.value !== "number") return undefined; + if (perPackageCount && !(typeof count === "number" && count > 0)) return undefined; + const scale = perPackageCount ? 1000 / count : 1; return { - value: result.mean * scale, - stddev: - typeof result.stddev === "number" ? result.stddev * scale : undefined, + ...stats, + value: stats.value * scale, + stddev: typeof stats.stddev === "number" ? stats.stddev * scale : undefined, + min: typeof stats.min === "number" ? stats.min * scale : undefined, + max: typeof stats.max === "number" ? stats.max * scale : undefined, }; }; +const addStatistics = (row, pm, entry) => { + for (const [key, value] of Object.entries({ + statistic: entry.statistic, + sample_count: entry.sampleCount, + successful_runs: entry.sampleCount, + attempted_runs: entry.attemptedRuns, + dropped_runs: entry.droppedRuns, + min: entry.min, + max: entry.max, + partial: entry.partial || undefined, + })) { + if (value !== undefined) row[`${pm}_${key}`] = value; + } +}; + // Read and process results function readResults(file) { try { @@ -83,21 +81,7 @@ function readResults(file) { console.warn(`Warning: Invalid results format in ${file}`); return []; } - return data.results.map((r) => { - const exitCodes = Array.isArray(r.exit_codes) ? r.exit_codes : []; - return { - command: r.command, - mean: parseNumeric(r.mean), - stddev: parseNumeric(r.stddev), - exitCodes, - failed: - exitCodes.some((code) => typeof code === "number" && code !== 0) || - r.success === false || - r.status === "failure" || - r.result === "failure" || - Boolean(r.error), - }; - }); + return data.results.map((r) => ({ command: r.command, ...benchmarkStatistics(r) })); } catch (error) { console.warn( `Warning: Could not read results from ${file}:`, @@ -175,7 +159,7 @@ function generateChartData(option = {}) { const pmResult = results.find((r) => r.command === pm); if (!pmResult) return; - const didFail = pmResult.failed || !Number.isFinite(pmResult.mean); + const didFail = pmResult.failed || !Number.isFinite(pmResult.value); const count = packageCounts[pm]; const timing = didFail ? undefined @@ -192,6 +176,8 @@ function generateChartData(option = {}) { } pmEntries[pm] = { + ...pmResult, + ...timing, didFail, value: timing?.value, stddev: timing?.stddev, @@ -218,6 +204,7 @@ function generateChartData(option = {}) { Object.entries(entry.pmEntries).forEach(([pm, pmEntry]) => { fixtureResults[`${pm}_fill`] = COLORS[pm]; + addStatistics(fixtureResults, pm, pmEntry); if (pmEntry.count !== undefined) { fixtureResults[`${pm}_count`] = pmEntry.count; } @@ -334,7 +321,7 @@ function generateRegistryChartData(option = {}) { if (!registryResult) return; const didFail = - registryResult.failed || !Number.isFinite(registryResult.mean); + registryResult.failed || !Number.isFinite(registryResult.value); const count = packageCounts[registry]; const timing = didFail ? undefined @@ -351,6 +338,8 @@ function generateRegistryChartData(option = {}) { } pmEntries[registry] = { + ...registryResult, + ...timing, didFail, value: timing?.value, stddev: timing?.stddev, @@ -377,6 +366,7 @@ function generateRegistryChartData(option = {}) { Object.entries(entry.pmEntries).forEach(([registry, regEntry]) => { fixtureResults[`${registry}_fill`] = REGISTRY_COLORS[registry]; + addStatistics(fixtureResults, registry, regEntry); if (regEntry.count !== undefined) { fixtureResults[`${registry}_count`] = regEntry.count; } diff --git a/scripts/process-results.sh b/scripts/process-results.sh index 5949f5dfd3..f8312b349b 100644 --- a/scripts/process-results.sh +++ b/scripts/process-results.sh @@ -54,7 +54,7 @@ print_summary() { fi echo "=== RESULTS: $fixture ($variation) ===" - if ! jq -r '.results[] | "\(.command): \(.mean)s (stddev: \(.stddev)s)"' "$file"; then + if ! node ./scripts/benchmark-statistics.js "$file"; then echo "Warning: Could not parse results from $file" return 1 fi