Use median benchmark timings with sample quality context

.github/workflows/benchmark.yaml

@@ -322,6 +322,7 @@ jobs:
             const path = require('path')
             // vlt must be more than 10% behind the fastest registry before a
             // difference counts as noticeably slower.
+            const { benchmarkStatistics } = require(`${process.env.GITHUB_WORKSPACE}/scripts/benchmark-statistics.js`)
             const THRESHOLD = 1.1
             const slower = []
 
@@ -339,14 +340,15 @@ jobs:
               // not establish a fair speed comparison. Failures are already
               // reported by the raw scan before processing.
               const ok = (JSON.parse(fs.readFileSync(path.join(dir, entry), 'utf8')).results ?? [])
-                .filter((r) => r.mean > 0 && r.status !== 'partial' && !(r.dropped_runs > 0) && (r.exit_codes ?? []).every((c) => c === 0))
+                .map((r) => ({ command: r.command, ...benchmarkStatistics(r) }))
+                .filter((r) => !r.failed && !r.partial && r.statistic === 'median' && r.value > 0)
               const vlt = ok.find((r) => r.command === 'vlt')
               if (!vlt) continue
 
-              const best = ok.reduce((a, b) => (b.mean < a.mean ? b : a))
-              if (best.command === 'vlt' || vlt.mean <= best.mean * THRESHOLD) continue
+              const best = ok.reduce((a, b) => (b.value < a.value ? b : a))
+              if (best.command === 'vlt' || vlt.value <= best.value * THRESHOLD) continue
               slower.push(
-                `- **${label}**: ${best.command} ${best.mean.toFixed(1)}s vs vlt ${vlt.mean.toFixed(1)}s (${(vlt.mean / best.mean).toFixed(2)}x slower)`
+                `- **${label}**: ${best.command} ${best.value.toFixed(1)}s median vs vlt ${vlt.value.toFixed(1)}s (${(vlt.value / best.value).toFixed(2)}x slower)`
               )
             }
 

README.md

@@ -216,7 +216,8 @@ aggregate CPU values, which cannot be recalculated for the surviving sample.
 Chart data adds optional `<command>_partial`, `<command>_attempted_runs`,
 `<command>_successful_runs`, and `<command>_dropped_runs` fields in both total and
 per-package datasets. Tables and chart notices label partial results with their
-success counts. Synthetic averages retain that warning. Commands with any
+success counts. Synthetic averages retain that warning and omit the affected
+command's value. Commands with any
 partial result in the selected comparisons are excluded from rankings; history
 omits that command's affected daily variation and category average.
 
@@ -227,14 +228,49 @@ recover counts when those artifacts are still available. CI checks that inspect
 raw `exit_codes` must run before `./bench process`; the registry failure scan
 already does so. Persisted original codes also remain available in dated result
 files on gh-pages.
+### Timing statistic and sample quality
+
+The canonical timing is the **median of successful measured runs** for each
+fixture, variation, and tool/registry. For `[3, 4, 294]` seconds, the chart shows
+4 seconds instead of the 100.3-second mean. No successful outlier is discarded:
+the table reports the successful/attempted sample count and full observed range
+(3–294 seconds here). The range is descriptive, not a confidence interval;
+three samples provide limited evidence, even when their median looks stable.
+
+The first **measured** run (run 0) is intentionally included. We do not assume
+that it is a warmup or steady-state run: each variation controls its cache and
+lockfile preparation. Explicit Hyperfine warmups are excluded from `times` and
+therefore from the median and sample count. A one-run result is labeled 1/1;
+its equal min/max does not imply certainty.
+
+Dated and latest raw results retain mean, median, standard deviation, and
+individual successful timings. Chart data uses the median (seconds), or the
+same median multiplied by 1000 and divided by package count (ms/package).
+Failed attempts are excluded from timing statistics but retained in completeness
+metadata. Partial survivor samples are labeled and excluded from averages,
+history, leaderboard timing/win calculations, and registry speed alerts.
+All-failed results remain DNF; the existing leaderboard DNF penalty uses the
+slowest successful median for that fixture.
+
+Average views and leaderboard timing values are arithmetic averages of
+per-benchmark medians, **not pooled medians**. History averages those medians
+across available fixtures, and category history additionally averages across
+variations. Coverage may vary by date; neither an averaged standard deviation
+nor an invented confidence interval is shown for aggregates.
+
+Older chart files without statistic metadata remain readable in the latest
+view and are labeled **legacy mean**. They are omitted from median history to
+avoid a false trend at the methodology change. Reprocessing the original dated
+results regenerates median points from `times` or stored `median`; a mean-only
+raw result stays labeled `legacy-mean` and cannot be converted to a median.
 
 ### Console Output
 
 Each benchmark run provides a summary in the console:
 
 ```
 === Project Name (cache type) ===
-package-manager: X.XXs (stddev: X.XXs)
+package-manager: 4s (median; 3/3 successful runs; range 3s–294s)
 ...
 ```
 

app/src/components/history-chart.tsx

@@ -236,6 +236,11 @@ export const HistoryChart = ({
         </div>
       </div>
 
+      <p className="text-xs text-muted-foreground">
+        Arithmetic averages of complete benchmark medians across available fixtures.
+        Legacy mean-only dates and incomplete samples are omitted. The average view
+        also averages across variations; coverage may differ by date.
+      </p>
       <div className="bg-card rounded-xl p-3 md:p-6 border-border border-[1px] overflow-hidden">
         <p className="mb-4 text-xs text-muted-foreground">
           Days with partial results are omitted for that command and variation.

app/src/components/variation/chart.tsx

@@ -408,7 +408,7 @@ export const VariationChart = ({
         const isActive = variationActivePackageManagers.has(pm);
         const hasNumber = typeof value === "number";
         const shouldFallback =
-          isActive && !hasNumber && typeof slowest === "number";
+          isActive && !hasNumber && item[`${pm}_partial`] !== true && typeof slowest === "number";
         const isDnf = item[dnfKey] === true || shouldFallback;
 
         if (hasNumber) {
@@ -499,10 +499,10 @@ export const VariationChart = ({
                 const value = fixtureResult[pm];
                 const hasNumber = typeof value === "number";
                 const shouldFallback =
-                  !hasNumber && typeof slowest === "number";
+                  !hasNumber && fixtureResult[`${pm}_partial`] !== true && typeof slowest === "number";
                 const isDnf =
                   fixtureResult[dnfKey] === true || shouldFallback;
-                const resolvedValue = hasNumber ? value : slowest;
+                const resolvedValue = hasNumber ? value : shouldFallback ? slowest : undefined;
 
                 if (typeof resolvedValue !== "number") return null;
 
@@ -762,10 +762,10 @@ export const VariationChart = ({
                   const value = fixtureResult[pm];
                   const hasNumber = typeof value === "number";
                   const shouldFallback =
-                    !hasNumber && typeof slowest === "number";
+                    !hasNumber && fixtureResult[`${pm}_partial`] !== true && typeof slowest === "number";
                   const isDnf =
                     fixtureResult[dnfKey] === true || shouldFallback;
-                  const resolvedValue = hasNumber ? value : slowest;
+                  const resolvedValue = hasNumber ? value : shouldFallback ? slowest : undefined;
 
                   if (typeof resolvedValue !== "number") return null;
 
@@ -1016,9 +1016,9 @@ export const VariationChart = ({
               const dnfKey = `${pm}_dnf` as keyof FixtureResult;
               const value = fixtureResult[pm];
               const hasNumber = typeof value === "number";
-              const shouldFallback = !hasNumber && typeof slowest === "number";
+              const shouldFallback = !hasNumber && fixtureResult[`${pm}_partial`] !== true && typeof slowest === "number";
               const isDnf = fixtureResult[dnfKey] === true || shouldFallback;
-              const resolvedValue = hasNumber ? value : slowest;
+              const resolvedValue = hasNumber ? value : shouldFallback ? slowest : undefined;
 
               if (typeof resolvedValue !== "number") {
                 return null;

app/src/components/variation/index.tsx

@@ -219,6 +219,14 @@ export const VariationPage = () => {
         </div>
       )}
 
+      <p className="text-sm text-muted-foreground">
+        Times use the median of measured runs, including the first measured run;
+        explicit warmups are excluded. Tables show successful/attempted runs and
+        the observed range, which is not a confidence interval. Averages combine
+        per-benchmark medians and exclude partial samples. Older results are
+        labeled as legacy means.
+      </p>
+
       {/* History chart - performance over time */}
       {historyData && (
         <HistoryChart

app/src/components/variation/table.tsx

@@ -55,6 +55,7 @@ interface TransposedVariationRow {
   fixtureValues: Partial<Record<Fixture, number>>;
   fixtureDnf: Partial<Record<Fixture, boolean>>;
   fixturePartial: Partial<Record<Fixture, string>>;
+  fixtureSamples: Partial<Record<Fixture, string>>;
 }
 
 const columnHelper = createColumnHelper<TransposedVariationRow>();
@@ -94,6 +95,7 @@ export const VariationTable = ({
         const fixtureValues: Partial<Record<Fixture, number>> = {};
         const fixtureDnf: Partial<Record<Fixture, boolean>> = {};
         const fixturePartial: Partial<Record<Fixture, string>> = {};
+        const fixtureSamples: Partial<Record<Fixture, string>> = {};
 
         variationData.forEach((fixtureResult) => {
           const fixture = fixtureResult.fixture;
@@ -104,6 +106,18 @@ export const VariationTable = ({
             { ...fixtureResult },
             packageManager,
           );
+          const stat = fixtureResult[`${packageManager}_statistic`];
+          const count = fixtureResult[`${packageManager}_sample_count`];
+          const attempted = fixtureResult[`${packageManager}_attempted_runs`];
+          const min = fixtureResult[`${packageManager}_min`];
+          const max = fixtureResult[`${packageManager}_max`];
+          const unit = isPerPackage ? "ms/pkg" : "s";
+          const range = typeof min === "number" && typeof max === "number"
+            ? ` · range ${min.toFixed(2)}–${max.toFixed(2)} ${unit}` : "";
+          const partial = fixtureResult[`${packageManager}_partial`] === true ? " · partial" : "";
+          fixtureSamples[fixture] = stat === "average-of-medians" || stat === "legacy-average"
+            ? `${stat === "average-of-medians" ? "Average of medians" : "Legacy average"} · ${fixtureResult[`${packageManager}_variation_count`]} variations`
+            : `${stat === "median" ? "Median" : "Legacy mean"} · ${count === undefined ? "sample count unknown" : `${count}/${attempted ?? count} runs`}${range}${partial}`;
 
           if (isDnf) {
             fixtureDnf[fixture] = true;
@@ -118,9 +132,10 @@ export const VariationTable = ({
           fixtureValues,
           fixtureDnf,
           fixturePartial,
+          fixtureSamples,
         };
       }),
-    [filteredPackageManagers, variationData],
+    [filteredPackageManagers, variationData, isPerPackage],
   );
 
   const columns = useMemo(
@@ -199,12 +214,19 @@ export const VariationTable = ({
                       {partial}
                     </div>
                   )}
+                  <div className="mt-1 text-xs text-muted-foreground max-w-56 mx-auto">
+                    {info.row.original.fixtureSamples[fixture]}
+                  </div>
                 </div>
               );
             }
             return (
               <div className="text-center">
-                <span className="text-muted-foreground">-</span>
+                {partial ? (
+                  <span className="text-xs text-amber-700 dark:text-amber-400">
+                    {partial} · excluded from average
+                  </span>
+                ) : <span className="text-muted-foreground">-</span>}
               </div>
             );
           },

app/src/hooks/use-history-data.ts

@@ -150,7 +150,15 @@ function extractFromDataSet(
 
     for (const fixture of fixtures) {
       for (const pm of PACKAGE_MANAGERS) {
-        if (partialPMs.has(pm) || fixture[`${pm}_dnf`] === true) continue;
+        // Never splice legacy mean points into a median series. Reprocessing
+        // dated raw files adds median metadata and restores those dates.
+        if (
+          partialPMs.has(pm) ||
+          fixture[`${pm}_statistic`] !== "median" ||
+          fixture[`${pm}_dnf`] === true ||
+          fixture[`${pm}_partial`] === true
+        )
+          continue;
         const val = fixture[pm];
         if (typeof val === "number" && Number.isFinite(val)) {
           if (!pmTotals[pm]) pmTotals[pm] = { sum: 0, count: 0 };

app/src/lib/utils.ts

@@ -42,10 +42,6 @@ export const calculateAverageVariationData = (
   },
 ): FixtureResult[] => {
   type FillKey = Extract<keyof PackageManagerData, `${PackageManager}_fill`>;
-  type StddevKey = Extract<
-    keyof PackageManagerData,
-    `${PackageManager}_stddev`
-  >;
   type CountKey = Extract<keyof PackageManagerData, `${PackageManager}_count`>;
   type DnfKey = Extract<keyof PackageManagerData, `${PackageManager}_dnf`>;
 
@@ -107,9 +103,12 @@ export const calculateAverageVariationData = (
             averagedResult[key] = results.reduce((sum, r) => sum + (r[key] ?? 0), 0);
           }
         }
+        // Keep the warning visible without publishing an average based on
+        // surviving runs or quietly omitting this command's slow cases.
+        return;
       }
       const values = results
-        .filter((r) => r[dnfKey] !== true)
+        .filter((r) => r[dnfKey] !== true && r[`${pm}_partial`] !== true)
         .map((r) => r[pm])
         .filter((val): val is number => typeof val === "number" && val > 0);
 
@@ -125,25 +124,20 @@ export const calculateAverageVariationData = (
           averagedResult[fillKey] = firstFill;
         }
 
-        // Calculate average standard deviation if available
-        const stddevKey: StddevKey = `${pm}_stddev`;
-        const stddevValues = results
-          .filter((r) => r[dnfKey] !== true)
-          .map((r) => r[stddevKey])
-          .filter((val): val is number => typeof val === "number" && val > 0);
-
-        if (stddevValues.length > 0) {
-          const avgStddev =
-            stddevValues.reduce((sum, val) => sum + val, 0) /
-            stddevValues.length;
-          averagedResult[stddevKey] = avgStddev;
-        }
+        // This is an arithmetic average of benchmark medians, not a pooled
+        // sample. Averaging standard deviations would misrepresent its spread.
+        const contributing = results.filter((r) =>
+          r[dnfKey] !== true && r[`${pm}_partial`] !== true &&
+          typeof r[pm] === "number" && r[pm]! > 0);
+        averagedResult[`${pm}_statistic`] = contributing.every((r) => r[`${pm}_statistic`] === "median")
+          ? "average-of-medians" : "legacy-average";
+        averagedResult[`${pm}_variation_count`] = values.length;
 
         // For per-package data, also average the count if available
         if (isPerPackage) {
           const countKey: CountKey = `${pm}_count`;
           const countValues = results
-            .filter((r) => r[dnfKey] !== true)
+            .filter((r) => r[dnfKey] !== true && r[`${pm}_partial`] !== true)
             .map((r) => r[countKey])
             .filter((val): val is number => typeof val === "number" && val > 0);
 
@@ -326,7 +320,7 @@ export const calculateLeaderboard = (
   }
 
   // Calculate performance — DNF runs are imputed as the slowest successful
-  // time for that fixture, matching the "Performance Over Time" chart data
+  // median for that fixture; history omits failures.
   variationsToUse.forEach((variation) => {
     const dataSource = usePerPackageData
       ? chartData.perPackageCountChartData.data
@@ -668,7 +662,7 @@ export const getAvailablePackageManagers = (
     allPackageManagers.forEach((pm) => {
       const value = fixtureResult[pm];
       const dnfKey = `${pm}_dnf` as keyof FixtureResult;
-      if (fixtureResult[dnfKey] === true) {
+      if (fixtureResult[dnfKey] === true || fixtureResult[`${pm}_partial`] === true) {
         availablePackageManagers.add(pm);
         return;
       }

app/src/types/chart-data.ts

@@ -65,15 +65,21 @@ export interface BaseFixtureResult {
   fixture: Fixture;
 }
 
-type RunCompletenessData = Partial<
-  Record<`${PackageManager}_partial`, boolean> &
+type SampleMetadata = Partial<
+  Record<
+    `${PackageManager}_${"sample_count" | "attempted_runs" | "successful_runs" | "dropped_runs" | "min" | "max" | "variation_count"}`,
+    number
+  >
+> &
+  Partial<
     Record<
-      `${PackageManager}_${"attempted_runs" | "successful_runs" | "dropped_runs"}`,
-      number
+      `${PackageManager}_statistic`,
+      "median" | "legacy-mean" | "average-of-medians" | "legacy-average"
     >
->;
+  > &
+  Partial<Record<`${PackageManager}_partial`, boolean>>;
 
-export interface PackageManagerData extends RunCompletenessData {
+export interface PackageManagerData extends SampleMetadata {
   npm?: number;
   yarn?: number;
   pnpm?: number;

app/tests/partial-results.test.js

@@ -10,8 +10,15 @@ import {
   hasPartialResult,
 } from "../src/hooks/use-history-data.ts";
 
-const complete = { fixture: "next", npm: 5, vlt: 4 };
+const complete = {
+  fixture: "next",
+  npm: 5,
+  vlt: 4,
+  npm_statistic: "median",
+  vlt_statistic: "median",
+};
 const partial = {
+  ...complete,
   fixture: "astro",
   npm: 0.1,
   vlt: 3,
@@ -84,15 +91,15 @@ test("daily history omits the entire partial command instead of averaging only s
   assert.equal(hasPartialResult(chartData, ["clean", "cache"], "vlt"), false);
 });
 
-test("complete and historical daily results remain usable and DNF placeholders are excluded", () => {
+test("complete median results remain usable and DNF placeholders are excluded", () => {
   const response = {
     ...chartData,
     perPackageCountChartData: {
       ...dataSet,
       data: {
         clean: [
           complete,
-          { fixture: "astro", npm: 100, npm_dnf: true, vlt: 6 },
+          { ...complete, fixture: "astro", npm: 100, npm_dnf: true, vlt: 6 },
         ],
       },
     },