From d31b851fd59da3ae93bea5ce19ed42d77e8128c8 Mon Sep 17 00:00:00 2001
From: hannahwestra25 <hannahwestra@microsoft.com>
Date: Wed, 16 Sep 2026 15:36:02 -0400
Subject: [PATCH] FEAT: Add metrics dashboard (scorer quality + benchmark
 leaderboard) to adversarial dataset branch

Transplants the dashboard-only work from PR #2667 (hannahwestra25-metrics-dashboard)
onto this branch's already-integrated benchmark-store/cache-reuse/pipeline-automation
work from PR #2551, producing a single hackathon-demo branch with the full
data -> pipeline -> committed store -> dashboard loop.

Brings over, unchanged from #2667:
- doc/dashboard/ (0_dashboard.md, 1_scorer_quality.md, 2_benchmark_leaderboard.md,
  and the committed _generated/*.html fragments)
- build_scripts/generate_dashboard_html.py (renders the committed JSONL/scorer_evals
  data into the static HTML fragments the docs pages include)
- build_scripts/import_adversarial_benchmark_snapshot.py + its test (manual fallback
  to upsert a technique-metrics.json snapshot, e.g. downloaded from an ADO pipeline
  artifact, when the scenario result isn't in local memory)
- pyrit/datasets/benchmark_results/adversarial_benchmark_metrics.jsonl seeded with
  real ADO-ingested demo data
- doc/myst.yml (nav entry) and pyrit/common/path.py (BENCHMARK_RESULTS_PATH constant)

Deliberately NOT brought over: the exporter identity-fields/cache-reuse logic and
scenario/pipeline changes from #2667's f973f5e69, since this branch already has its
own, more advanced version of that same logic (55bece4a1, cherry-picked earlier from
f973f5e69 and reconciled against this branch's dynamic BenchmarkTechnique registry).
Re-applying #2667's version here would conflict with and regress that integration.

Also updates 0_dashboard.md's "Refreshing the data" section: it previously said there
was no CI automation for the benchmark store; this branch's pipeline now has an opt-in
publishBenchmarkStore parameter that runs the exporter and opens a PR automatically,
so the doc now describes that path alongside the fully-manual one.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
---
 build_scripts/generate_dashboard_html.py      | 287 ++++++++++++++++++
 .../import_adversarial_benchmark_snapshot.py  | 171 +++++++++++
 doc/dashboard/0_dashboard.md                  | 116 +++++++
 doc/dashboard/1_scorer_quality.md             |  39 +++
 doc/dashboard/2_benchmark_leaderboard.md      |  41 +++
 .../_generated/benchmark_leaderboard.html     |  50 +++
 .../_generated/harm_scorer_leaderboard.html   |  50 +++
 .../objective_scorer_leaderboard.html         |  50 +++
 doc/myst.yml                                  |   4 +
 pyrit/common/path.py                          |   4 +
 .../adversarial_benchmark_metrics.jsonl       |   9 +
 ...t_import_adversarial_benchmark_snapshot.py | 232 ++++++++++++++
 12 files changed, 1053 insertions(+)
 create mode 100644 build_scripts/generate_dashboard_html.py
 create mode 100644 build_scripts/import_adversarial_benchmark_snapshot.py
 create mode 100644 doc/dashboard/0_dashboard.md
 create mode 100644 doc/dashboard/1_scorer_quality.md
 create mode 100644 doc/dashboard/2_benchmark_leaderboard.md
 create mode 100644 doc/dashboard/_generated/benchmark_leaderboard.html
 create mode 100644 doc/dashboard/_generated/harm_scorer_leaderboard.html
 create mode 100644 doc/dashboard/_generated/objective_scorer_leaderboard.html
 create mode 100644 pyrit/datasets/benchmark_results/adversarial_benchmark_metrics.jsonl
 create mode 100644 tests/unit/build_scripts/test_import_adversarial_benchmark_snapshot.py

diff --git a/build_scripts/generate_dashboard_html.py b/build_scripts/generate_dashboard_html.py
new file mode 100644
index 0000000000..a294132a52
--- /dev/null
+++ b/build_scripts/generate_dashboard_html.py
@@ -0,0 +1,287 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""
+Generate the static HTML leaderboard fragments used by the metrics dashboard docs pages.
+
+The dashboard pages under `doc/dashboard/` are plain MyST Markdown files that `{include}`
+pre-rendered HTML fragments rather than notebooks, so no Python source is shown to readers.
+This script renders those fragments from the committed scorer/benchmark evaluation data and
+writes them to `doc/dashboard/_generated/`. Re-run it and commit the output whenever the
+underlying `pyrit/datasets/scorer_evals/` or `pyrit/datasets/benchmark_results/` data changes.
+
+Usage:
+    python -m build_scripts.generate_dashboard_html
+"""
+
+import asyncio
+import html
+import sys
+from typing import TYPE_CHECKING
+
+from pyrit.common.path import DOCS_PATH
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+_OUTPUT_DIR = DOCS_PATH / "dashboard" / "_generated"
+
+# Static (non-interactive) dark leaderboard-card styling shared by every card this script
+# renders. MyST includes each fragment's raw HTML directly into the page DOM (no <script>
+# execution model to rely on), so cards are pre-sorted/pre-formatted here rather than offering
+# the live search/sort a browser-side script would normally provide.
+_CARD_STYLE = """<style>
+.pyrit-leaderboard-card {
+  font-family: "SFMono-Regular", Consolas, "Liberation Mono", monospace;
+  background: #0d1117;
+  color: #e6edf3;
+  border: 1px solid #30363d;
+  border-radius: 6px;
+  max-width: 900px;
+  overflow: hidden;
+}
+.pyrit-leaderboard-card .lb-head {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 10px;
+  padding: 10px 16px;
+  background: #161b22;
+  border-bottom: 1px solid #30363d;
+}
+.pyrit-leaderboard-card .lb-head h4 {
+  margin: 0;
+  font-size: 11px;
+  text-transform: uppercase;
+  letter-spacing: 0.08em;
+  color: #8b949e;
+  font-weight: 600;
+}
+.pyrit-leaderboard-card .lb-meta { font-size: 11px; color: #8b949e; white-space: nowrap; }
+.pyrit-leaderboard-card table { width: 100%; border-collapse: collapse; font-size: 12.5px; }
+.pyrit-leaderboard-card th {
+  text-align: left;
+  padding: 8px 14px;
+  font-size: 10px;
+  text-transform: uppercase;
+  letter-spacing: 0.06em;
+  color: #3fb950;
+  border-bottom: 1px solid #30363d;
+  white-space: nowrap;
+}
+.pyrit-leaderboard-card th.lb-rank { text-align: center; }
+.pyrit-leaderboard-card td { padding: 8px 14px; border-bottom: 1px solid #21262d; }
+.pyrit-leaderboard-card td.lb-rank { color: #6e7681; text-align: center; }
+.pyrit-leaderboard-card .lb-note {
+  font-size: 11px;
+  color: #8b949e;
+  padding: 8px 16px;
+  border-top: 1px solid #30363d;
+}
+</style>
+"""
+
+
+def _format_cell(value: object, *, as_percent: bool) -> str:
+    """Format a single leaderboard cell value as display text, escaping any string content."""
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        if as_percent:
+            return f"{float(value):.0%}"
+        return f"{value:.2f}" if isinstance(value, float) else str(value)
+    return html.escape(str(value))
+
+
+def _render_leaderboard_card(
+    df: "pd.DataFrame", *, title: str, note: str, percent_columns: frozenset[str] = frozenset()
+) -> str:
+    """Render a pre-sorted DataFrame as a static dark leaderboard card (HTML/CSS only, no JS)."""
+    header_cells = "".join(f"<th>{html.escape(str(column))}</th>" for column in df.columns)
+    body_rows = "".join(
+        "<tr><td class='lb-rank'>{rank}</td>{cells}</tr>".format(
+            rank=rank,
+            cells="".join(
+                f"<td>{_format_cell(value, as_percent=column in percent_columns)}</td>" for column, value in row.items()
+            ),
+        )
+        for rank, (_, row) in enumerate(df.iterrows(), start=1)
+    )
+    row_count = len(df)
+    meta = f"{row_count} row{'s' if row_count != 1 else ''}"
+    return (
+        f'{_CARD_STYLE}<div class="pyrit-leaderboard-card">'
+        f'<div class="lb-head"><h4>{html.escape(title)}</h4><span class="lb-meta">{meta}</span></div>'
+        f'<table><thead><tr><th class="lb-rank">#</th>{header_cells}</tr></thead>'
+        f"<tbody>{body_rows}</tbody></table>"
+        f'<div class="lb-note">{html.escape(note)}</div></div>'
+    )
+
+
+def _render_empty_card(*, title: str, note: str) -> str:
+    """Render a styled card with just a note, used in place of a table when there's no data yet."""
+    return (
+        f'{_CARD_STYLE}<div class="pyrit-leaderboard-card">'
+        f'<div class="lb-head"><h4>{html.escape(title)}</h4></div>'
+        f'<div class="lb-note">{html.escape(note)}</div></div>'
+    )
+
+
+def _render_objective_scorer_leaderboard() -> str:
+    """Render the Objective Scorer Leaderboard, ranked by F1 score, as an HTML fragment."""
+    import pandas as pd
+
+    from pyrit.score import get_all_objective_metrics
+
+    objective_metrics = get_all_objective_metrics()
+    objective_metrics.sort(key=lambda entry: entry.metrics.f1_score, reverse=True)
+
+    objective_rows = [
+        {
+            "Name": entry.scorer_identifier.unique_name,
+            "Accuracy": entry.metrics.accuracy,
+            "F1 Score": entry.metrics.f1_score,
+            "Precision": entry.metrics.precision,
+            "Recall": entry.metrics.recall,
+            "Samples": entry.metrics.num_responses,
+        }
+        for entry in objective_metrics
+    ]
+
+    objective_df = pd.DataFrame(objective_rows)
+    return _render_leaderboard_card(
+        objective_df,
+        title="Objective Scorer Leaderboard",
+        note="All metrics computed against human-labeled ground truth. Ranked by F1 (higher is better across "
+        "all four metrics).",
+        percent_columns=frozenset({"Accuracy", "F1 Score", "Precision", "Recall"}),
+    )
+
+
+def _render_harm_scorer_leaderboard() -> str:
+    """Render the Harm Scorer Leaderboard, ranked by combined Krippendorff's alpha, as an HTML fragment."""
+    import pandas as pd
+
+    from pyrit.common.path import SCORER_EVALS_HARM_PATH
+    from pyrit.score import get_all_harm_metrics
+
+    # Harm categories are discovered from the files present on disk rather than a hardcoded
+    # list, so a newly added category shows up here without a code change.
+    harm_categories = sorted(
+        path.name.removesuffix("_metrics.jsonl") for path in SCORER_EVALS_HARM_PATH.glob("*_metrics.jsonl")
+    )
+
+    harm_metrics = [
+        (harm_category, entry)
+        for harm_category in harm_categories
+        for entry in get_all_harm_metrics(harm_category=harm_category)
+    ]
+    harm_metrics.sort(key=lambda item: item[1].metrics.krippendorff_alpha_combined, reverse=True)
+
+    harm_rows = [
+        {
+            "Name": entry.scorer_identifier.unique_name,
+            "Harm Category": harm_category,
+            "MAE": entry.metrics.mean_absolute_error,
+            "Alpha Combined": entry.metrics.krippendorff_alpha_combined,
+            "Alpha Humans": entry.metrics.krippendorff_alpha_humans,
+            "Alpha Model": entry.metrics.krippendorff_alpha_model,
+            "Samples": entry.metrics.num_responses,
+        }
+        for harm_category, entry in harm_metrics
+    ]
+
+    harm_df = pd.DataFrame(harm_rows)
+    return _render_leaderboard_card(
+        harm_df,
+        title="Harm Scorer Leaderboard",
+        note="MAE: lower is better (0-1 scale). Alpha = Krippendorff's alpha: higher is better (agreement "
+        "between scorer and ground truth). Not comparable across harm categories.",
+    )
+
+
+def _render_benchmark_leaderboard() -> str:
+    """Render the Technique / Adversarial Model Leaderboard, ranked by success rate, as an HTML fragment."""
+    import pandas as pd
+
+    from pyrit.common.path import BENCHMARK_RESULTS_PATH
+
+    store_path = BENCHMARK_RESULTS_PATH / "adversarial_benchmark_metrics.jsonl"
+    if not store_path.exists():
+        return _render_empty_card(
+            title="Technique / Adversarial Model Leaderboard",
+            note=f"No benchmark data yet at {store_path}.",
+        )
+
+    column_labels = {
+        "technique": "Technique",
+        "adversarial_model": "Adversarial Model",
+        "objective_target": "Objective Target",
+        "objective_scorer": "Objective Scorer",
+        "dataset": "Dataset",
+        "total": "N",
+        "success": "Success",
+        "failure": "Failure",
+        "error": "Error",
+        "undetermined": "Undetermined",
+        "retry_records": "Retries",
+        "success_rate": "Success Rate",
+    }
+
+    benchmark_df = pd.read_json(store_path, lines=True)
+    benchmark_df = benchmark_df.sort_values("success_rate", ascending=False)
+    display_df = benchmark_df.rename(columns=column_labels)[list(column_labels.values())]
+    return _render_leaderboard_card(
+        display_df,
+        title="Technique / Adversarial Model Leaderboard",
+        note="Ranked by success rate. N is the number of objectives attempted for that combination; treat rows "
+        "with a small N as directional, not statistically robust.",
+        percent_columns=frozenset({"Success Rate"}),
+    )
+
+
+async def generate_dashboard_html_async() -> None:
+    """
+    Regenerate every HTML fragment the metrics dashboard docs pages `{include}`.
+
+    Initializes PyRIT with an in-memory database, renders the Objective Scorer, Harm Scorer,
+    and Benchmark leaderboards from the committed evaluation data, and writes each as a
+    standalone HTML fragment to `doc/dashboard/_generated/`.
+    """
+    from pyrit.setup import IN_MEMORY, initialize_pyrit_async
+
+    print("Initializing PyRIT...")
+    await initialize_pyrit_async(memory_db_type=IN_MEMORY, silent=True)
+
+    _OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    fragments = {
+        "objective_scorer_leaderboard.html": _render_objective_scorer_leaderboard(),
+        "harm_scorer_leaderboard.html": _render_harm_scorer_leaderboard(),
+        "benchmark_leaderboard.html": _render_benchmark_leaderboard(),
+    }
+    for filename, content in fragments.items():
+        output_path = _OUTPUT_DIR / filename
+        output_path.write_text(content, encoding="utf-8")
+        print(f"  Wrote {output_path}")
+
+    print("Done. Commit the updated files under doc/dashboard/_generated/ through a normal PR.")
+
+
+if __name__ == "__main__":
+    print("=" * 60)
+    print("PyRIT Dashboard HTML Generator")
+    print("=" * 60)
+    print(f"Fragments will be written to: {_OUTPUT_DIR}")
+    print("=" * 60)
+    print()
+
+    try:
+        asyncio.run(generate_dashboard_html_async())
+    except KeyboardInterrupt:
+        print("\n\nInterrupted by user.")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n\nFatal error: {e}")
+        import traceback
+
+        traceback.print_exc()
+        sys.exit(1)
diff --git a/build_scripts/import_adversarial_benchmark_snapshot.py b/build_scripts/import_adversarial_benchmark_snapshot.py
new file mode 100644
index 0000000000..d6eaa16a33
--- /dev/null
+++ b/build_scripts/import_adversarial_benchmark_snapshot.py
@@ -0,0 +1,171 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""
+Upsert an already-exported ``technique-metrics.json`` snapshot into the committed benchmark store.
+
+Use this when the technique-metrics rows were produced somewhere this checkout can't reach directly
+— for example a ``technique-metrics.json`` downloaded from an Azure DevOps
+``adversarial-benchmark-<BuildId>`` pipeline artifact (see ``.azuredevops/adversarial-benchmark.yml``)
+— so there's no local SQLite memory holding the underlying scenario result.
+``build_scripts.export_adversarial_benchmark_result --update-benchmark-store`` remains the right tool
+when the scenario result *is* available locally; this script only merges a snapshot that already went
+through that exporter elsewhere.
+
+Snapshots produced before ``objective_target``/``objective_scorer``/``dataset`` existed on the
+exporter (e.g. a ``main``-branch pipeline run predating this dashboard's schema) won't carry those
+fields at all. Pass ``--default-objective-target``/``--default-objective-scorer``/``--default-dataset``
+to backfill whichever ones are missing; a row that already has a field keeps its own value.
+
+Example:
+    python -m build_scripts.import_adversarial_benchmark_snapshot \\
+        --technique-metrics-json ./adversarial-benchmark-16150/technique-metrics.json \\
+        --default-objective-target openai_chat \\
+        --default-dataset adversarial_benchmark_v1
+"""
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+from build_scripts.export_adversarial_benchmark_result import DEFAULT_BENCHMARK_STORE_PATH, upsert_benchmark_metrics
+
+#: Fallback label for an identity field that's missing from the source snapshot and wasn't
+#: backfilled via a ``--default-*`` flag. Mirrors the fallback
+#: ``pyrit.scenario.scenarios.benchmark.adversarial.resolve_objective_identity`` uses for an
+#: unresolved identifier, so an unlabeled row reads the same way here as it would coming
+#: straight out of the exporter.
+_UNKNOWN_IDENTITY = "<unknown>"
+
+#: The identity fields ``upsert_benchmark_metrics`` requires on every row (see
+#: ``_BENCHMARK_METRICS_KEY_FIELDS``) that older snapshots may not carry.
+_IDENTITY_FIELDS = ("objective_target", "objective_scorer", "dataset")
+
+
+def _load_metrics(*, technique_metrics_json: Path) -> list[dict[str, Any]]:
+    """Read and validate a previously exported technique-metrics.json file."""
+    metrics = json.loads(technique_metrics_json.read_text(encoding="utf-8"))
+    if not isinstance(metrics, list):
+        raise ValueError(f"{technique_metrics_json} must contain a JSON list of technique-metrics rows.")
+    return metrics
+
+
+def _fill_missing_identity(
+    *, metrics: list[dict[str, Any]], identity_defaults: dict[str, str | None]
+) -> list[dict[str, Any]]:
+    """
+    Backfill identity fields that are absent from a row, without touching fields it already has.
+
+    Older snapshots (e.g. from an exporter version that predates ``objective_target``/
+    ``objective_scorer``/``dataset``) may be missing some or all of these fields entirely.
+    ``upsert_benchmark_metrics`` requires all of them to compute its upsert key, so each missing
+    field is set to the matching ``identity_defaults`` value if one was supplied, or to
+    ``_UNKNOWN_IDENTITY`` otherwise. A field a row already has is never overwritten.
+
+    Args:
+        metrics (list[dict[str, Any]]): Technique-metrics rows to backfill.
+        identity_defaults (dict[str, str | None]): Optional default value for each of
+            ``_IDENTITY_FIELDS``, keyed by field name.
+
+    Returns:
+        list[dict[str, Any]]: New row dicts with every identity field present.
+    """
+    filled: list[dict[str, Any]] = []
+    for row in metrics:
+        row = dict(row)
+        for field in _IDENTITY_FIELDS:
+            if not row.get(field):
+                row[field] = identity_defaults.get(field) or _UNKNOWN_IDENTITY
+        filled.append(row)
+    return filled
+
+
+def import_benchmark_snapshot(
+    *,
+    technique_metrics_json: Path,
+    benchmark_store_path: Path | None = None,
+    default_objective_target: str | None = None,
+    default_objective_scorer: str | None = None,
+    default_dataset: str | None = None,
+) -> int:
+    """
+    Upsert a technique-metrics.json snapshot into the committed benchmark metrics store.
+
+    Args:
+        technique_metrics_json (Path): Path to a ``technique-metrics.json`` file, as written by
+            ``export_adversarial_benchmark_result.py``.
+        benchmark_store_path (Path | None): Override for the committed store path. Defaults to
+            ``DEFAULT_BENCHMARK_STORE_PATH``.
+        default_objective_target (str | None): Value to backfill onto any row missing
+            ``objective_target``. Leaves the field untouched on rows that already have it.
+        default_objective_scorer (str | None): Value to backfill onto any row missing
+            ``objective_scorer``. Leaves the field untouched on rows that already have it.
+        default_dataset (str | None): Value to backfill onto any row missing ``dataset``. Leaves
+            the field untouched on rows that already have it.
+
+    Returns:
+        int: The number of rows upserted.
+    """
+    metrics = _load_metrics(technique_metrics_json=technique_metrics_json)
+    metrics = _fill_missing_identity(
+        metrics=metrics,
+        identity_defaults={
+            "objective_target": default_objective_target,
+            "objective_scorer": default_objective_scorer,
+            "dataset": default_dataset,
+        },
+    )
+    store_path = benchmark_store_path or DEFAULT_BENCHMARK_STORE_PATH
+    upsert_benchmark_metrics(metrics=metrics, store_path=store_path)
+    return len(metrics)
+
+
+def main() -> int:
+    """Run the snapshot import."""
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument(
+        "--technique-metrics-json",
+        type=Path,
+        required=True,
+        help="Path to a technique-metrics.json file exported from a benchmark run.",
+    )
+    parser.add_argument(
+        "--benchmark-store-path",
+        type=Path,
+        default=None,
+        help=f"Override the committed benchmark metrics store path (default: {DEFAULT_BENCHMARK_STORE_PATH}).",
+    )
+    parser.add_argument(
+        "--default-objective-target",
+        default=None,
+        help="Backfill onto any row missing 'objective_target' (rows that already have it are left alone).",
+    )
+    parser.add_argument(
+        "--default-objective-scorer",
+        default=None,
+        help="Backfill onto any row missing 'objective_scorer' (rows that already have it are left alone).",
+    )
+    parser.add_argument(
+        "--default-dataset",
+        default=None,
+        help="Backfill onto any row missing 'dataset' (rows that already have it are left alone).",
+    )
+    args = parser.parse_args()
+    try:
+        count = import_benchmark_snapshot(
+            technique_metrics_json=args.technique_metrics_json,
+            benchmark_store_path=args.benchmark_store_path,
+            default_objective_target=args.default_objective_target,
+            default_objective_scorer=args.default_objective_scorer,
+            default_dataset=args.default_dataset,
+        )
+    except Exception as error:
+        parser.exit(1, f"Import failed: {error}\n")
+    store_path = args.benchmark_store_path or DEFAULT_BENCHMARK_STORE_PATH
+    print(f"Upserted {count} row(s) into {store_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/doc/dashboard/0_dashboard.md b/doc/dashboard/0_dashboard.md
new file mode 100644
index 0000000000..960b5a29a2
--- /dev/null
+++ b/doc/dashboard/0_dashboard.md
@@ -0,0 +1,116 @@
+# Metrics Dashboard
+
+PyRIT tracks numeric "how good is this component" metrics for several parts of the framework.
+This section renders those metrics as leaderboard tables, so you can compare configurations at
+a glance instead of digging through JSONL files by hand.
+
+## What's here today
+
+- **[Scorer Quality](1_scorer_quality.md)** — an Objective Scorer Leaderboard (accuracy,
+  F1, precision, recall) and a Harm Scorer Leaderboard (mean absolute error, Krippendorff's
+  alpha), built from the evaluation registries the team already maintains under
+  `pyrit/datasets/scorer_evals/`. See [Scorer Metrics](../code/scoring/4_scorer_metrics.ipynb)
+  for what these numbers mean and [Scoring Scorers](../blog/2026_04_14_scoring_scorers.md) for
+  the full evaluation framework.
+- **[Benchmark Leaderboard](2_benchmark_leaderboard.md)** — attack success rate by technique
+  and adversarial model, built from `AdversarialBenchmark` scenario runs via
+  `build_scripts/export_adversarial_benchmark_result.py --update-benchmark-store`. The data
+  behind it today is a small demo-scale run (see the page's "Note on scope") — treat it as a
+  preview of the mechanism, not a statistically robust evaluation yet.
+
+## What's planned
+
+An objective-target robustness leaderboard (comparing target models against a fixed
+adversarial model, the mirror image of today's benchmark leaderboard) reuses the same
+scenario, exporter, and store — only the grouping changes. A larger, regularly-refreshed
+sweep across more techniques, models, and dataset items is future work; see the "Note on
+scope" section on the Benchmark Leaderboard page.
+
+## Refreshing the data
+
+Both pages `{include}` static HTML fragments under `_generated/` rather than running any Python
+when the site builds, so refreshing the dashboard is a human-in-the-loop process: regenerate
+the underlying JSONL data, regenerate the fragments, then commit both.
+
+**Scorer Quality:**
+
+1. Regenerate scorer metrics locally:
+
+   ```bash
+   python -m build_scripts.evaluate_scorers
+   ```
+
+   This is safe to re-run — scorer configurations that already have up-to-date metrics are
+   skipped automatically (see [Scorer Metrics](../code/scoring/4_scorer_metrics.ipynb)).
+   Commit the updated files under `pyrit/datasets/scorer_evals/` through a normal PR.
+2. Regenerate the HTML fragments (see below) and commit them alongside the data.
+
+**Benchmark Leaderboard:**
+
+The `.azuredevops/adversarial-benchmark.yml` pipeline can run the scenario, upsert the
+committed store, and open a PR with the change automatically — set its `publishBenchmarkStore`
+parameter to `true` (requires a `GITHUB_PAT` secret pipeline variable; see the comment at the
+top of that file). This still only updates the JSONL store — regenerating the HTML fragments
+(step 2 below) remains a separate, manual step after that PR merges. To do the whole flow by
+hand instead:
+
+1. Run the scenario and refresh the committed store:
+
+   ```bash
+   python -m build_scripts.run_adversarial_benchmark \
+     --objective-target <target> \
+     --adversarial-targets <target> [<target> ...] \
+     --output-dir <dir>
+   ```
+
+   This is safe to re-run — combinations already present in the committed store, or already
+   completed in memory, are skipped automatically, so re-running after adding a new adversarial
+   target only executes the new combinations. Pass `--force` to ignore both caches and re-run
+   everything. Omit `--output-dir` to only run the scenario and print its `scenario_result_id`
+   (export separately with the command below).
+
+   For finer control over dataset selection or techniques, run the scenario directly instead
+   (see [Benchmark Scenarios](../scanner/benchmark.ipynb) for the full `pyrit_scan` invocation
+   and available techniques/targets), then export and upsert its result into the committed
+   store:
+
+   ```bash
+   python -m build_scripts.export_adversarial_benchmark_result \
+     --scenario-result-id <id> \
+     --output-dir <dir> \
+     --update-benchmark-store
+   ```
+
+   Upserting is keyed on `(technique, adversarial_model, objective_target, objective_scorer,
+   dataset)`, so re-running the same combination replaces its row instead of appending. Commit
+   the updated `pyrit/datasets/benchmark_results/adversarial_benchmark_metrics.jsonl` through a
+   normal PR.
+
+   Already have a `technique-metrics.json` from elsewhere — for example downloaded from an
+   Azure DevOps `adversarial-benchmark-<BuildId>` pipeline artifact (see
+   `.azuredevops/adversarial-benchmark.yml`) — rather than a scenario result in local memory?
+   Import it directly instead of re-running the scenario:
+
+   ```bash
+   python -m build_scripts.import_adversarial_benchmark_snapshot \
+     --technique-metrics-json <path-to-technique-metrics.json>
+   ```
+
+   This upserts into the same committed store using the same key, so it composes with either
+   path above.
+2. Regenerate the HTML fragments (see below) and commit them alongside the data.
+
+**Regenerating the HTML fragments:**
+
+```bash
+python -m build_scripts.generate_dashboard_html
+```
+
+This re-reads all of the committed JSONL files above and (re)writes every fragment under
+`doc/dashboard/_generated/` — both pages' tables at once, regardless of which data changed.
+Commit the updated fragment files through a normal PR. The site itself rebuilds automatically
+on every merge to `main`, but only re-renders whatever HTML fragments are already committed —
+it never runs this generator for you.
+
+Scorer Quality and the HTML fragment regeneration step are still fully manual today. Only the
+Benchmark Leaderboard's store-update step has opt-in CI automation, as described above.
diff --git a/doc/dashboard/1_scorer_quality.md b/doc/dashboard/1_scorer_quality.md
new file mode 100644
index 0000000000..b62e770586
--- /dev/null
+++ b/doc/dashboard/1_scorer_quality.md
@@ -0,0 +1,39 @@
+# Scorer Quality
+
+Leaderboards for the scorer evaluation metrics PyRIT already tracks under
+`pyrit/datasets/scorer_evals/`. See [Scorer Metrics](../code/scoring/4_scorer_metrics.ipynb) for
+what each metric means and how these numbers are produced.
+
+## Objective Scorer Leaderboard
+
+Objective scorers answer a true/false question (e.g. "was the objective achieved?"). Ranked by
+F1 score, the harmonic mean of precision and recall.
+
+```{include} _generated/objective_scorer_leaderboard.html
+```
+
+## Harm Scorer Leaderboard
+
+Harm scorers produce a severity score (0.0-1.0). Ranked by `krippendorff_alpha_combined` —
+agreement between the model's scores and human raters, ranging from -1.0 (systematic
+disagreement) to 1.0 (perfect agreement) — across every harm category PyRIT currently has
+metrics for. Alpha isn't comparable *across* categories (each has its own human-labeled
+dataset), so treat this as one leaderboard per category, stacked into a single table for
+convenience.
+
+```{include} _generated/harm_scorer_leaderboard.html
+```
+
+## Note on scope
+
+`get_all_objective_metrics()` reads `objective/objective_achieved_metrics.jsonl` only, matching
+how it's documented and used elsewhere in PyRIT. A separate `refusal_scorer/refusal_metrics.jsonl`
+registry evaluates refusal scorers against its own human-labeled dataset, using the same
+`ObjectiveScorerMetrics` shape. It isn't merged into the leaderboard above because it measures a
+different task (refusal detection, not objective achievement) against a different ground truth
+set, and mixing the two would make the F1 ranking misleading. A follow-up could add it as its own
+leaderboard.
+
+These leaderboards are static snapshots, not live views: they're rendered from committed JSONL
+data by `python -m build_scripts.generate_dashboard_html` (see
+[Refreshing the data](0_dashboard.md#refreshing-the-data)), not computed when this page loads.
diff --git a/doc/dashboard/2_benchmark_leaderboard.md b/doc/dashboard/2_benchmark_leaderboard.md
new file mode 100644
index 0000000000..e70ec0ddba
--- /dev/null
+++ b/doc/dashboard/2_benchmark_leaderboard.md
@@ -0,0 +1,41 @@
+# Benchmark Leaderboard
+
+Attack success rate (ASR) for adversarial techniques and models, built from `AdversarialBenchmark`
+scenario runs. See [Benchmark Scenarios](../scanner/benchmark.ipynb) for how to run the scenario
+yourself and `build_scripts/export_adversarial_benchmark_result.py` for how a run's results land
+in the committed store this page reads.
+
+## Technique / Adversarial Model Leaderboard
+
+Ranked by success rate — the share of objectives where the target's response was scored as
+achieving the objective — descending. `N` is the number of objectives attempted for that
+(technique, adversarial model, objective target, objective scorer, dataset) combination; treat
+rows with a small `N` as directional, not statistically robust.
+
+```{include} _generated/benchmark_leaderboard.html
+```
+
+## Note on scope
+
+This table is a snapshot upserted by `--update-benchmark-store`, keyed on
+`(technique, adversarial_model, objective_target, objective_scorer, dataset)` — a fresh run of
+the same combination replaces its row rather than appending, so this stays a single
+best-known-result table rather than an unbounded history. Small `N` values (as in the rows
+above) reflect small demo-scale runs (`--max-dataset-size`), not a statistically robust
+evaluation; treat them as a preview of the mechanism rather than a final verdict on any
+technique or model. Widening this into a larger, regularly-refreshed sweep — and adding a
+native objective-target comparison alongside the current adversarial-model comparison — is
+future work.
+
+The `crescendo_simulated` / `role_play_video_game` / `tap` rows on the `adversarial_benchmark_v1`
+dataset were imported via `build_scripts/import_adversarial_benchmark_snapshot.py` from an Azure
+DevOps pipeline run of PyRIT PR #2551's dataset and pipeline changes, which had not yet merged to
+`main` at the time. They're included as a preview of what that pipeline produces once merged.
+`objective_scorer` reads `<unknown>` for these rows because the run predates this store's identity
+fields and its exact scorer wasn't captured alongside the artifact; treat these rows as
+provisional until #2551 merges and a run through the updated exporter reproduces them with full
+identity.
+
+This leaderboard is a static snapshot, not a live view: it's rendered from the committed JSONL
+store by `python -m build_scripts.generate_dashboard_html` (see
+[Refreshing the data](0_dashboard.md#refreshing-the-data)), not computed when this page loads.
diff --git a/doc/dashboard/_generated/benchmark_leaderboard.html b/doc/dashboard/_generated/benchmark_leaderboard.html
new file mode 100644
index 0000000000..89450fa12c
--- /dev/null
+++ b/doc/dashboard/_generated/benchmark_leaderboard.html
@@ -0,0 +1,50 @@
+<style>
+.pyrit-leaderboard-card {
+  font-family: "SFMono-Regular", Consolas, "Liberation Mono", monospace;
+  background: #0d1117;
+  color: #e6edf3;
+  border: 1px solid #30363d;
+  border-radius: 6px;
+  max-width: 900px;
+  overflow: hidden;
+}
+.pyrit-leaderboard-card .lb-head {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 10px;
+  padding: 10px 16px;
+  background: #161b22;
+  border-bottom: 1px solid #30363d;
+}
+.pyrit-leaderboard-card .lb-head h4 {
+  margin: 0;
+  font-size: 11px;
+  text-transform: uppercase;
+  letter-spacing: 0.08em;
+  color: #8b949e;
+  font-weight: 600;
+}
+.pyrit-leaderboard-card .lb-meta { font-size: 11px; color: #8b949e; white-space: nowrap; }
+.pyrit-leaderboard-card table { width: 100%; border-collapse: collapse; font-size: 12.5px; }
+.pyrit-leaderboard-card th {
+  text-align: left;
+  padding: 8px 14px;
+  font-size: 10px;
+  text-transform: uppercase;
+  letter-spacing: 0.06em;
+  color: #3fb950;
+  border-bottom: 1px solid #30363d;
+  white-space: nowrap;
+}
+.pyrit-leaderboard-card th.lb-rank { text-align: center; }
+.pyrit-leaderboard-card td { padding: 8px 14px; border-bottom: 1px solid #21262d; }
+.pyrit-leaderboard-card td.lb-rank { color: #6e7681; text-align: center; }
+.pyrit-leaderboard-card .lb-note {
+  font-size: 11px;
+  color: #8b949e;
+  padding: 8px 16px;
+  border-top: 1px solid #30363d;
+}
+</style>
+<div class="pyrit-leaderboard-card"><div class="lb-head"><h4>Technique / Adversarial Model Leaderboard</h4><span class="lb-meta">9 rows</span></div><table><thead><tr><th class="lb-rank">#</th><th>Technique</th><th>Adversarial Model</th><th>Objective Target</th><th>Objective Scorer</th><th>Dataset</th><th>N</th><th>Success</th><th>Failure</th><th>Error</th><th>Undetermined</th><th>Retries</th><th>Success Rate</th></tr></thead><tbody><tr><td class='lb-rank'>1</td><td>role_play_video_game</td><td>adversarial_chat</td><td>gpt-4o</td><td>TrueFalseInverterScorer</td><td>harmbench</td><td>1</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0</td><td>100%</td></tr><tr><td class='lb-rank'>2</td><td>tap</td><td>azure_gpt4o_unsafe_chat</td><td>openai_chat</td><td>&lt;unknown&gt;</td><td>adversarial_benchmark_v1</td><td>24</td><td>24</td><td>0</td><td>0</td><td>0</td><td>0</td><td>100%</td></tr><tr><td class='lb-rank'>3</td><td>red_teaming</td><td>adversarial_chat</td><td>gpt-4o</td><td>TrueFalseInverterScorer</td><td>harmbench</td><td>1</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0</td><td>100%</td></tr><tr><td class='lb-rank'>4</td><td>tap</td><td>azure_openai_gpt4o</td><td>openai_chat</td><td>&lt;unknown&gt;</td><td>adversarial_benchmark_v1</td><td>24</td><td>22</td><td>2</td><td>0</td><td>0</td><td>0</td><td>92%</td></tr><tr><td class='lb-rank'>5</td><td>role_play_video_game</td><td>azure_openai_gpt4o</td><td>openai_chat</td><td>&lt;unknown&gt;</td><td>adversarial_benchmark_v1</td><td>24</td><td>8</td><td>15</td><td>0</td><td>1</td><td>0</td><td>33%</td></tr><tr><td class='lb-rank'>6</td><td>crescendo_simulated</td><td>azure_openai_gpt4o</td><td>openai_chat</td><td>&lt;unknown&gt;</td><td>adversarial_benchmark_v1</td><td>24</td><td>8</td><td>15</td><td>0</td><td>1</td><td>0</td><td>33%</td></tr><tr><td class='lb-rank'>7</td><td>role_play_video_game</td><td>azure_gpt4o_unsafe_chat</td><td>openai_chat</td><td>&lt;unknown&gt;</td><td>adversarial_benchmark_v1</td><td>24</td><td>4</td><td>19</td><td>0</td><td>1</td><td>0</td><td>17%</td></tr><tr><td class='lb-rank'>8</td><td>crescendo_simulated</td><td>azure_gpt4o_unsafe_chat</td><td>openai_chat</td><td>&lt;unknown&gt;</td><td>adversarial_benchmark_v1</td><td>24</td><td>3</td><td>20</td><td>0</td><td>1</td><td>0</td><td>12%</td></tr><tr><td class='lb-rank'>9</td><td>context_compliance</td><td>adversarial_chat</td><td>gpt-4o</td><td>TrueFalseInverterScorer</td><td>harmbench</td><td>1</td><td>0</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0%</td></tr></tbody></table><div class="lb-note">Ranked by success rate. N is the number of objectives attempted for that combination; treat rows with a small N as directional, not statistically robust.</div></div>
diff --git a/doc/dashboard/_generated/harm_scorer_leaderboard.html b/doc/dashboard/_generated/harm_scorer_leaderboard.html
new file mode 100644
index 0000000000..92d893de7f
--- /dev/null
+++ b/doc/dashboard/_generated/harm_scorer_leaderboard.html
@@ -0,0 +1,50 @@
+<style>
+.pyrit-leaderboard-card {
+  font-family: "SFMono-Regular", Consolas, "Liberation Mono", monospace;
+  background: #0d1117;
+  color: #e6edf3;
+  border: 1px solid #30363d;
+  border-radius: 6px;
+  max-width: 900px;
+  overflow: hidden;
+}
+.pyrit-leaderboard-card .lb-head {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 10px;
+  padding: 10px 16px;
+  background: #161b22;
+  border-bottom: 1px solid #30363d;
+}
+.pyrit-leaderboard-card .lb-head h4 {
+  margin: 0;
+  font-size: 11px;
+  text-transform: uppercase;
+  letter-spacing: 0.08em;
+  color: #8b949e;
+  font-weight: 600;
+}
+.pyrit-leaderboard-card .lb-meta { font-size: 11px; color: #8b949e; white-space: nowrap; }
+.pyrit-leaderboard-card table { width: 100%; border-collapse: collapse; font-size: 12.5px; }
+.pyrit-leaderboard-card th {
+  text-align: left;
+  padding: 8px 14px;
+  font-size: 10px;
+  text-transform: uppercase;
+  letter-spacing: 0.06em;
+  color: #3fb950;
+  border-bottom: 1px solid #30363d;
+  white-space: nowrap;
+}
+.pyrit-leaderboard-card th.lb-rank { text-align: center; }
+.pyrit-leaderboard-card td { padding: 8px 14px; border-bottom: 1px solid #21262d; }
+.pyrit-leaderboard-card td.lb-rank { color: #6e7681; text-align: center; }
+.pyrit-leaderboard-card .lb-note {
+  font-size: 11px;
+  color: #8b949e;
+  padding: 8px 16px;
+  border-top: 1px solid #30363d;
+}
+</style>
+<div class="pyrit-leaderboard-card"><div class="lb-head"><h4>Harm Scorer Leaderboard</h4><span class="lb-meta">36 rows</span></div><table><thead><tr><th class="lb-rank">#</th><th>Name</th><th>Harm Category</th><th>MAE</th><th>Alpha Combined</th><th>Alpha Humans</th><th>Alpha Model</th><th>Samples</th></tr></thead><tbody><tr><td class='lb-rank'>1</td><td>SelfAskLikertScorer::ce5da81b</td><td>sexual</td><td>0.13</td><td>0.90</td><td>None</td><td>0.98</td><td>78</td></tr><tr><td class='lb-rank'>2</td><td>AzureContentFilterScorer::1a9b9789</td><td>hate_speech</td><td>0.17</td><td>0.86</td><td>None</td><td>1.00</td><td>59</td></tr><tr><td class='lb-rank'>3</td><td>SelfAskLikertScorer::ce31ba14</td><td>hate_speech</td><td>0.17</td><td>0.85</td><td>None</td><td>0.95</td><td>59</td></tr><tr><td class='lb-rank'>4</td><td>SelfAskLikertScorer::772e51c8</td><td>violence</td><td>0.16</td><td>0.85</td><td>None</td><td>0.93</td><td>96</td></tr><tr><td class='lb-rank'>5</td><td>SelfAskLikertScorer::1607ea13</td><td>hate_speech</td><td>0.17</td><td>0.85</td><td>None</td><td>0.95</td><td>59</td></tr><tr><td class='lb-rank'>6</td><td>SelfAskLikertScorer::8e9637f6</td><td>hate_speech</td><td>0.18</td><td>0.85</td><td>None</td><td>0.95</td><td>59</td></tr><tr><td class='lb-rank'>7</td><td>SelfAskLikertScorer::118336dd</td><td>violence</td><td>0.17</td><td>0.85</td><td>None</td><td>0.96</td><td>96</td></tr><tr><td class='lb-rank'>8</td><td>SelfAskLikertScorer::67df3fb9</td><td>violence</td><td>0.18</td><td>0.84</td><td>None</td><td>0.97</td><td>96</td></tr><tr><td class='lb-rank'>9</td><td>AzureContentFilterScorer::9e2a1052</td><td>sexual</td><td>0.19</td><td>0.84</td><td>None</td><td>1.00</td><td>78</td></tr><tr><td class='lb-rank'>10</td><td>SelfAskLikertScorer::0317dc04</td><td>sexual</td><td>0.15</td><td>0.82</td><td>None</td><td>0.87</td><td>78</td></tr><tr><td class='lb-rank'>11</td><td>SelfAskLikertScorer::e5d67218</td><td>sexual</td><td>0.18</td><td>0.82</td><td>None</td><td>0.90</td><td>78</td></tr><tr><td class='lb-rank'>12</td><td>SelfAskLikertScorer::4e3e41ce</td><td>hate_speech</td><td>0.17</td><td>0.82</td><td>None</td><td>0.90</td><td>59</td></tr><tr><td class='lb-rank'>13</td><td>SelfAskLikertScorer::5d757b74</td><td>violence</td><td>0.19</td><td>0.81</td><td>None</td><td>0.91</td><td>96</td></tr><tr><td class='lb-rank'>14</td><td>SelfAskLikertScorer::62858f6e</td><td>sexual</td><td>0.18</td><td>0.80</td><td>None</td><td>0.86</td><td>78</td></tr><tr><td class='lb-rank'>15</td><td>AzureContentFilterScorer::c4552a47</td><td>violence</td><td>0.24</td><td>0.78</td><td>None</td><td>1.00</td><td>96</td></tr><tr><td class='lb-rank'>16</td><td>SelfAskLikertScorer::32c60e51</td><td>exploits</td><td>0.28</td><td>0.67</td><td>None</td><td>0.96</td><td>80</td></tr><tr><td class='lb-rank'>17</td><td>AzureContentFilterScorer::bfa15fad</td><td>self_harm</td><td>0.27</td><td>0.66</td><td>None</td><td>1.00</td><td>93</td></tr><tr><td class='lb-rank'>18</td><td>SelfAskLikertScorer::ab49f40e</td><td>information_integrity</td><td>0.27</td><td>0.62</td><td>None</td><td>0.88</td><td>76</td></tr><tr><td class='lb-rank'>19</td><td>SelfAskLikertScorer::5381888a</td><td>information_integrity</td><td>0.29</td><td>0.61</td><td>None</td><td>0.90</td><td>76</td></tr><tr><td class='lb-rank'>20</td><td>SelfAskLikertScorer::1d068582</td><td>information_integrity</td><td>0.29</td><td>0.61</td><td>None</td><td>0.89</td><td>76</td></tr><tr><td class='lb-rank'>21</td><td>SelfAskLikertScorer::4574a6f9</td><td>information_integrity</td><td>0.30</td><td>0.59</td><td>None</td><td>0.86</td><td>76</td></tr><tr><td class='lb-rank'>22</td><td>SelfAskLikertScorer::0ac1f197</td><td>representational</td><td>0.32</td><td>0.58</td><td>None</td><td>0.96</td><td>29</td></tr><tr><td class='lb-rank'>23</td><td>SelfAskLikertScorer::f6470940</td><td>representational</td><td>0.33</td><td>0.56</td><td>None</td><td>0.94</td><td>29</td></tr><tr><td class='lb-rank'>24</td><td>SelfAskLikertScorer::146a2e44</td><td>representational</td><td>0.28</td><td>0.56</td><td>None</td><td>0.75</td><td>29</td></tr><tr><td class='lb-rank'>25</td><td>SelfAskLikertScorer::ac9997f6</td><td>representational</td><td>0.32</td><td>0.55</td><td>None</td><td>0.92</td><td>29</td></tr><tr><td class='lb-rank'>26</td><td>SelfAskLikertScorer::aa245673</td><td>self_harm</td><td>0.28</td><td>0.54</td><td>None</td><td>0.75</td><td>93</td></tr><tr><td class='lb-rank'>27</td><td>SelfAskLikertScorer::d79a7e70</td><td>exploits</td><td>0.34</td><td>0.49</td><td>None</td><td>0.89</td><td>80</td></tr><tr><td class='lb-rank'>28</td><td>SelfAskLikertScorer::a69c552b</td><td>exploits</td><td>0.33</td><td>0.49</td><td>None</td><td>0.92</td><td>80</td></tr><tr><td class='lb-rank'>29</td><td>SelfAskLikertScorer::d48dfb38</td><td>exploits</td><td>0.34</td><td>0.49</td><td>None</td><td>0.88</td><td>80</td></tr><tr><td class='lb-rank'>30</td><td>SelfAskLikertScorer::e6464803</td><td>privacy</td><td>0.37</td><td>0.45</td><td>None</td><td>0.89</td><td>93</td></tr><tr><td class='lb-rank'>31</td><td>SelfAskLikertScorer::9df5a574</td><td>self_harm</td><td>0.35</td><td>0.43</td><td>None</td><td>0.85</td><td>93</td></tr><tr><td class='lb-rank'>32</td><td>SelfAskLikertScorer::af0ef7c7</td><td>self_harm</td><td>0.35</td><td>0.43</td><td>None</td><td>0.90</td><td>93</td></tr><tr><td class='lb-rank'>33</td><td>SelfAskLikertScorer::bbbe93be</td><td>privacy</td><td>0.37</td><td>0.42</td><td>None</td><td>0.65</td><td>93</td></tr><tr><td class='lb-rank'>34</td><td>SelfAskLikertScorer::6896be1e</td><td>self_harm</td><td>0.36</td><td>0.42</td><td>None</td><td>0.84</td><td>93</td></tr><tr><td class='lb-rank'>35</td><td>SelfAskLikertScorer::d43c4413</td><td>privacy</td><td>0.40</td><td>0.39</td><td>None</td><td>0.84</td><td>93</td></tr><tr><td class='lb-rank'>36</td><td>SelfAskLikertScorer::73eb64f9</td><td>privacy</td><td>0.38</td><td>0.38</td><td>None</td><td>0.81</td><td>93</td></tr></tbody></table><div class="lb-note">MAE: lower is better (0-1 scale). Alpha = Krippendorff&#x27;s alpha: higher is better (agreement between scorer and ground truth). Not comparable across harm categories.</div></div>
diff --git a/doc/dashboard/_generated/objective_scorer_leaderboard.html b/doc/dashboard/_generated/objective_scorer_leaderboard.html
new file mode 100644
index 0000000000..5ddf896c33
--- /dev/null
+++ b/doc/dashboard/_generated/objective_scorer_leaderboard.html
@@ -0,0 +1,50 @@
+<style>
+.pyrit-leaderboard-card {
+  font-family: "SFMono-Regular", Consolas, "Liberation Mono", monospace;
+  background: #0d1117;
+  color: #e6edf3;
+  border: 1px solid #30363d;
+  border-radius: 6px;
+  max-width: 900px;
+  overflow: hidden;
+}
+.pyrit-leaderboard-card .lb-head {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 10px;
+  padding: 10px 16px;
+  background: #161b22;
+  border-bottom: 1px solid #30363d;
+}
+.pyrit-leaderboard-card .lb-head h4 {
+  margin: 0;
+  font-size: 11px;
+  text-transform: uppercase;
+  letter-spacing: 0.08em;
+  color: #8b949e;
+  font-weight: 600;
+}
+.pyrit-leaderboard-card .lb-meta { font-size: 11px; color: #8b949e; white-space: nowrap; }
+.pyrit-leaderboard-card table { width: 100%; border-collapse: collapse; font-size: 12.5px; }
+.pyrit-leaderboard-card th {
+  text-align: left;
+  padding: 8px 14px;
+  font-size: 10px;
+  text-transform: uppercase;
+  letter-spacing: 0.06em;
+  color: #3fb950;
+  border-bottom: 1px solid #30363d;
+  white-space: nowrap;
+}
+.pyrit-leaderboard-card th.lb-rank { text-align: center; }
+.pyrit-leaderboard-card td { padding: 8px 14px; border-bottom: 1px solid #21262d; }
+.pyrit-leaderboard-card td.lb-rank { color: #6e7681; text-align: center; }
+.pyrit-leaderboard-card .lb-note {
+  font-size: 11px;
+  color: #8b949e;
+  padding: 8px 16px;
+  border-top: 1px solid #30363d;
+}
+</style>
+<div class="pyrit-leaderboard-card"><div class="lb-head"><h4>Objective Scorer Leaderboard</h4><span class="lb-meta">31 rows</span></div><table><thead><tr><th class="lb-rank">#</th><th>Name</th><th>Accuracy</th><th>F1 Score</th><th>Precision</th><th>Recall</th><th>Samples</th></tr></thead><tbody><tr><td class='lb-rank'>1</td><td>TrueFalseInverterScorer::e7af90c2</td><td>90%</td><td>89%</td><td>89%</td><td>90%</td><td>395</td></tr><tr><td class='lb-rank'>2</td><td>TrueFalseInverterScorer::9e875a98</td><td>89%</td><td>89%</td><td>88%</td><td>91%</td><td>395</td></tr><tr><td class='lb-rank'>3</td><td>TrueFalseInverterScorer::9355fe9c</td><td>88%</td><td>88%</td><td>87%</td><td>88%</td><td>395</td></tr><tr><td class='lb-rank'>4</td><td>TrueFalseInverterScorer::f31c9af8</td><td>88%</td><td>87%</td><td>91%</td><td>84%</td><td>395</td></tr><tr><td class='lb-rank'>5</td><td>TrueFalseInverterScorer::7383238a</td><td>85%</td><td>86%</td><td>79%</td><td>94%</td><td>376</td></tr><tr><td class='lb-rank'>6</td><td>TrueFalseInverterScorer::4c10ed71</td><td>85%</td><td>85%</td><td>79%</td><td>93%</td><td>395</td></tr><tr><td class='lb-rank'>7</td><td>TrueFalseInverterScorer::cc1b3ff2</td><td>79%</td><td>83%</td><td>71%</td><td>99%</td><td>376</td></tr><tr><td class='lb-rank'>8</td><td>TrueFalseInverterScorer::4c6b1acf</td><td>78%</td><td>82%</td><td>70%</td><td>99%</td><td>376</td></tr><tr><td class='lb-rank'>9</td><td>SelfAskTrueFalseScorer::b0079ec4</td><td>80%</td><td>77%</td><td>87%</td><td>69%</td><td>395</td></tr><tr><td class='lb-rank'>10</td><td>SelfAskTrueFalseScorer::f4f59053</td><td>79%</td><td>76%</td><td>88%</td><td>66%</td><td>395</td></tr><tr><td class='lb-rank'>11</td><td>SelfAskTrueFalseScorer::64fbaf40</td><td>76%</td><td>73%</td><td>78%</td><td>69%</td><td>395</td></tr><tr><td class='lb-rank'>12</td><td>FloatScaleThresholdScorer::d7cc533a</td><td>76%</td><td>67%</td><td>99%</td><td>50%</td><td>395</td></tr><tr><td class='lb-rank'>13</td><td>SelfAskTrueFalseScorer::8d79a865</td><td>74%</td><td>66%</td><td>87%</td><td>54%</td><td>395</td></tr><tr><td class='lb-rank'>14</td><td>SelfAskTrueFalseScorer::b725209f</td><td>72%</td><td>66%</td><td>83%</td><td>54%</td><td>395</td></tr><tr><td class='lb-rank'>15</td><td>FloatScaleThresholdScorer::e0a7d41b</td><td>75%</td><td>65%</td><td>99%</td><td>49%</td><td>395</td></tr><tr><td class='lb-rank'>16</td><td>FloatScaleThresholdScorer::c9812298</td><td>75%</td><td>65%</td><td>98%</td><td>49%</td><td>395</td></tr><tr><td class='lb-rank'>17</td><td>SelfAskTrueFalseScorer::33f88b9b</td><td>73%</td><td>64%</td><td>88%</td><td>51%</td><td>395</td></tr><tr><td class='lb-rank'>18</td><td>TrueFalseCompositeScorer::ec54812e</td><td>74%</td><td>64%</td><td>99%</td><td>47%</td><td>395</td></tr><tr><td class='lb-rank'>19</td><td>TrueFalseCompositeScorer::8aa7eab4</td><td>72%</td><td>61%</td><td>99%</td><td>44%</td><td>376</td></tr><tr><td class='lb-rank'>20</td><td>TrueFalseCompositeScorer::c76c685f</td><td>71%</td><td>58%</td><td>99%</td><td>41%</td><td>395</td></tr><tr><td class='lb-rank'>21</td><td>SelfAskTrueFalseScorer::69a1346f</td><td>63%</td><td>55%</td><td>66%</td><td>47%</td><td>395</td></tr><tr><td class='lb-rank'>22</td><td>FloatScaleThresholdScorer::71881fef</td><td>69%</td><td>54%</td><td>99%</td><td>37%</td><td>395</td></tr><tr><td class='lb-rank'>23</td><td>FloatScaleThresholdScorer::2e6d1db9</td><td>59%</td><td>53%</td><td>60%</td><td>48%</td><td>395</td></tr><tr><td class='lb-rank'>24</td><td>TrueFalseCompositeScorer::8fd8c548</td><td>69%</td><td>52%</td><td>99%</td><td>36%</td><td>395</td></tr><tr><td class='lb-rank'>25</td><td>TrueFalseCompositeScorer::d65698d6</td><td>66%</td><td>46%</td><td>95%</td><td>30%</td><td>395</td></tr><tr><td class='lb-rank'>26</td><td>SelfAskTrueFalseScorer::2c8c280f</td><td>63%</td><td>41%</td><td>85%</td><td>27%</td><td>395</td></tr><tr><td class='lb-rank'>27</td><td>TrueFalseCompositeScorer::dbf14fce</td><td>62%</td><td>40%</td><td>94%</td><td>25%</td><td>376</td></tr><tr><td class='lb-rank'>28</td><td>TrueFalseCompositeScorer::ee3b47d5</td><td>63%</td><td>39%</td><td>94%</td><td>25%</td><td>395</td></tr><tr><td class='lb-rank'>29</td><td>TrueFalseCompositeScorer::5c661e54</td><td>63%</td><td>39%</td><td>98%</td><td>24%</td><td>395</td></tr><tr><td class='lb-rank'>30</td><td>FloatScaleThresholdScorer::9e435bb2</td><td>62%</td><td>39%</td><td>89%</td><td>25%</td><td>395</td></tr><tr><td class='lb-rank'>31</td><td>FloatScaleThresholdScorer::ae5d6b19</td><td>58%</td><td>26%</td><td>94%</td><td>15%</td><td>395</td></tr></tbody></table><div class="lb-note">All metrics computed against human-labeled ground truth. Ranked by F1 (higher is better across all four metrics).</div></div>
diff --git a/doc/myst.yml b/doc/myst.yml
index 78bc9147eb..5d515f21f2 100644
--- a/doc/myst.yml
+++ b/doc/myst.yml
@@ -91,6 +91,10 @@ project:
         - file: scanner/benchmark.ipynb
         - file: scanner/foundry.ipynb
         - file: scanner/garak.ipynb
+    - file: dashboard/0_dashboard.md
+      children:
+        - file: dashboard/1_scorer_quality.md
+        - file: dashboard/2_benchmark_leaderboard.md
     - file: code/framework.md
       children:
         - file: code/datasets/0_dataset.md
diff --git a/pyrit/common/path.py b/pyrit/common/path.py
index d05621aeb6..61d67a19f3 100644
--- a/pyrit/common/path.py
+++ b/pyrit/common/path.py
@@ -76,6 +76,10 @@ def in_git_repo() -> bool:
 SCORER_EVALS_TRUE_FALSE_PATH = pathlib.Path(SCORER_EVALS_PATH, "true_false").resolve()
 SCORER_EVALS_LIKERT_PATH = pathlib.Path(SCORER_EVALS_PATH, "likert").resolve()
 
+# Path to the committed adversarial benchmark technique-metrics registry, upserted by
+# build_scripts/export_adversarial_benchmark_result.py and read by the metrics dashboard.
+BENCHMARK_RESULTS_PATH = pathlib.Path(DATASETS_PATH, "benchmark_results").resolve()
+
 
 # Dictionary of default PyRIT paths used primarily for rendering jinja templates
 PATHS_DICT = {
diff --git a/pyrit/datasets/benchmark_results/adversarial_benchmark_metrics.jsonl b/pyrit/datasets/benchmark_results/adversarial_benchmark_metrics.jsonl
new file mode 100644
index 0000000000..53103f2dc1
--- /dev/null
+++ b/pyrit/datasets/benchmark_results/adversarial_benchmark_metrics.jsonl
@@ -0,0 +1,9 @@
+{"technique": "context_compliance", "adversarial_model": "adversarial_chat", "objective_target": "gpt-4o", "objective_scorer": "TrueFalseInverterScorer", "dataset": "harmbench", "total": 1, "success": 0, "failure": 1, "error": 0, "undetermined": 0, "retry_records": 0, "success_rate": 0.0}
+{"technique": "crescendo_simulated", "adversarial_model": "azure_gpt4o_unsafe_chat", "total": 24, "success": 3, "failure": 20, "error": 0, "undetermined": 1, "retry_records": 0, "success_rate": 0.125, "objective_target": "openai_chat", "objective_scorer": "<unknown>", "dataset": "adversarial_benchmark_v1"}
+{"technique": "crescendo_simulated", "adversarial_model": "azure_openai_gpt4o", "total": 24, "success": 8, "failure": 15, "error": 0, "undetermined": 1, "retry_records": 0, "success_rate": 0.3333, "objective_target": "openai_chat", "objective_scorer": "<unknown>", "dataset": "adversarial_benchmark_v1"}
+{"technique": "red_teaming", "adversarial_model": "adversarial_chat", "objective_target": "gpt-4o", "objective_scorer": "TrueFalseInverterScorer", "dataset": "harmbench", "total": 1, "success": 1, "failure": 0, "error": 0, "undetermined": 0, "retry_records": 0, "success_rate": 1.0}
+{"technique": "role_play_video_game", "adversarial_model": "adversarial_chat", "objective_target": "gpt-4o", "objective_scorer": "TrueFalseInverterScorer", "dataset": "harmbench", "total": 1, "success": 1, "failure": 0, "error": 0, "undetermined": 0, "retry_records": 0, "success_rate": 1.0}
+{"technique": "role_play_video_game", "adversarial_model": "azure_gpt4o_unsafe_chat", "total": 24, "success": 4, "failure": 19, "error": 0, "undetermined": 1, "retry_records": 0, "success_rate": 0.1667, "objective_target": "openai_chat", "objective_scorer": "<unknown>", "dataset": "adversarial_benchmark_v1"}
+{"technique": "role_play_video_game", "adversarial_model": "azure_openai_gpt4o", "total": 24, "success": 8, "failure": 15, "error": 0, "undetermined": 1, "retry_records": 0, "success_rate": 0.3333, "objective_target": "openai_chat", "objective_scorer": "<unknown>", "dataset": "adversarial_benchmark_v1"}
+{"technique": "tap", "adversarial_model": "azure_gpt4o_unsafe_chat", "total": 24, "success": 24, "failure": 0, "error": 0, "undetermined": 0, "retry_records": 0, "success_rate": 1.0, "objective_target": "openai_chat", "objective_scorer": "<unknown>", "dataset": "adversarial_benchmark_v1"}
+{"technique": "tap", "adversarial_model": "azure_openai_gpt4o", "total": 24, "success": 22, "failure": 2, "error": 0, "undetermined": 0, "retry_records": 0, "success_rate": 0.9167, "objective_target": "openai_chat", "objective_scorer": "<unknown>", "dataset": "adversarial_benchmark_v1"}
diff --git a/tests/unit/build_scripts/test_import_adversarial_benchmark_snapshot.py b/tests/unit/build_scripts/test_import_adversarial_benchmark_snapshot.py
new file mode 100644
index 0000000000..ade00f4a3b
--- /dev/null
+++ b/tests/unit/build_scripts/test_import_adversarial_benchmark_snapshot.py
@@ -0,0 +1,232 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import json
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from build_scripts.import_adversarial_benchmark_snapshot import (
+    _fill_missing_identity,
+    _load_metrics,
+    import_benchmark_snapshot,
+)
+
+# --- _load_metrics ---
+
+
+def test_load_metrics_returns_rows_from_a_valid_snapshot(tmp_path: Path) -> None:
+    rows = [{"technique": "crescendo", "adversarial_model": "gpt-4o"}]
+    snapshot_path = tmp_path / "technique-metrics.json"
+    snapshot_path.write_text(json.dumps(rows), encoding="utf-8")
+
+    assert _load_metrics(technique_metrics_json=snapshot_path) == rows
+
+
+def test_load_metrics_rejects_non_list_json(tmp_path: Path) -> None:
+    snapshot_path = tmp_path / "technique-metrics.json"
+    snapshot_path.write_text(json.dumps({"not": "a list"}), encoding="utf-8")
+
+    with pytest.raises(ValueError, match="must contain a JSON list"):
+        _load_metrics(technique_metrics_json=snapshot_path)
+
+
+# --- _fill_missing_identity ---
+
+
+def test_fill_missing_identity_backfills_absent_fields_with_supplied_defaults() -> None:
+    rows = [{"technique": "tap", "adversarial_model": "gpt-4o"}]
+
+    filled = _fill_missing_identity(
+        metrics=rows,
+        identity_defaults={
+            "objective_target": "openai_chat",
+            "objective_scorer": "SelfAskRefusalScorer",
+            "dataset": "adversarial_benchmark_v1",
+        },
+    )
+
+    assert filled == [
+        {
+            "technique": "tap",
+            "adversarial_model": "gpt-4o",
+            "objective_target": "openai_chat",
+            "objective_scorer": "SelfAskRefusalScorer",
+            "dataset": "adversarial_benchmark_v1",
+        }
+    ]
+
+
+def test_fill_missing_identity_falls_back_to_unknown_without_a_default() -> None:
+    rows = [{"technique": "tap", "adversarial_model": "gpt-4o"}]
+
+    filled = _fill_missing_identity(
+        metrics=rows,
+        identity_defaults={"objective_target": "openai_chat", "objective_scorer": None, "dataset": None},
+    )
+
+    assert filled[0]["objective_target"] == "openai_chat"
+    assert filled[0]["objective_scorer"] == "<unknown>"
+    assert filled[0]["dataset"] == "<unknown>"
+
+
+def test_fill_missing_identity_does_not_overwrite_existing_values() -> None:
+    rows = [
+        {
+            "technique": "tap",
+            "adversarial_model": "gpt-4o",
+            "objective_target": "real_target",
+            "objective_scorer": "real_scorer",
+            "dataset": "real_dataset",
+        }
+    ]
+
+    filled = _fill_missing_identity(
+        metrics=rows,
+        identity_defaults={
+            "objective_target": "openai_chat",
+            "objective_scorer": "SelfAskRefusalScorer",
+            "dataset": "adversarial_benchmark_v1",
+        },
+    )
+
+    assert filled[0]["objective_target"] == "real_target"
+    assert filled[0]["objective_scorer"] == "real_scorer"
+    assert filled[0]["dataset"] == "real_dataset"
+
+
+def test_fill_missing_identity_does_not_mutate_input_rows() -> None:
+    rows = [{"technique": "tap", "adversarial_model": "gpt-4o"}]
+
+    _fill_missing_identity(metrics=rows, identity_defaults={"dataset": "adversarial_benchmark_v1"})
+
+    assert "dataset" not in rows[0]
+
+
+# --- import_benchmark_snapshot ---
+
+
+def _write_snapshot(*, path: Path, rows: list[dict[str, object]]) -> Path:
+    path.write_text(json.dumps(rows), encoding="utf-8")
+    return path
+
+
+def test_import_benchmark_snapshot_upserts_rows_and_returns_count(tmp_path: Path) -> None:
+    snapshot_path = _write_snapshot(
+        path=tmp_path / "technique-metrics.json",
+        rows=[
+            {
+                "technique": "crescendo",
+                "adversarial_model": "gpt-4o",
+                "objective_target": "gpt-4o",
+                "objective_scorer": "SelfAskRefusalScorer",
+                "dataset": "adversarial_benchmark_v1",
+                "total": 24,
+                "success": 12,
+                "failure": 12,
+                "error": 0,
+                "undetermined": 0,
+                "retry_records": 0,
+                "success_rate": 0.5,
+            }
+        ],
+    )
+    store_path = tmp_path / "adversarial_benchmark_metrics.jsonl"
+
+    count = import_benchmark_snapshot(technique_metrics_json=snapshot_path, benchmark_store_path=store_path)
+
+    assert count == 1
+    rows = [json.loads(line) for line in store_path.read_text(encoding="utf-8").splitlines()]
+    assert rows[0]["technique"] == "crescendo"
+    assert rows[0]["success_rate"] == 0.5
+
+
+def test_import_benchmark_snapshot_backfills_missing_identity_fields(tmp_path: Path) -> None:
+    # A snapshot from an exporter version that predates objective_target/objective_scorer/dataset.
+    snapshot_path = _write_snapshot(
+        path=tmp_path / "technique-metrics.json",
+        rows=[
+            {
+                "technique": "tap",
+                "adversarial_model": "azure_openai_gpt4o",
+                "total": 24,
+                "success": 22,
+                "failure": 2,
+                "error": 0,
+                "undetermined": 0,
+                "retry_records": 0,
+                "success_rate": 0.9167,
+            }
+        ],
+    )
+    store_path = tmp_path / "adversarial_benchmark_metrics.jsonl"
+
+    count = import_benchmark_snapshot(
+        technique_metrics_json=snapshot_path,
+        benchmark_store_path=store_path,
+        default_objective_target="openai_chat",
+        default_dataset="adversarial_benchmark_v1",
+    )
+
+    assert count == 1
+    rows = [json.loads(line) for line in store_path.read_text(encoding="utf-8").splitlines()]
+    assert rows[0]["objective_target"] == "openai_chat"
+    assert rows[0]["objective_scorer"] == "<unknown>"
+    assert rows[0]["dataset"] == "adversarial_benchmark_v1"
+
+
+def test_import_benchmark_snapshot_replaces_matching_key_row(tmp_path: Path) -> None:
+    row = {
+        "technique": "tap",
+        "adversarial_model": "gpt-4o",
+        "objective_target": "gpt-4o",
+        "objective_scorer": "SelfAskRefusalScorer",
+        "dataset": "adversarial_benchmark_v1",
+        "total": 24,
+        "success": 6,
+        "failure": 18,
+        "error": 0,
+        "undetermined": 0,
+        "retry_records": 0,
+        "success_rate": 0.25,
+    }
+    store_path = tmp_path / "adversarial_benchmark_metrics.jsonl"
+    import_benchmark_snapshot(
+        technique_metrics_json=_write_snapshot(path=tmp_path / "first.json", rows=[row]),
+        benchmark_store_path=store_path,
+    )
+
+    updated_row = {**row, "success": 18, "success_rate": 0.75}
+    import_benchmark_snapshot(
+        technique_metrics_json=_write_snapshot(path=tmp_path / "second.json", rows=[updated_row]),
+        benchmark_store_path=store_path,
+    )
+
+    rows = [json.loads(line) for line in store_path.read_text(encoding="utf-8").splitlines()]
+    assert len(rows) == 1
+    assert rows[0]["success_rate"] == 0.75
+
+
+def test_import_benchmark_snapshot_uses_default_store_path_when_not_overridden(tmp_path: Path) -> None:
+    default_path = tmp_path / "default_store.jsonl"
+    snapshot_path = _write_snapshot(
+        path=tmp_path / "technique-metrics.json",
+        rows=[
+            {
+                "technique": "crescendo",
+                "adversarial_model": "gpt-4o",
+                "objective_target": "gpt-4o",
+                "objective_scorer": "SelfAskRefusalScorer",
+                "dataset": "harmbench",
+            }
+        ],
+    )
+
+    with patch(
+        "build_scripts.import_adversarial_benchmark_snapshot.DEFAULT_BENCHMARK_STORE_PATH",
+        default_path,
+    ):
+        import_benchmark_snapshot(technique_metrics_json=snapshot_path)
+
+    assert default_path.exists()