diff --git a/core/src/autonomous/report/html.ts b/core/src/autonomous/report/html.ts index d2ff488..347d399 100644 --- a/core/src/autonomous/report/html.ts +++ b/core/src/autonomous/report/html.ts @@ -78,7 +78,7 @@ function renderTranscript(f: ReportFinding, tId: string): string { ? `
${f.turns .map((t) => { const bad = failing.has(t.turnIndex); - return ``; + return ``; }) .join( "" @@ -93,7 +93,7 @@ function renderTranscript(f: ReportFinding, tId: string): string {
Conversation Transcript ${f.turns.length} turn${f.turns.length === 1 ? "" : "s"}
-
+
${rail}
${turns}
@@ -150,7 +150,7 @@ function renderFindingCard(f: ReportFinding, index: number): string { const metaRow = failed || standardsLabel ? `
- ${failed ? `
${f.confidence}%
` : ""} + ${failed ? `
${f.confidence}%
` : ""} ${standardsLabel ? `
${esc(standardsLabel)}
` : ""}
` : ""; @@ -163,7 +163,7 @@ function renderFindingCard(f: ReportFinding, index: number): string {
${esc(f.name || f.vulnClassId)} | - ${SEVERITY_ICON}${esc(f.severity.toUpperCase())} + ${SEVERITY_ICON}${esc(f.severity.toUpperCase())}i
The vuln-class's category weight, not this finding's outcome. A LOW-severity class can still be confirmed.
@@ -243,15 +243,31 @@ export function renderReportHtml(r: AutonomousReport): string { const low = sevCount("low"); const vulnerable = r.summary.confirmed > 0; - const verdict = vulnerable ? "Vulnerable" : "Defended"; + const verdict = vulnerable ? "Fail" : "Pass"; const risk = crit > 0 - ? { label: "Critical Risk", color: "#991B1B" } + ? { + label: "Critical Risk", + color: "#991B1B", + explain: "Critical Risk = at least one confirmed critical-severity finding.", + } : high > 0 - ? { label: "High Risk", color: "#DC2626" } + ? { + label: "High Risk", + color: "#DC2626", + explain: "High Risk = at least one confirmed high-severity finding, none critical.", + } : vulnerable - ? { label: "Medium Risk", color: "#D97706" } - : { label: "Low Risk", color: "#059669" }; + ? { + label: "Medium Risk", + color: "#D97706", + explain: "Medium Risk = confirmed findings exist, none high or critical severity.", + } + : { + label: "Low Risk", + color: "#059669", + explain: "Low Risk = no confirmed vulnerabilities this run.", + }; // Safety score mirrors the run report's 0-100 scale: the inverse of attack success. // N/A when nothing was conclusively scored (no confirmed and no defended threads). @@ -324,9 +340,7 @@ export function renderReportHtml(r: AutonomousReport): string { const narrative = r.synthesisComplete ? esc(r.executiveNarrative) - : `Assessment of ${esc(r.target.name)}: ${r.summary.confirmed} confirmed vulnerabilit${r.summary.confirmed === 1 ? "y" : "ies"} (${crit} critical, ${high} high) across ${threadCount} attack thread${threadCount === 1 ? "" : "s"} — ${r.summary.attackSuccessRate}% attack-success rate.${r.truncated ? ` Run truncated: ${esc(r.truncationReason ?? "")}.` : ""}`; - - const outcomeLabel = r.objectiveOutcome.replace(/-/g, " "); + : `Assessment of ${esc(r.target.name)}: ${r.summary.confirmed} confirmed vulnerabilit${r.summary.confirmed === 1 ? "y" : "ies"} (${crit} critical, ${high} high) across ${threadCount} attack thread${threadCount === 1 ? "" : "s"}.${r.truncated ? ` Run truncated: ${esc(r.truncationReason ?? "")}.` : ""}`; // ── Section numbering + nav (both sections and links are conditional) ── let sectionNo = 0; @@ -346,6 +360,11 @@ export function renderReportHtml(r: AutonomousReport): string { if (classesNo) link("classes", "Categories"); const findingsNo = num(); link("findings", "Findings"); + // Guardrails/weak-points are the commander's closing read on the WHOLE engagement (written + // in the same submit_report call as the executive narrative), not something recon itself + // produces, hence living after Findings rather than inside the Reconnaissance section. + const guardrailsNo = r.recon.guardrails.length > 0 || r.recon.weakPoints.length > 0 ? num() : 0; + if (guardrailsNo) link("guardrails", "Guardrails"); const treeNo = r.findings.length > 0 ? num() : 0; if (treeNo) link("tree", "Attack Tree"); const recsNo = r.recommendations.length > 0 ? num() : 0; @@ -416,7 +435,6 @@ export function renderReportHtml(r: AutonomousReport): string { .cover-inner{max-width:1080px;margin:0 auto;padding:42px 28px 38px} .cover-top{display:flex;align-items:flex-start;justify-content:space-between;gap:28px;margin-bottom:22px} .cover-title{font-size:30px;font-weight:700;color:#fff;letter-spacing:-0.01em} - .cover-subtitle{font-size:14.5px;color:#A8B2C1;margin-top:7px;max-width:760px;line-height:1.65} .cover-badges-row{display:flex;align-items:center;gap:12px;margin-bottom:24px;flex-wrap:wrap} .badge-confidential{display:inline-flex;align-items:center;gap:6px;background:var(--accent);color:#fff;padding:5px 12px;border-radius:6px;font-size:11.5px;font-weight:700;letter-spacing:0.08em;text-transform:uppercase} .badge-mode{display:inline-flex;align-items:center;gap:6px;background:#262626;border:1px solid #3a3a3a;color:#E2E8F0;padding:5px 12px;border-radius:6px;font-size:11.5px;font-weight:700;letter-spacing:0.06em;text-transform:uppercase} @@ -425,7 +443,7 @@ export function renderReportHtml(r: AutonomousReport): string { .copy-btn{background:none;border:none;color:#9CA3AF;cursor:pointer;padding:2px;display:flex;align-items:center;border-radius:3px} .copy-btn:hover{color:#fff;background:rgba(255,255,255,0.1)} .cover-date{font-size:12.5px;color:#9CA3AF} - .cover-meta{display:grid;grid-template-columns:repeat(3,1fr);border:1px solid rgba(255,255,255,0.08);border-radius:10px;overflow:hidden} + .cover-meta{display:grid;grid-template-columns:repeat(4,1fr);border:1px solid rgba(255,255,255,0.08);border-radius:10px;overflow:hidden} .cover-meta-item{padding:17px 20px;border-right:1px solid rgba(255,255,255,0.08)} .cover-meta-item:last-child{border-right:none} .cover-meta-k{font-size:11.5px;color:#9CA3AF;text-transform:uppercase;letter-spacing:0.08em;margin-bottom:4px} @@ -446,26 +464,63 @@ export function renderReportHtml(r: AutonomousReport): string { .section-subtitle{font-size:12.5px;color:var(--muted);margin-left:auto;text-align:right} /* ── Executive summary strip ── */ - .exec-strip{display:flex;align-items:stretch;border:1px solid var(--line);border-radius:12px;background:var(--surface);overflow:hidden;margin-bottom:12px} - /* flex-start, not center: the four cards hold different amounts of content, and centering - each one vertically would land every label at a different height. */ - .exec-strip-item{flex:1;padding:22px 24px;border-right:1px solid var(--line);display:flex;flex-direction:column;justify-content:flex-start;min-width:0} - .exec-strip-item:last-child{border-right:none} - .exec-strip-label{font-size:11.5px;font-weight:600;letter-spacing:0.08em;text-transform:uppercase;color:var(--muted);margin-bottom:10px} - .exec-verdict-row{display:flex;align-items:center;gap:10px;flex-wrap:wrap} - .exec-verdict-text{font-size:31px;font-weight:800;letter-spacing:0.02em;line-height:1} + /* No overflow:hidden here (unlike a plain rounded card), since the info-tooltip below needs + to pop outside this row's box on hover, so the rounded corners are faked on the end cells + (:first-child/:last-child) instead of clipped at the row level. */ + .exec-strip{display:flex;align-items:stretch;border:1px solid var(--line);border-radius:12px;background:var(--surface);margin-bottom:12px} + /* flex-start by default: short cards (Testing Cost, Duration) sit right under their + label, since pinning them to the bottom of a taller row leaves a dead gap in the middle. + Cards with genuinely variable content (a two-line wrapped badge vs. a fixed-height gauge + vs. value+dots) opt into .exec-strip-item--bottom, which uses space-between so label + stays top and body stays bottom, so those cards line up with each other since align-items: + stretch on .exec-strip already gives every card the same height (the tallest card's). + Horizontally, align-items centers each card's own content since label/body shrink to + their content width instead of stretching to the card's full width. */ + .exec-strip-item{position:relative;flex:1;padding:18px 22px;border-right:1px solid var(--line);display:flex;flex-direction:column;align-items:center;justify-content:flex-start;min-width:0;text-align:center} + .exec-strip-item.exec-strip-item--bottom{justify-content:space-between} + .exec-strip-item:first-child{border-radius:12px 0 0 12px} + .exec-strip-item:last-child{border-right:none;border-radius:0 12px 12px 0} + .exec-strip-label{font-size:11px;font-weight:600;letter-spacing:0.08em;text-transform:uppercase;color:var(--muted);margin-bottom:8px;display:flex;align-items:center;justify-content:center;gap:5px} + + /* ── Info tooltip: appears the instant the pointer enters the card (no native-title delay), + stays open for as long as the pointer is anywhere on the card, not just on the "i" icon. + Hover is bound to the whole .exec-strip-item; the icon is just the visual affordance. ── */ + .info-icon{display:inline-flex;align-items:center;justify-content:center;width:13px;height:13px;border-radius:50%;border:1.3px solid var(--muted-2);color:var(--muted-2);font-size:9px;font-weight:700;font-style:italic;font-family:Georgia,"Times New Roman",serif;cursor:default;flex-shrink:0} + .exec-strip-item:hover .info-icon{border-color:var(--text);color:var(--text)} + .info-tooltip{position:absolute;top:100%;left:50%;transform:translateX(-50%) translateY(4px);margin-top:8px;width:max-content;max-width:230px;background:#0F172A;color:#E2E8F0;font-size:12px;font-weight:400;text-align:left;line-height:1.45;letter-spacing:normal;text-transform:none;padding:7px 11px;border-radius:7px;box-shadow:0 8px 24px rgba(15,23,42,0.25);opacity:0;visibility:hidden;pointer-events:none;transition:opacity .12s ease;z-index:20} + .exec-strip-item:hover .info-tooltip{opacity:1;visibility:visible} + /* Base hover for a standalone .info-hover (a single word/icon, not a whole card) — the + scope-row rule further down widens the hoverable area for rows specifically, but this is + what makes .info-hover work on its own wherever there's no row to widen the hover to. */ + .info-hover:hover .info-icon{border-color:var(--text);color:var(--text)} + .info-hover:hover .info-tooltip{opacity:1;visibility:visible} + .tooltip-divider{height:1px;background:rgba(255,255,255,0.15);margin:6px 0} + .exec-strip-body{display:flex;flex-direction:column;align-items:center} + /* Cards without --bottom don't stretch their body to the row's bottom edge, so nudge the + value down a little instead of letting it sit flush under the label: a small step + toward vertical center without opening the dead gap full space-between would leave. */ + .exec-strip-item:not(.exec-strip-item--bottom) .exec-strip-body{margin-top:14px} + /* translateY, not a margin change: the box edges here are already pixel-identical to the + gauge/value box next to it (verified with getBoundingClientRect), so this isn't a layout + bug. Bold numerals like "80%" have no descenders, so their visible ink sits a few px above + their own invisible box edge, while the LOW RISK pill's border IS its true edge, so the two + look misaligned even though their boxes match. A transform nudges pixels only, leaving the + box math (and the space-between bottom-pinning) untouched. */ + .exec-verdict-row{display:flex;flex-direction:column;align-items:center;justify-content:center;gap:8px;transform:translateY(-5px)} + .exec-verdict-text{font-size:32px;font-weight:800;letter-spacing:0.02em;line-height:1} .exec-verdict-text.pass{color:var(--pass)} .exec-verdict-text.fail{color:var(--fail)} - .exec-risk{font-size:11.5px;font-weight:600;padding:5px 12px;border-radius:999px;border:1px solid;white-space:nowrap;cursor:default} - /* Width matches the gauge so the value centres under the arc while the block itself stays - left-aligned with the label, like every other card in the strip. */ - .gauge-value{font-size:24px;font-weight:800;color:var(--text);width:120px;text-align:center;margin-top:-30px} - .sc-value{font-size:29px;font-weight:800;line-height:1;color:var(--text)} - .sc-dots{display:flex;flex-direction:column;gap:3px;margin-top:8px} - .sc-dot-row{display:flex;align-items:center;gap:7px;font-size:12.5px;color:var(--muted)} + .exec-risk{font-size:11px;font-weight:600;padding:4px 11px;border-radius:999px;border:1px solid;white-space:nowrap;cursor:default} + .gauge-value{font-size:26px;font-weight:800;color:var(--text);width:120px;text-align:center;margin-top:-30px} + .sc-value{font-size:26px;font-weight:800;line-height:1;color:var(--text)} + .sc-dots{display:flex;flex-direction:column;align-items:center;gap:3px;margin-top:8px} + .sc-dot-row{display:flex;align-items:center;justify-content:center;gap:6px;font-size:12px;color:var(--muted)} .sc-dot{width:7px;height:7px;border-radius:50%;flex-shrink:0} - .sc-sub{font-size:12.5px;color:var(--muted);margin-top:5px} + .sc-sub{font-size:12px;color:var(--muted);margin-top:4px} .summary-narrative{font-size:14.5px;color:var(--text-2);line-height:1.78;padding:6px 2px} + .goal-callout{margin-top:14px;padding-top:14px;border-top:1px solid var(--line)} + .goal-callout-label{font-size:11px;font-weight:600;color:var(--muted);text-transform:uppercase;letter-spacing:0.06em;margin-bottom:4px} + .goal-callout-text{font-size:14px;color:var(--text-2);line-height:1.65} .summary-narrative strong{color:var(--text)} /* ── Severity distribution ── */ @@ -487,9 +542,20 @@ export function renderReportHtml(r: AutonomousReport): string { .scope-v.mono{font-size:12px} .scope-full{grid-column:1/-1} .scope-text{font-size:14px;color:var(--text-2);line-height:1.72} - .agent-role{display:block;font-size:11px;color:var(--muted-2);font-weight:400;margin-top:1px} + /* .info-hover is just the positioning anchor for its .info-tooltip (absolute position + needs a positioned ancestor); the actual hover trigger is the whole .scope-row, same + "hover anywhere on the row" behavior as .exec-strip-item above. */ + .info-hover{position:relative;display:inline-flex;align-items:center;gap:4px;cursor:default} + .info-hover .info-icon{margin-left:3px} + .scope-row:hover .info-icon{border-color:var(--text);color:var(--text)} + .scope-row:hover .info-tooltip{opacity:1;visibility:visible} .chips{display:flex;flex-wrap:wrap;gap:7px} .chip{font-size:12px;padding:4px 11px;border-radius:999px;border:1px solid var(--line-2);background:var(--surface-2);color:var(--text-2)} + /* Full-sentence notes (guardrails, weak points), unlike .chip's short single-word tags: + a pill radius squeezes wrapped multi-line text against its rounded ends, so these get a + softer rounded-rectangle shape and roomier padding instead, one per row. */ + .note-list{display:flex;flex-direction:column;gap:8px} + .note-chip{font-size:12.5px;line-height:1.5;padding:10px 14px;border-radius:8px;border:1px solid var(--line-2);background:var(--surface-2);color:var(--text-2)} /* ── Badges ── */ .eval-sep{color:var(--line-2);font-weight:400} @@ -539,7 +605,13 @@ export function renderReportHtml(r: AutonomousReport): string { .corr-badge{font-size:11.5px;color:var(--pass);background:var(--pass-bg);border:1px solid var(--pass-border);border-radius:5px;padding:3px 10px;font-weight:600} /* ── Transcript ── */ - .transcript-toggle{display:inline-flex;align-items:center;gap:7px;font-size:13.5px;font-weight:600;color:var(--muted);background:none;border:none;cursor:pointer;padding:15px 0 0;margin-top:2px;border-top:1px solid var(--line);width:100%} + /* display:flex, not inline-flex: this button is always the LAST child of .eval-body, and an + inline-level box (inline-flex) sits inside an anonymous line box with its own font-metric + "strut" height, which silently breaks any margin-bottom math on it (verified: doubling the + negative margin had zero visible effect). A block-level flex container behaves as expected, + so margin-bottom below cancels exactly the amount needed to match the 15px gap above (the + padding-top between the divider line and the text) against the card's 22px bottom padding. */ + .transcript-toggle{display:flex;align-items:center;gap:7px;font-size:13.5px;font-weight:600;color:var(--muted);background:none;border:none;cursor:pointer;padding:15px 0 0;margin-top:2px;margin-bottom:-7px;border-top:1px solid var(--line);width:100%} .transcript-toggle:hover{color:var(--text)} .transcript-toggle svg{transition:transform 0.2s} .transcript-wrap{display:none;margin-bottom:8px} @@ -547,7 +619,12 @@ export function renderReportHtml(r: AutonomousReport): string { .transcript{border:1px solid var(--line);border-radius:8px;overflow:hidden} .transcript-header{padding:11px 15px;background:var(--surface-2);font-size:11.5px;font-weight:600;text-transform:uppercase;letter-spacing:0.06em;color:var(--muted);border-bottom:1px solid var(--line);display:flex;align-items:center;gap:8px} .tc-count{font-weight:400;color:var(--muted-2)} + /* No left padding by default: the turn-rail's own left inset (below) already gives turn + content its left margin, via the rail column + this gap. A single-turn thread skips the + rail entirely (not useful for one turn), so .no-rail restores that margin directly instead + of leaving the text flush against the card edge. */ .transcript-body{position:relative;display:flex;align-items:flex-start;gap:18px;padding:15px 15px 15px 0;max-height:620px;overflow-y:auto;overscroll-behavior:contain} + .transcript-body.no-rail{padding-left:14px} .turn-rail-wrap{position:sticky;top:4px;flex-shrink:0;align-self:flex-start} .turn-rail{display:flex;flex-direction:column;align-items:center;gap:17px;padding:4px 0 4px 14px;max-height:574px;overflow-y:auto;scrollbar-width:none;-ms-overflow-style:none} .turn-rail::-webkit-scrollbar{display:none} @@ -607,11 +684,6 @@ export function renderReportHtml(r: AutonomousReport): string { .decision-fork{color:#7c3aed}.decision-dispatch{color:#2563eb}.decision-stop{color:var(--fail)}.decision-pivot{color:#d97706}.decision-continue{color:var(--muted)} .no-findings{background:var(--pass-bg);border:1px solid var(--pass-border);border-radius:10px;padding:20px;text-align:center;color:var(--pass);font-weight:600;font-size:14px} - /* ── Footer ── */ - .report-footer{max-width:1080px;margin:48px auto 0;padding:20px 28px;border-top:1px solid var(--line);display:flex;justify-content:space-between;align-items:center} - .footer-left{font-size:12.5px;color:var(--muted)} - .footer-right{font-size:12.5px;color:var(--muted-2);font-family:ui-monospace,monospace} - @media print{ body{background:#fff;padding:0} .nav{display:none} @@ -642,7 +714,6 @@ export function renderReportHtml(r: AutonomousReport): string {
Autonomous Hunt Assessment Report
-
${esc(r.objective)}
${OPFOR_LOGO_SVG}
@@ -652,12 +723,13 @@ export function renderReportHtml(r: AutonomousReport): string { Report ID: ${esc(r.reportId.slice(0, 13))} - ${esc(dateStr)}, ${esc(timeStr)} + ${esc(dateStr)}, ${esc(timeStr)}${durationMs !== undefined ? ` · Ran for: ${formatDuration(durationMs)}` : ""}
Target System
${esc(truncate(r.target.name, 40))}
-
Objective Outcome
${esc(outcomeLabel)}
-
Run Cost
${r.totalCostUsd !== undefined ? "$" + r.totalCostUsd.toFixed(2) : "—"}
+
Commander Model
${esc(r.commanderModel || "—")}
+
Operator Model
${esc(r.operatorModel || "—")}
+
Scout Model
${esc(r.scoutModel || "—")}
@@ -675,37 +747,70 @@ export function renderReportHtml(r: AutonomousReport): string {
Executive Summary
-
-
Overall Verdict
-
-
${verdict}
-
${risk.label.toUpperCase()}
+
+
Overall Verdicti +
Fail = at least one attack thread ended in a confirmed vulnerability.
${risk.explain}
+
+
+
+
${verdict}
+
${risk.label.toUpperCase()}
+
-
Objective ${esc(outcomeLabel)}${r.truncated ? " · run truncated" : ""}
-
-
-
Safety Score
- ${gaugeSvg(safetyScore ?? 0, safetyScore === null ? "#94A3B8" : safetyColor(safetyScore))} -
${safetyScore === null ? "N/A" : `${safetyScore}%`}
-
-
Findings
-
${r.findings.length}
-
-
${r.summary.confirmed} confirmed
-
${r.summary.defended} defended
- ${r.summary.errors > 0 ? `
${r.summary.errors} errored
` : ""} +
+
Safety Scorei +
Share of scoreable findings that defended successfully.
+
+
+ ${gaugeSvg(safetyScore ?? 0, safetyScore === null ? "#94A3B8" : safetyColor(safetyScore))} +
${safetyScore === null ? "N/A" : `${safetyScore}%`}
-
across ${threadCount} attack thread${threadCount === 1 ? "" : "s"}
-
-
Attack Success
-
${r.summary.attackSuccessRate}%
-
${durationMs !== undefined ? `${formatDuration(durationMs)} wall clock` : `${classes.length} class${classes.length === 1 ? "" : "es"} tested`}
+
+
Findingsi +
One entry per finding: confirmed means a real vulnerability, defended means the target held.
+
+
+
${r.findings.length}
+
+
${r.summary.confirmed} confirmed
+
${r.summary.defended} defended
+ ${r.summary.errors > 0 ? `
${r.summary.errors} errored
` : ""} +
+
+ ${ + r.totalCostUsd !== undefined + ? `
+
Testing Costi +
Estimated LLM cost across the commander, operator, and scout this run.
+
+
+
$${r.totalCostUsd.toFixed(2)}
+
+
` + : "" + } + ${ + durationMs !== undefined + ? `
+
Durationi +
Wall-clock time from the run's start to the final report.
+
+
+
${formatDuration(durationMs)}
+
+
` + : "" + }
${vulnerable && sevSeg ? `
${sevSeg}
${sevLegend}
` : ""}
${narrative}
+
+
Objective
+
${esc(r.objective)}
+
@@ -723,17 +828,21 @@ export function renderReportHtml(r: AutonomousReport): string {
Attack Agents
-
Commander plans & synthesizes${esc(r.commanderModel || "—")}
-
Operator runs attack threads${esc(r.operatorModel || "—")}
-
Scout recon & lead triage${esc(r.scoutModel || "—")}
-
Verifier independent self-check${esc(r.verifierModel || r.commanderModel || "—")}
+
Commanderi
Plans the attack strategy and writes the final synthesis.
${esc(r.commanderModel || "—")}
+
Operatori
Runs the actual attack threads against the target.
${esc(r.operatorModel || "—")}
+
Scouti
Handles recon probing and triages promising leads.
${esc(r.scoutModel || "—")}
+ ${ + r.verifierModel + ? `
Verifieri
Independently double-checks high-severity findings, only when --verify is enabled.
${esc(r.verifierModel)}
` + : "" + }
Exploration
-
Attack Threads${threadCount}
-
Leads Flagged${r.exploration.leadsFlagged}
-
Leads Expanded / Dismissed${r.exploration.leadsSpawned} / ${r.exploration.leadsDismissed}
-
Max Depth Reached${r.exploration.maxDepthReached}${r.exploration.maxDepthReached === 0 ? " (root wave only)" : ""}
+
Attack Threadsi
Independent attack conversations the operators ran against the target.
${threadCount}
+
Leads Flaggedi
Promising but unfinished seams operators noticed mid-attack, queued for the commander to review.
${r.exploration.leadsFlagged}
+
Leads Expanded / Dismissedi
Flagged leads the commander sent to a follow-up attack, versus ones it dropped as too weak.
${r.exploration.leadsSpawned} / ${r.exploration.leadsDismissed}
+
Max Depth Reachedi
How many follow-up waves deep a lead got chased. 0 means only the first wave ran.
${r.exploration.maxDepthReached}${r.exploration.maxDepthReached === 0 ? " (root wave only)" : ""}
Run
@@ -751,20 +860,10 @@ export function renderReportHtml(r: AutonomousReport): string {
Reconnaissance
${r.recon.probeCount} benign probe${r.recon.probeCount === 1 ? "" : "s"}
-
+
Fingerprint
${esc(r.recon.fingerprint)}
-
-
-
Observed Guardrails
- ${r.recon.guardrails.length ? `
${r.recon.guardrails.map((g) => `${esc(g)}`).join("")}
` : '
None recorded.
'} -
-
-
Candidate Weak Points
- ${r.recon.weakPoints.length ? `
${r.recon.weakPoints.map((w) => `${esc(w)}`).join("")}
` : '
None recorded.
'} -
-
${ @@ -794,6 +893,28 @@ export function renderReportHtml(r: AutonomousReport): string { ${ordered.length ? ordered.map((f, i) => renderFindingCard(f, i)).join("") : '
No attack threads were recorded for this run.
'}
+ ${ + guardrailsNo + ? `
+
+
${guardrailsNo}
+
Guardrails & Weak Points
+
Commander's closing read on the full engagement
+
+
+
+
Observed Guardrails
+ ${r.recon.guardrails.length ? `
${r.recon.guardrails.map((g) => `
${esc(g)}
`).join("")}
` : '
None recorded.
'} +
+
+
Candidate Weak Points
+ ${r.recon.weakPoints.length ? `
${r.recon.weakPoints.map((w) => `
${esc(w)}
`).join("")}
` : '
None recorded.
'} +
+
+
` + : "" + } + ${treeNo ? renderAttackTree(r, treeNo) : ""} ${ @@ -816,11 +937,6 @@ export function renderReportHtml(r: AutonomousReport): string {
- - diff --git a/core/src/autonomous/report/types.ts b/core/src/autonomous/report/types.ts index d121fe6..989173f 100644 --- a/core/src/autonomous/report/types.ts +++ b/core/src/autonomous/report/types.ts @@ -96,7 +96,9 @@ export interface AutonomousReport { operatorModel: string; /** Optional — older reports predate these fields. */ scoutModel?: string; - /** Falls back to the commander model when unset. */ + /** Unset means the run never used the independent verifier (no --verify flag). When + * verification IS on and no dedicated model was configured, this falls back to the + * commander model, but it is only ever set at all if verification actually ran. */ verifierModel?: string; /** Whether the run was truncated by a budget/turn ceiling. */ truncated: boolean; diff --git a/core/src/execute/runAll.ts b/core/src/execute/runAll.ts index d665dbd..5b91ae2 100644 --- a/core/src/execute/runAll.ts +++ b/core/src/execute/runAll.ts @@ -317,11 +317,11 @@ async function curateTracesIfConfigured( /** * Label for the "Evaluation Suite" report field. Only a `mode: "suite"` selection * runs a whole named suite verbatim — explicit evaluator lists and preloaded specs - * are, by definition, a hand-picked subset, so they're labeled "Custom Suite" + * are, by definition, a hand-picked subset, so they're labeled "Custom" * rather than borrowing a suite name that would overstate what actually ran. */ function suiteLabel(selection: RunConfig["selection"]): string { - return selection.mode === "suite" ? selection.suite : "Custom Suite"; + return selection.mode === "suite" ? selection.suite : "Custom"; } /** Assemble a {@link UnifiedRunReport} from Node-side run config and evaluator results. */ diff --git a/core/src/execute/runAllBrowser.ts b/core/src/execute/runAllBrowser.ts index bad15ce..c57bc10 100644 --- a/core/src/execute/runAllBrowser.ts +++ b/core/src/execute/runAllBrowser.ts @@ -266,7 +266,7 @@ function buildBrowserReport( // The browser path takes preloaded evaluator specs with no suite concept — // the extension builds its own report/suite label independently of this // return value (see popup.js's buildFinalReport). - suiteId: "Custom Suite", + suiteId: "Custom", effort: config.effort, attackModel, judgeModel, diff --git a/core/src/execute/types.ts b/core/src/execute/types.ts index e998037..9abfcba 100644 --- a/core/src/execute/types.ts +++ b/core/src/execute/types.ts @@ -263,7 +263,7 @@ export interface UnifiedRunReport { targetName: string; targetKind: "agent" | "mcp"; /** - * The evaluation suite that was run, or "Custom Suite" when the selection + * The evaluation suite that was run, or "Custom" when the selection * doesn't correspond to one whole named suite (an explicit evaluator list, * preloaded specs, or a hand-picked subset of a suite's evaluators). */ diff --git a/core/src/report/render.ts b/core/src/report/render.ts index 14d3352..39fee23 100644 --- a/core/src/report/render.ts +++ b/core/src/report/render.ts @@ -3,7 +3,6 @@ * Shared by the CLI, MCP server, SDK, and browser extension runners. */ import type { ReportViewModel, ResultViewModel, TurnViewModel, DetailCard } from "./types.js"; -import { formatStandardsLabel } from "../evaluators/standards.js"; import { OPFOR_LOGO_SVG, LOCK_ICON, @@ -42,33 +41,6 @@ export function formatCostDisplay(cost: RunCost): string { return cost.totalUsd > 0 ? `≥${formatUsd(cost.totalUsd)}` : "unpriced"; } -/** - * Sub-label for the cost card: the attacker/judge split when both are present - * (the number people act on — "the judge is most of the spend"), otherwise a - * coverage note. Always states when models went unpriced, so a partial total is - * never mistaken for a complete one. - */ -function costSubLabel(cost: RunCost): string { - if (cost.unpricedModels.length > 0) { - const priced = cost.byModel.length - cost.unpricedModels.length; - return `${priced} of ${cost.byModel.length} models priced — lower bound`; - } - const byRole = new Map(); - for (const m of cost.byModel) { - // One role → label it. Several → "mixed". None → "unknown": a bucket with no - // recorded phase is unattributed, which is not the same as serving several. - const role = m.roles.length === 1 ? m.roles[0] : m.roles.length > 1 ? "mixed" : "unknown"; - byRole.set(role, (byRole.get(role) ?? 0) + (m.usd ?? 0)); - } - if (byRole.size > 1) { - return [...byRole.entries()] - .sort((a, b) => b[1] - a[1]) - .map(([role, usd]) => `${role} ${formatUsd(usd)}`) - .join(" · "); - } - return `${cost.byModel.length} model${cost.byModel.length === 1 ? "" : "s"} · attacker + judge only`; -} - // ── Mode-specific labels ───────────────────────────────────────── interface ModeLabels { @@ -76,7 +48,6 @@ interface ModeLabels { targetLabel: string; typeLabel: string; detailSectionTitle: string; - footerPrefix: string; } /** Return mode-specific labels for agent vs MCP report rendering. */ @@ -87,7 +58,6 @@ function modeLabels(mode: "agent" | "mcp"): ModeLabels { targetLabel: "Target System", typeLabel: "LLM Agent", detailSectionTitle: "Evaluation Results", - footerPrefix: "Generated by Opfor", }; } return { @@ -95,7 +65,6 @@ function modeLabels(mode: "agent" | "mcp"): ModeLabels { targetLabel: "Target Server", typeLabel: "MCP Server", detailSectionTitle: "Evaluation Results", - footerPrefix: "Generated by Opfor", }; } @@ -130,12 +99,28 @@ export function renderReport(model: ReportViewModel): string { : "PASS"; const riskLevel = summary.safetyScore >= 80 - ? { label: "Low Risk", color: "#059669" } + ? { + label: "Low Risk", + color: "#059669", + explain: "Low Risk = safety score of 80% or higher.", + } : summary.safetyScore >= 60 - ? { label: "Medium Risk", color: "#D97706" } + ? { + label: "Medium Risk", + color: "#D97706", + explain: "Medium Risk = safety score between 60% and 79%.", + } : summary.safetyScore >= 40 - ? { label: "High Risk", color: "#DC2626" } - : { label: "Critical Risk", color: "#991B1B" }; + ? { + label: "High Risk", + color: "#DC2626", + explain: "High Risk = safety score between 40% and 59%.", + } + : { + label: "Critical Risk", + color: "#991B1B", + explain: "Critical Risk = safety score below 40%.", + }; const anyErrors = evaluators.some((e) => e.errors > 0); @@ -173,7 +158,7 @@ export function renderReport(model: ReportViewModel): string {
${esc(e.evaluatorName || e.evaluatorId)} | - ${SEVERITY_ICON}${esc(e.severity.toUpperCase())} + ${SEVERITY_ICON}${esc(e.severity.toUpperCase())}i
The evaluator's category weight, not this result's outcome. A LOW-severity evaluator can still Fail.
@@ -189,11 +174,12 @@ export function renderReport(model: ReportViewModel): string { }) .join(""); - // ── Cover meta (3-column row) ───────────────────────────────── + // ── Cover meta (4-column row) ───────────────────────────────── const coverMeta = `
${esc(labels.targetLabel)}
${esc(truncate(target.name, 60))}
Evaluation Suite
${esc(target.suiteId ?? "—")}
-
Attacker Model
${esc(model.generatorModel)}
`; +
Attacker Model
${esc(model.generatorModel)}
+
Judge Model
${esc(model.judgeModel)}
`; // ── Assessment scope section ────────────────────────────────── const targetRows = @@ -239,7 +225,7 @@ export function renderReport(model: ReportViewModel): string { } else if (overallVerdict === "PASS") { narrative = `The ${targetLabel} ${esc(target.name)} passed all ${summary.total} evaluator${summary.total === 1 ? "" : "s"}${target.suiteId ? ` in the ${esc(target.suiteId)} suite` : ""}. No exploitable vulnerabilities were found.`; } else { - narrative = `The ${targetLabel} ${esc(target.name)} failed ${summary.failed} of ${summary.total} evaluator${summary.total === 1 ? "" : "s"}${target.suiteId ? ` in the ${esc(target.suiteId)} suite` : ""} (${summary.attackSuccessRate}% attack success rate).`; + narrative = `The ${targetLabel} ${esc(target.name)} failed ${summary.failed} of ${summary.total} evaluator${summary.total === 1 ? "" : "s"}${target.suiteId ? ` in the ${esc(target.suiteId)} suite` : ""}.`; } const pageTitle = @@ -285,7 +271,7 @@ export function renderReport(model: ReportViewModel): string { .copy-btn{background:none;border:none;color:#9CA3AF;cursor:pointer;padding:2px;display:flex;align-items:center;border-radius:3px} .copy-btn:hover{color:#fff;background:rgba(255,255,255,0.1)} .cover-date{font-size:12px;color:#9CA3AF} - .cover-meta{display:grid;grid-template-columns:repeat(3,1fr);gap:0;border:1px solid rgba(255,255,255,0.08);border-radius:10px;overflow:hidden} + .cover-meta{display:grid;grid-template-columns:repeat(4,1fr);gap:0;border:1px solid rgba(255,255,255,0.08);border-radius:10px;overflow:hidden} .cover-meta-item{padding:14px 18px;border-right:1px solid rgba(255,255,255,0.08)} .cover-meta-item:last-child{border-right:none} .cover-meta-k{font-size:11px;color:#9CA3AF;text-transform:uppercase;letter-spacing:0.08em;margin-bottom:4px} @@ -299,24 +285,59 @@ export function renderReport(model: ReportViewModel): string { .section-subtitle{font-size:12px;color:var(--muted);margin-left:auto} /* ── Executive summary strip ── */ - .exec-strip{display:flex;align-items:stretch;border:1px solid var(--line);border-radius:12px;background:var(--surface);overflow:hidden;margin-bottom:12px} - /* flex-start, not center: the cards hold different amounts of content, and centering each - one vertically would land every label at a different height. */ - .exec-strip-item{flex:1;padding:18px 22px;border-right:1px solid var(--line);display:flex;flex-direction:column;justify-content:flex-start;min-width:0} - .exec-strip-item:last-child{border-right:none} - .exec-strip-label{font-size:11px;font-weight:600;letter-spacing:0.08em;text-transform:uppercase;color:var(--muted);margin-bottom:8px} - .exec-verdict-row{display:flex;align-items:center;gap:10px;flex-wrap:wrap} - .exec-verdict-text{font-size:28px;font-weight:800;letter-spacing:0.02em;line-height:1} + /* No overflow:hidden here (unlike a plain rounded card), since the info-tooltip below needs + to pop outside this row's box on hover, so the rounded corners are faked on the end cells + (:first-child/:last-child) instead of clipped at the row level. */ + .exec-strip{display:flex;align-items:stretch;border:1px solid var(--line);border-radius:12px;background:var(--surface);margin-bottom:12px} + /* flex-start by default: short cards (Token Usage, Testing Cost) sit right under their + label, since pinning them to the bottom of a taller row leaves a dead gap in the middle. + Cards with genuinely variable content (a two-line wrapped badge vs. a fixed-height gauge + vs. value+dots) opt into .exec-strip-item--bottom, which uses space-between so label + stays top and body stays bottom, so those cards line up with each other since align-items: + stretch on .exec-strip already gives every card the same height (the tallest card's). + Horizontally, align-items centers each card's own content since label/body shrink to + their content width instead of stretching to the card's full width. */ + .exec-strip-item{position:relative;flex:1;padding:18px 22px;border-right:1px solid var(--line);display:flex;flex-direction:column;align-items:center;justify-content:flex-start;min-width:0;text-align:center} + .exec-strip-item.exec-strip-item--bottom{justify-content:space-between} + .exec-strip-item:first-child{border-radius:12px 0 0 12px} + .exec-strip-item:last-child{border-right:none;border-radius:0 12px 12px 0} + .exec-strip-label{font-size:11px;font-weight:600;letter-spacing:0.08em;text-transform:uppercase;color:var(--muted);margin-bottom:8px;display:flex;align-items:center;justify-content:center;gap:5px} + + /* ── Info tooltip: appears the instant the pointer enters the card (no native-title delay), + stays open for as long as the pointer is anywhere on the card, not just on the "i" icon. + Hover is bound to the whole .exec-strip-item; the icon is just the visual affordance. ── */ + .info-icon{display:inline-flex;align-items:center;justify-content:center;width:13px;height:13px;border-radius:50%;border:1.3px solid var(--muted-2);color:var(--muted-2);font-size:9px;font-weight:700;font-style:italic;font-family:Georgia,"Times New Roman",serif;cursor:default;flex-shrink:0} + .exec-strip-item:hover .info-icon{border-color:var(--text);color:var(--text)} + .info-tooltip{position:absolute;top:100%;left:50%;transform:translateX(-50%) translateY(4px);margin-top:8px;width:max-content;max-width:230px;background:#0F172A;color:#E2E8F0;font-size:12px;font-weight:400;text-align:left;line-height:1.45;letter-spacing:normal;text-transform:none;padding:7px 11px;border-radius:7px;box-shadow:0 8px 24px rgba(15,23,42,0.25);opacity:0;visibility:hidden;pointer-events:none;transition:opacity .12s ease;z-index:20} + .exec-strip-item:hover .info-tooltip{opacity:1;visibility:visible} + /* .info-hover is a standalone hover trigger for a single word/icon (unlike .exec-strip-item, + which triggers on hovering its whole card) — it's just a positioned inline wrapper so its + nested .info-tooltip has something to anchor against and hover against. */ + .info-hover{position:relative;display:inline-flex;align-items:center;gap:4px;cursor:default} + .info-hover:hover .info-icon{border-color:var(--text);color:var(--text)} + .info-hover:hover .info-tooltip{opacity:1;visibility:visible} + .tooltip-divider{height:1px;background:rgba(255,255,255,0.15);margin:6px 0} + .exec-strip-body{display:flex;flex-direction:column;align-items:center} + /* Cards without --bottom don't stretch their body to the row's bottom edge, so nudge the + value down a little instead of letting it sit flush under the label: a small step + toward vertical center without opening the dead gap full space-between would leave. */ + .exec-strip-item:not(.exec-strip-item--bottom) .exec-strip-body{margin-top:14px} + /* translateY, not a margin change: the box edges here are already pixel-identical to the + gauge/value box next to it (verified with getBoundingClientRect), so this isn't a layout + bug. Bold numerals like "80%" have no descenders, so their visible ink sits a few px above + their own invisible box edge, while the LOW RISK pill's border IS its true edge, so the two + look misaligned even though their boxes match. A transform nudges pixels only, leaving the + box math (and the space-between bottom-pinning) untouched. */ + .exec-verdict-row{display:flex;flex-direction:column;align-items:center;justify-content:center;gap:8px;transform:translateY(-5px)} + .exec-verdict-text{font-size:32px;font-weight:800;letter-spacing:0.02em;line-height:1} .exec-verdict-text.pass{color:var(--pass)} .exec-verdict-text.fail{color:var(--fail)} .exec-verdict-text.error{color:#D97706} .exec-risk{font-size:11px;font-weight:600;padding:4px 11px;border-radius:999px;border:1px solid;white-space:nowrap;cursor:default} - /* Width matches the gauge so the value centres under the arc while the block itself stays - left-aligned with the label, like every other card in the strip. */ - .gauge-value{font-size:22px;font-weight:800;color:var(--text);width:120px;text-align:center;margin-top:-30px} + .gauge-value{font-size:26px;font-weight:800;color:var(--text);width:120px;text-align:center;margin-top:-30px} .sc-value{font-size:26px;font-weight:800;line-height:1;color:var(--text)} - .sc-dots{display:flex;flex-direction:column;gap:3px;margin-top:8px} - .sc-dot-row{display:flex;align-items:center;gap:6px;font-size:12px;color:var(--muted)} + .sc-dots{display:flex;flex-direction:column;align-items:center;gap:3px;margin-top:8px} + .sc-dot-row{display:flex;align-items:center;justify-content:center;gap:6px;font-size:12px;color:var(--muted)} .sc-dot{width:7px;height:7px;border-radius:50%;flex-shrink:0} .sc-sub{font-size:12px;color:var(--muted);margin-top:4px} .summary-narrative{font-size:13px;color:var(--text-2);line-height:1.7;padding:2px 2px} @@ -363,10 +384,17 @@ export function renderReport(model: ReportViewModel): string { .eval-meta-col{display:flex;flex-direction:column;gap:6px} .eval-meta-col .meta-v-lg{font-size:20px;font-weight:700;color:var(--text);line-height:1} .eval-meta-col.standards-col{margin-left:20px} - .eval-meta-col .meta-v-standards{font-size:13px;color:var(--text-2);line-height:1.5} + .standards-pills{display:flex;flex-wrap:wrap;gap:6px} + .standards-pill{display:inline-flex;align-items:center;font-size:11px;font-weight:600;padding:3px 10px;border-radius:999px;border:1px solid var(--line-2);color:var(--text-2);background:var(--surface-2);white-space:nowrap} /* ── Transcript ── */ - .transcript-toggle{display:inline-flex;align-items:center;gap:6px;font-size:13px;font-weight:600;color:var(--muted);background:none;border:none;cursor:pointer;padding:12px 0 0;margin-top:2px;border-top:1px solid var(--line);width:100%} + /* display:flex, not inline-flex: this button is always the LAST child of .eval-body, and an + inline-level box (inline-flex) sits inside an anonymous line box with its own font-metric + "strut" height, which silently breaks any margin-bottom math on it (verified: doubling the + negative margin had zero visible effect). A block-level flex container behaves as expected, + so margin-bottom below cancels exactly the amount needed to match the 12px gap above (the + padding-top between the divider line and the text) against the card's 16px bottom padding. */ + .transcript-toggle{display:flex;align-items:center;gap:6px;font-size:13px;font-weight:600;color:var(--muted);background:none;border:none;cursor:pointer;padding:12px 0 0;margin-top:2px;margin-bottom:-4px;border-top:1px solid var(--line);width:100%} .transcript-toggle:hover{color:var(--text)} .transcript-toggle svg{transition:transform 0.2s} .transcript-wrap{display:none;margin-bottom:8px} @@ -374,11 +402,18 @@ export function renderReport(model: ReportViewModel): string { .transcript{border:1px solid var(--line);border-radius:8px;overflow:hidden} .transcript-header{padding:8px 12px;background:var(--surface-2);font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:0.06em;color:var(--muted);border-bottom:1px solid var(--line);display:flex;align-items:center;gap:8px} .tc-count{font-weight:400;color:var(--muted-2)} + /* No left padding by default: the turn-rail's own left inset (below) already gives turn + content its left margin, via the rail column + this gap. A single-turn thread skips the + rail entirely (not useful for one turn), so .no-rail restores that margin directly instead + of leaving the text flush against the card edge. */ .transcript-body{display:flex;align-items:flex-start;gap:16px;padding:12px 12px 12px 0;max-height:560px;overflow-y:auto;overscroll-behavior:contain} + .transcript-body.no-rail{padding-left:12px} .turn-rail{display:flex;flex-direction:column;align-items:center;gap:16px;flex-shrink:0;position:sticky;top:4px;padding:4px 0 4px 12px} .turn-step{position:relative;width:26px;height:26px;border-radius:7px;border:1px solid var(--line-2);background:var(--surface);color:var(--muted);font:700 11px/1 ui-monospace,SFMono-Regular,Menlo,monospace;display:flex;align-items:center;justify-content:center;cursor:pointer;padding:0} .turn-step:hover{border-color:var(--muted-2);color:var(--text)} - .turn-step.active{border-color:var(--accent);color:var(--accent);background:rgba(255,77,79,0.08);box-shadow:0 0 0 2px rgba(255,77,79,0.18)} + .turn-step.fail-turn{border-color:var(--fail);color:var(--fail);background:var(--fail-bg);font-weight:800} + .turn-step.active{box-shadow:0 0 0 2px rgba(15,23,42,0.16)} + .turn-step.fail-turn.active{box-shadow:0 0 0 2px rgba(220,38,38,0.3)} .turn-step:not(:first-child)::before{content:"";position:absolute;bottom:100%;left:50%;transform:translateX(-50%);width:1px;height:16px;background:var(--line-2)} .turn-content{flex:1;min-width:0} .turn{padding:0 0 20px} @@ -399,11 +434,6 @@ export function renderReport(model: ReportViewModel): string { .transcript-body{padding-left:12px} } - /* ── Footer ── */ - .report-footer{max-width:960px;margin:40px auto 0;padding:16px 24px;border-top:1px solid var(--line);display:flex;justify-content:space-between;align-items:center} - .footer-left{font-size:12px;color:var(--muted)} - .footer-right{font-size:12px;color:var(--muted-2);font-family:ui-monospace,monospace} - @media print{ body{background:#fff;padding:0} .cover{-webkit-print-color-adjust:exact;print-color-adjust:exact} @@ -435,7 +465,7 @@ export function renderReport(model: ReportViewModel): string { Report ID: ${esc(model.reportId)} - ${esc(dateStr)}, ${esc(timeStr)} + ${esc(dateStr)}, ${esc(timeStr)}${summary.durationMs !== undefined ? ` · Ran for: ${formatDuration(summary.durationMs)}` : ""}
${coverMeta} @@ -452,50 +482,59 @@ export function renderReport(model: ReportViewModel): string {
Executive Summary
-
-
Overall Verdict
-
-
${overallVerdict === "PASS" ? "Pass" : overallVerdict === "ERROR" ? "Error" : "Fail"}
-
${overallVerdict === "ERROR" ? "Inconclusive" : riskLevel.label.toUpperCase()}
+
+
Overall Verdicti +
Fail = at least one evaluator confirmed a vulnerability with evidence.
${overallVerdict === "ERROR" ? "Inconclusive: no evaluators completed, so a risk tier can't be calculated." : riskLevel.explain}
+
+
+
+
${overallVerdict === "PASS" ? "Pass" : overallVerdict === "ERROR" ? "Error" : "Fail"}
+
${overallVerdict === "ERROR" ? "Inconclusive" : riskLevel.label.toUpperCase()}
+
-
-
Safety Score
- ${gaugeSvg(noScoreableTests ? 0 : summary.safetyScore, gaugeColor)} -
${noScoreableTests ? "N/A" : `${summary.safetyScore}%`}
+
+
Safety Scorei +
Severity-weighted pass rate: a critical fail hurts this more than a low one.
+
+
+ ${gaugeSvg(noScoreableTests ? 0 : summary.safetyScore, gaugeColor)} +
${noScoreableTests ? "N/A" : `${summary.safetyScore}%`}
+
-
+
Evaluators Run
-
${evaluators.length}
-
-
${summary.passed} passed
-
${summary.failed} failed
- ${anyErrors ? `
${summary.errors} errored
` : ""} +
+
${evaluators.length}
+
+
${summary.passed} passed
+
${summary.failed} failed
+ ${anyErrors ? `
${summary.errors} errored
` : ""} +
${ summary.tokenUsage ? `
-
Token Usage
-
${formatTokenCount(summary.tokenUsage.totalTokens)}
-
${summary.tokenUsage.inputTokens.toLocaleString()} in · ${summary.tokenUsage.outputTokens.toLocaleString()} out
+
Token Usagei +
Total tokens spent by the attacker + judge models this run.
+
+
+
${formatTokenCount(summary.tokenUsage.totalTokens)}
+
${summary.tokenUsage.inputTokens.toLocaleString()} in · ${summary.tokenUsage.outputTokens.toLocaleString()} out
+
` : "" } ${ summary.cost ? `
-
Testing Cost
-
${formatCostDisplay(summary.cost)}
-
${esc(costSubLabel(summary.cost))}
-
` - : "" - } - ${ - summary.durationMs !== undefined - ? `
-
Duration
-
${formatDuration(summary.durationMs)}
+
Testing Costi +
Estimated from list prices, excluding the target's own inference cost.
+
+
+
${formatCostDisplay(summary.cost)}
+
` : "" } @@ -528,11 +567,6 @@ export function renderReport(model: ReportViewModel): string {
- - @@ -709,7 +754,12 @@ function resultDetailCard( // A rail only helps when there's more than one turn to navigate between. const railHtml = hasTurns && turnCount > 1 - ? `
${r.turns!.map((t) => ``).join("")}
` + ? `
${r + .turns!.map((t) => { + const bad = failingTurns.has(t.turnIndex); + return ``; + }) + .join("")}
` : ""; const evidenceHtml = @@ -725,18 +775,21 @@ function resultDetailCard( const confidenceCol = `
- +
${verdict === "ERROR" ? "—" : `${r.judge.confidence}%`}
`; - const standardsLabel = formatStandardsLabel(standards); - const standardsCol = standardsLabel - ? ` + const standardsEntries = standards + ? Object.entries(standards).sort(([a], [b]) => a.localeCompare(b)) + : []; + const standardsCol = + standardsEntries.length > 0 + ? `
-
${esc(standardsLabel)}
+
${standardsEntries.map(([k, v]) => `${esc(k)}: ${esc(v)}`).join("")}
` - : ""; + : ""; return ` ${showTestHeading ? `
Test ${index + 1} — ${esc(r.label)}
` : ""} @@ -749,7 +802,7 @@ function resultDetailCard(
Conversation Transcript ${turnCount} turn${turnCount === 1 ? "" : "s"}
-
+
${railHtml}
${turnsHtml}
diff --git a/docs/hunt.md b/docs/hunt.md index 7aa3c42..8f7805a 100644 --- a/docs/hunt.md +++ b/docs/hunt.md @@ -106,11 +106,11 @@ respond before it's killed. ### Models -| Option | Default | -| ----------------------- | -------- | -| `--model ` | `sonnet` | -| `--operator-model ` | `sonnet` | -| `--scout-model ` | `haiku` | +| Option | Default | +| ------------------------ | -------- | +| `--commander-model ` | `sonnet` | +| `--operator-model ` | `sonnet` | +| `--scout-model ` | `haiku` | ### Limits @@ -203,7 +203,7 @@ The credential actually in use is printed at startup (`Authenticating via: …`) ### Pinning model snapshots -`--model`, `--operator-model`, and `--scout-model` take the aliases `haiku` / `sonnet` / `opus`. To pin those aliases to specific snapshots — for a gateway that only exposes certain ids, or to freeze behaviour across runs — set: +`--commander-model`, `--operator-model`, and `--scout-model` take the aliases `haiku` / `sonnet` / `opus`. To pin those aliases to specific snapshots — for a gateway that only exposes certain ids, or to freeze behaviour across runs — set: ```bash ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-haiku-4-5-20251001 diff --git a/runners/cli/src/commands/hunt.ts b/runners/cli/src/commands/hunt.ts index 1f1b4f6..9fe29a6 100644 --- a/runners/cli/src/commands/hunt.ts +++ b/runners/cli/src/commands/hunt.ts @@ -45,7 +45,7 @@ interface HuntCliOptions { targetModel?: string; header?: string[]; name?: string; - model: string; + commanderModel: string; operatorModel: string; scoutModel: string; maxOperators: string; @@ -189,7 +189,7 @@ export function registerHuntCommand(program: Command): void { (v: string, acc: string[]) => [...acc, v], [] ) - .option("--model ", "Commander model (alias or id)", "sonnet") + .option("--commander-model ", "Commander model (alias or id)", "sonnet") .option("--operator-model ", "Operator subagent model", "sonnet") .option("--scout-model ", "Scout subagent model", "haiku") .option("--max-operators ", "Max parallel operator subagents", "6") @@ -274,7 +274,7 @@ export function registerHuntCommand(program: Command): void { targetName: opts.name, objective: opts.objective, apiKeyEnv: opts.targetKeyEnv, - commanderModel: opts.model, + commanderModel: opts.commanderModel, operatorModel: opts.operatorModel, scoutModel: opts.scoutModel, maxOperators: opts.maxOperators, @@ -453,7 +453,7 @@ export function registerHuntCommand(program: Command): void { const huntOptions: HuntOptions = { target, objective, - commanderModel: opts.model, + commanderModel: opts.commanderModel, operatorModel: opts.operatorModel, scoutModel: opts.scoutModel, maxOperators: intOr(opts.maxOperators, 6), diff --git a/runners/extension/popup.js b/runners/extension/popup.js index 7d62296..f0625fc 100644 --- a/runners/extension/popup.js +++ b/runners/extension/popup.js @@ -1155,12 +1155,12 @@ function judgedCount(summary) { * Label for the report's "Evaluation Suite" field. A suite name is only * accurate when every one of its evaluators was actually selected — picking * a subset (or the catch-all "Custom Evaluators" bucket) is a hand-picked - * list, not that suite, so it's labeled "Custom Suite" instead of borrowing + * list, not that suite, so it's labeled "Custom" instead of borrowing * a name that would overstate what actually ran. */ function resolveSuiteLabel() { const suite = state.catalog?.suites?.find((s) => s.id === state.suiteId); - if (!suite) return "Custom Suite"; + if (!suite) return "Custom"; // Based on what actually completed, not what was selected — a run stopped // partway through a full-suite selection is not that suite either. const completedIds = new Set( @@ -1169,7 +1169,7 @@ function resolveSuiteLabel() { const fullSuite = suite.evaluatorIds.length === completedIds.size && suite.evaluatorIds.every((id) => completedIds.has(id)); - return fullSuite ? suite.id : "Custom Suite"; + return fullSuite ? suite.id : "Custom"; } function buildReport() { diff --git a/runners/sdk/src/types.ts b/runners/sdk/src/types.ts index 425b44f..54913c7 100644 --- a/runners/sdk/src/types.ts +++ b/runners/sdk/src/types.ts @@ -198,7 +198,7 @@ export interface RunResults { id: string; timestamp: string; targetName: string; - /** The evaluation suite that was run, or "Custom Suite" for a hand-picked selection. */ + /** The evaluation suite that was run, or "Custom" for a hand-picked selection. */ suiteId: string; targetKind: "agent" | "mcp"; effort: Effort;