diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.css b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.css index b296520650..50e57021e2 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.css +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.css @@ -41,6 +41,13 @@ .analysis-method summary { min-height: 44px; display: flex; align-items: center; cursor: pointer; color: var(--pw-text); } .analysis-method dl { display: grid; grid-template-columns: 100px minmax(0, 1fr); gap: 8px 16px; } .analysis-method dd { margin: 0; color: var(--pw-text); } +.analysis-continuation { margin-block: 16px; border-block: 1px solid var(--pw-line); padding-block: 12px; } +.analysis-continuation-summary { display: flex; align-items: center; flex-wrap: wrap; gap: 8px 16px; } +.analysis-continuation-summary strong { font-weight: 500; } +.analysis-continuation-summary > span { color: var(--pw-muted); } +.analysis-continuation-summary button { margin-inline-start: auto; } +.analysis-continuation .analysis-method { border: 0; margin: 0; padding: 0; } +.analysis-continuation .analysis-method > summary { display: list-item; list-style-position: inside; min-height: 36px; } .analysis-spin { animation: analysis-request-spin 1s linear infinite; } @keyframes analysis-request-spin { to { transform: rotate(360deg); } } @media (prefers-reduced-motion: reduce) { .analysis-spin { animation: none; } } diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx index de8f90b495..07bb9fe8e4 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx @@ -1,6 +1,6 @@ import {useCallback, useEffect, useRef, useState} from "react"; import {ArrowUpRight, LoaderCircle, RefreshCw, Sparkles, X} from "lucide-react"; -import type {AssessmentCitation, TrajectoryEpisode, TrajectoryOutputLanguage} from "../../../../../../loopx/control_plane/capabilities/trajectory_assessment_types.js"; +import type {AssessmentCitation, TrajectoryEpisode, TrajectoryOutputLanguage, TrajectoryContinuation} from "../../../../../../loopx/control_plane/capabilities/trajectory_assessment_types.js"; import {ResearchProgress, metricCopy, verdictLabel, type ResearchProgressPack} from "./goal-research-progress"; import {requestJson} from "../../data/chat"; import {sourceReadApiUrl} from "../../data/local-status-query"; @@ -10,12 +10,15 @@ import "./goal-research-analysis.css"; // Transport views of the existing toolkit producer, not an assessment, score, // source selection or job-lifecycle owner. Runtime paths never reach this view. type Job = {status: string; attempt?: number; analysis_wall_seconds?: number | null; error?: string; cancel_requested?: boolean; + reused_from_analysis_key?: string; config?: {model: string; effort: string; service_tier: string}; served_service_tier?: string | null; usage?: {input_tokens?: number; output_tokens?: number; reasoning_output_tokens?: number} | null}; type Pack = ResearchProgressPack & {source_count: number; analysis_key: string; method_version: string; config: {model: string; effort: string; service_tier: string; output_language?: TrajectoryOutputLanguage}; availability: string; reason_code: string | null; - objective?: string; episodes?: TrajectoryEpisode[]; job?: Job | null}; -type Snapshot = {ok: true; execution_enabled?: boolean; catalog: {availability: string; reason_code: string | null; default_pack_id: string | null; packs: Pack[]}; pack: Pack | null}; + objective?: string; episodes?: TrajectoryEpisode[]; job?: Job | null; + continuation_unavailable?: {base_pack_id: string}; + continuation?: TrajectoryContinuation & {base_pack_id: string; base_title: string; base_available: boolean; base_episode_count: number}}; +type Snapshot = {ok: true; execution_enabled?: boolean; catalog: {availability: string; reason_code: string | null; default_pack_id: string | null; packs: Pack[]; unreadable_pack_ids?: string[]}; pack: Pack | null}; type Citation = AssessmentCitation & {artifact: string; text: string; applicability: string}; const errorText = (error: unknown) => error instanceof Error ? error.message : String(error); const locationFor = (goal: string) => { @@ -184,13 +187,49 @@ export function ResearchAnalysisButton({analysis: a}: {analysis: Controller}) { // explanation is optional; cached results and unavailable states are reads. const readOnly = unavailable || a.pack?.result || a.pending || a.recordedRunning; return
{a.pack ?

{a.zh ? "分析请求设置（固定）：" : "Fixed analysis request: "}{a.pack.config.model} / {a.pack.config.effort} / {a.pack.config.service_tier}

: null}

{a.zh ? "问题解读、理由与限制说明按界面语言生成；切换语言后单独缓存，需点击才生成。引用保留原文。" : "Questions, reasons and limitations follow the interface language. Each language is cached separately and generated only on request. Citations preserve the original text."}

-

{a.pack?.result ? (a.zh ? "所选范围已有分析，查看不调用 Agent。" : "An analysis exists for the selected scope. Viewing never invokes an Agent.") : (a.zh ? "点击深度分析使用本机 Codex 账户，单独产生分析用量；普通浏览不调用 Agent。" : "Deep analysis uses the local Codex account with separate analysis usage; ordinary browsing never invokes an Agent.")}

+

{a.pack?.result ? (a.zh ? "所选范围已有分析，查看不调用 Agent。" : "An analysis exists for the selected scope. Viewing never invokes an Agent.") : a.pack?.continuation?.can_reuse ? (a.zh ? "本次复用已有判读，不调用模型；原用量保留在上一版。" : "This action reuses existing judgments without a model call; original usage stays with the previous version.") : (a.zh ? "点击深度分析使用本机 Codex 账户，单独产生分析用量；普通浏览不调用 Agent。" : "Deep analysis uses the local Codex account with separate analysis usage; ordinary browsing never invokes an Agent.")}

{a.zh ? "只分析已授权的固定轨迹，不改变研究历史。范围、方法与来源版本可在结果中查看。" : "Analyzes an authorized fixed trajectory without changing research history. Scope, method and source revisions remain available with the result."}

-
{a.pack && !a.pack.result && !unavailable ? {a.pack.config.model} · {a.zh ? "单独用量" : "Separate usage"} : null}
; + {a.pack?.continuation ?

{a.zh ? "接续保留上一版。新增、更正或删除可能影响旧判断，本版重新评估全部片段；仅证据内容与配置等价时复用已有判读。浏览与来源更新不启动分析。" : "Continuation preserves the previous version. Additions, corrections and removals can affect earlier judgments, so all episodes are reassessed. Existing judgments are reused only for equivalent evidence and configuration. Browsing and source updates never start analysis."}

: null} + {a.pack && !a.pack.result && !unavailable ? {a.pack.continuation?.can_reuse ? (a.zh ? "无需新模型调用" : "No new model call") : <>{a.pack.config.model} · {a.zh ? "单独用量" : "Separate usage"}} : null}; +} + +const analysisActionLabel = (p: Pack | null, zh: boolean) => p?.continuation?.can_reuse + ? (zh ? "复用分析" : "Reuse analysis") : p?.continuation ? (zh ? "接续分析" : "Continue analysis") : (zh ? "深度分析" : "Deep analysis"); + +function AnalysisContinuation({pack: p, analysis: a}: {pack: Pack; analysis: Controller}) { + const c = p.continuation, zh = a.zh; + if (!c) return p.continuation_unavailable ?

{zh ? "基础分析暂不可读；本版仍可独立分析完整范围。" : "The base analysis cannot be read; this version can still assess its complete scope independently."}

: null; + const changes = [ + c.sources.added.length ? `${zh ? "新增来源" : "New sources"} ${c.sources.added.length}` : "", + c.sources.revised.length ? `${zh ? "更新来源" : "Updated sources"} ${c.sources.revised.length}` : "", + c.sources.removed.length ? `${zh ? "移除来源" : "Removed sources"} ${c.sources.removed.length}` : "", + c.episodes.added.length ? `${zh ? "新增片段" : "New episodes"} ${c.episodes.added.length}` : "", + c.episodes.revised.length ? `${zh ? "更新片段" : "Updated episodes"} ${c.episodes.revised.length}` : "", + c.episodes.removed.length ? `${zh ? "移除片段" : "Removed episodes"} ${c.episodes.removed.length}` : "", + c.can_reuse ? (zh ? "复用原判读" : "Reuse judgments") : `${zh ? (p.result ? "已复核全部" : "完整复核") : "Reassess all"} ${p.episode_count} ${zh ? "片段" : "episodes"}`, + ].filter(Boolean).join(" · "); + return
+
{c.can_reuse ? (zh ? "证据内容未变" : "Evidence content unchanged") : (zh ? "接续上一版" : "Continue from previous version")} + {changes} + +
+ {!c.base_available ?

{zh ? "上一版尚无适用分析，本次将独立分析完整范围。" : "No applicable previous analysis is available. This version will assess the full scope independently."}

: null} + {c.configuration_changed || c.objective_changed ?

{zh ? "分析设置或目标已变更，两版判断不作为同条件比较。" : "The analysis configuration or objective changed. Judgments are not a matched comparison."}

: null} +
{p.result && c.base_available ? `${c.verdict_changes.length} ${zh ? "项判断变化" : "changed judgments"}` : (zh ? "查看变化" : "Inspect changes")} +

{c.base_title} · {c.base_scope.revision} → {c.current_scope.revision}

+

{zh ? "上次覆盖" : "Previous coverage"}: {c.base_episode_count} · {c.base_scope.window}
{zh ? "本次覆盖" : "Current coverage"}: {p.episode_count} · {c.current_scope.window}

+

{zh ? "来源（新增 / 更正 / 删除）" : "Sources (added / revised / removed)"}: {c.sources.added.length} / {c.sources.revised.length} / {c.sources.removed.length}
{zh ? "片段（新增 / 更正 / 删除）" : "Episodes (added / revised / removed)"}: {c.episodes.added.length} / {c.episodes.revised.length} / {c.episodes.removed.length}
{zh ? "适用判断项（新增 / 移除）" : "Eligible judgments (added / removed)"}: {c.new_pair_count} / {c.removed_pair_count}

+ {c.episodes.added.length ?

{zh ? "新增片段" : "New episodes"}: {c.episodes.added.join(" · ")}

: null} + {c.episodes.revised.length ?

{zh ? "更正片段" : "Revised episodes"}: {c.episodes.revised.join(" · ")}

: null} + {c.episodes.removed.length ?

{zh ? "移除片段" : "Removed episodes"}: {c.episodes.removed.join(" · ")}

: null} +

{zh ? "无法仅凭局部变化证明旧结论不受影响，因此实质变化均复核完整新版证据；指标按本版适用项重新计算，旧版不覆盖、不相加。" : "Local changes cannot prove earlier conclusions unaffected. Substantive changes therefore reassess the complete new evidence. Metrics are recalculated from this version’s eligible items; previous results are neither overwritten nor added."}

+ {c.verdict_changes.length ? : null} +
+
; } export function ResearchAnalysisStatus({analysis: a}: {analysis: Controller}) { @@ -220,6 +259,7 @@ export function ResearchAnalysisPanel({analysis: a}: {analysis: Controller}) { return
{if (event.key === "Escape") {event.stopPropagation(); a.close();}}}>

{zh ? "轨迹深度分析" : "Trajectory analysis"}

{p?.title ?? (zh ? "只分析已授权的固定轨迹，不改变研究历史。" : "Analyze an authorized fixed trajectory without changing its history.")}

{result ? : null}
{a.error ?

{a.error}

: null} + {a.snapshot?.catalog.unreadable_pack_ids?.length && !p?.continuation_unavailable ?

{zh ? "部分历史分析暂不可读" : "Some historical analyses cannot be read"} · {a.snapshot.catalog.unreadable_pack_ids.length}

: null} {a.loading && !p ?

{zh ? "正在核对可分析来源…" : "Checking authorized sources…"}

: null} {!a.loading && !p ?

{reasonCopy(a.snapshot?.catalog.reason_code, zh)}

: null} {p ? <> @@ -228,12 +268,13 @@ export function ResearchAnalysisPanel({analysis: a}: {analysis: Controller}) {

{zh ? "正向诊断、证据采用与最终任务收益分别判断。没有综合分数，也不据此推断因果或整体研究质量。引用存在不代表模型解释已被独立核验。" : "Diagnostic value, evidence adoption and task utility are separate. There is no total score, causal claim or whole-run quality rating. Citations do not independently verify model interpretations."}

{zh ? "下方仍显示当前研究 Goal 的已记录历史。" : "The current research Goal’s recorded history remains below."}

+ {!available ?

{reasonCopy(p.reason_code, zh)}

: null} {!result || a.pending || a.recordedRunning ?
{a.pending || a.recordedRunning ? <>

{a.pending ? : null}{a.pending ? (zh ? "正在处理分析请求…" : "Processing analysis request…") : (zh ? "上次记录仍待分析结果；当前执行状态尚未核验。" : "The last receipt awaits a result; current execution is unverified.")}

: result ? - : } - {!result && !a.pending && !a.recordedRunning ? {zh ? "使用本机 Codex 账户，单独产生分析用量；浏览不调用 Agent。" : "Uses the local Codex account with separate analysis usage. Browsing never invokes an Agent."} : null} + : } + {!result && !a.pending && !a.recordedRunning ? {p.continuation?.can_reuse ? (zh ? "复用已有判读，无需新模型调用。" : "Reuses existing judgments without a new model call.") : zh ? "使用本机 Codex 账户，单独产生分析用量；浏览不调用 Agent。" : "Uses the local Codex account with separate analysis usage. Browsing never invokes an Agent."} : null}
: null} {failed && !a.pending ?

{p.job?.status === "cancelled" ? (zh ? "分析已取消，研究历史保持可读。" : "Analysis cancelled. Research history remains readable.") : (zh ? "上次分析未完成；可重试，原失败记录会保留。" : "The last analysis did not complete. Retry preserves its failure record.")}

: null} {result ? <> @@ -248,7 +289,7 @@ export function ResearchAnalysisPanel({analysis: a}: {analysis: Controller}) { const label = metricCopy[row.metric]; return
{label?.[zh ? 0 : 1] ?? row.metric}{verdictLabel(row.metric, row.verdict, zh)}

{label?.[zh ? 2 : 3]}

{row.reason}

{citationButtons(row.citations)}
; })})} -
{zh ? "来源、方法、覆盖与分析成本" : "Sources, method, coverage and analysis cost"}

{p.objective}

{zh ? "来源范围" : "Source scope"}
{p.scope.case_id} · {p.scope.run_id}
{p.scope.window}
{zh ? "方法" : "Method"}
{result.method_version} · {zh ? "模型判断未校准" : "Uncalibrated model judgment"}
{zh ? "模型设置" : "Requested model"}
{p.config.model} / {p.config.effort} / {p.config.service_tier} · {zh ? "实际服务档位" : "Served tier"}: {p.job?.served_service_tier ?? (zh ? "未知" : "unknown")}
{zh ? "分析用量" : "Analysis usage"}
{p.job?.analysis_wall_seconds == null ? "—" : `${p.job.analysis_wall_seconds.toFixed(1)}s`} · {zh ? "输入" : "input"} {p.job?.usage?.input_tokens?.toLocaleString() ?? "—"} / {zh ? "输出" : "output"} {p.job?.usage?.output_tokens?.toLocaleString() ?? "—"} tokens · {zh ? "金额未采集，推理 token 已包含于输出" : "Dollar cost unmeasured; reasoning tokens are included in output"}
{zh ? "覆盖" : "Coverage"}
{p.episode_count} {zh ? "个固定片段" : "fixed episodes"} · {p.source_count} {zh ? "份来源" : "sources"} · {zh ? "总体片段数" : "Population"}: {p.selection.population_episodes ?? (zh ? "未知" : "unknown")}
{zh ? "取样规则" : "Selection"}
{p.selection.rule}
{zh ? "版本" : "Revision"}
{p.scope.revision}

{zh ? "引用行存在不等于解释正确；本页不提供独立科学校准结论。" : "A valid citation range does not establish entailment or independent scientific calibration."}

+
{zh ? "来源、方法、覆盖与分析成本" : "Sources, method, coverage and analysis cost"}

{p.objective}

{zh ? "来源范围" : "Source scope"}
{p.scope.case_id} · {p.scope.run_id}
{p.scope.window}
{zh ? "方法" : "Method"}
{result.method_version} · {zh ? "模型判断未校准" : "Uncalibrated model judgment"}
{zh ? "模型设置" : "Requested model"}
{p.config.model} / {p.config.effort} / {p.config.service_tier} · {zh ? "实际服务档位" : "Served tier"}: {p.job?.served_service_tier ?? (zh ? "未知" : "unknown")}
{zh ? "分析用量" : "Analysis usage"}
{p.job?.reused_from_analysis_key ? (zh ? "复用上一版判读，无新增模型调用；原用量保留在上一版。" : "Reused previous judgments without a new model call; original usage stays with the previous version.") : <>{p.job?.analysis_wall_seconds == null ? "—" : `${p.job.analysis_wall_seconds.toFixed(1)}s`} · {zh ? "输入" : "input"} {p.job?.usage?.input_tokens?.toLocaleString() ?? "—"} / {zh ? "输出" : "output"} {p.job?.usage?.output_tokens?.toLocaleString() ?? "—"} tokens · {zh ? "金额未采集，推理 token 已包含于输出" : "Dollar cost unmeasured; reasoning tokens are included in output"}}
{zh ? "覆盖" : "Coverage"}
{p.episode_count} {zh ? "个固定片段" : "fixed episodes"} · {p.source_count} {zh ? "份来源" : "sources"} · {zh ? "总体片段数" : "Population"}: {p.selection.population_episodes ?? (zh ? "未知" : "unknown")}
{zh ? "取样规则" : "Selection"}
{p.selection.rule}
{zh ? "版本" : "Revision"}
{p.scope.revision}

{zh ? "引用行存在不等于解释正确；本页不提供独立科学校准结论。" : "A valid citation range does not establish entailment or independent scientific calibration."}

: null} {/* Bound native evidence is readable before any model assessment exists. */} {a.citation || a.citationBusy || a.citationError ?
{if (event.key === "Escape") {event.stopPropagation(); a.closeCitation();}}}> diff --git a/docs/development/assets/runtime-source-analysis/analysis-continuation-desktop.png b/docs/development/assets/runtime-source-analysis/analysis-continuation-desktop.png new file mode 100644 index 0000000000..5c104d27f1 Binary files /dev/null and b/docs/development/assets/runtime-source-analysis/analysis-continuation-desktop.png differ diff --git a/docs/development/assets/runtime-source-analysis/analysis-continuation-mobile.png b/docs/development/assets/runtime-source-analysis/analysis-continuation-mobile.png new file mode 100644 index 0000000000..23fd9e7588 Binary files /dev/null and b/docs/development/assets/runtime-source-analysis/analysis-continuation-mobile.png differ diff --git a/docs/development/design.md b/docs/development/design.md index c0f8a390b4..705e41c7cc 100644 --- a/docs/development/design.md +++ b/docs/development/design.md @@ -154,6 +154,11 @@ source, method and limitation paragraphs before adding another note. every item. Use declared identities rather than extracting guessed labels from prose. Keep unknowns, exclusions and denominators visible at the metric, and preserve keyboard and touch access to the full evidence. +- When analysis has a successor, show one compact version/change summary and + the existing previous-version entry. Keep the new result separate from the + last valid result while pending or failed. Put detailed input differences and + changed judgments in the same optional disclosure; do not introduce another + setup page, repeated methodology paragraphs or automatic model execution. For example, research metrics can show “19 episodes · purposive sample · uncalibrated” once, with a nearby explanation of yes/eligible counts and unknowns. diff --git a/docs/reference/trajectory-assessment.md b/docs/reference/trajectory-assessment.md index 3554ad2956..64e41d4b0c 100644 --- a/docs/reference/trajectory-assessment.md +++ b/docs/reference/trajectory-assessment.md @@ -85,6 +85,39 @@ Never impute missing dollars from an unverified model price. ## Producer and consumer boundary +### Continuing a frozen analysis + +The trusted curator can register a complete successor snapshot using the +existing `benchmark trajectory-pack-register --base-pack-id `. +The base must already belong to the same owner Goal and original Goal/case/run. +This records an immutable version link; it neither appends to the original +source nor starts a model. The Agent prepares this link with the input, rather +than asking the reader to enter internal identifiers. + +The existing Goal analysis action becomes **Continue analysis**. Its inline +version summary identifies added, corrected and removed sources/episodes, +previous and current coverage, and changed judgments after completion. Previous +versions remain accessible through the same reader and browser history. +Reading, registering or refreshing a successor never starts analysis. A failed, +cancelled or concurrently running successor does not replace a valid base result. + +The typed comparison reuses a qualified output only when literal excerpts, +source identities/paths/line positions, episode questions, eligibility, +objective, selection/window and analyst configuration are equivalent. Capture +revision/hash changes outside those excerpts do not themselves prove new +information. Explicit reuse requalifies every row against the current manifest +without starting a model; the original usage stays with its original job. +Missing or unreadable base evidence is not eligible for reuse. + +All substantive additions, corrections, deletions or contract changes currently +trigger **full reassessment of the complete successor input**, including old +episodes that later evidence may affect. No heuristic certifies an unaffected +row and no prior model summary substitutes for original evidence. New metrics +are reduced from the complete new eligible pairs, including unknowns; old and +overlapping-window counts are never summed. This conservative continuation does +not claim tail-only execution, lower token cost or scientific uplift. Method or +configuration changes retain their original applicability boundaries. + Capability owner: existing `benchmark-toolkit`. Shared input validation, fixed rubric, applicability identity and count reduction live in TypeScript at `loopx/control_plane/capabilities/trajectory_assessment.ts`; browser-safe shapes diff --git a/examples/personal-workspace-browser/research-history.mjs b/examples/personal-workspace-browser/research-history.mjs index 55caccf838..50200918b6 100644 --- a/examples/personal-workspace-browser/research-history.mjs +++ b/examples/personal-workspace-browser/research-history.mjs @@ -125,7 +125,7 @@ export const researchHistoryScenario = { let analysisFixture = false, holdOldAnalysis = false, slowStarted, releaseOld, oldDelivered; let holdCitation = false, failedCitation = false, citationStarted, releaseCitation; let unavailableAnalysis = false, episodeMetadataLimit = null, comparisonFixture = false; - let analysisWrites = 0, localizedFixture = false; + let analysisWrites = 0, localizedFixture = false, continuationFixture = false, equivalentContinuation = false; try { context = await openWorkspacePage(browser, url, {collectCoverage, beforeGoto: async (_, page) => { await page.route("**/api/chat/research-analysis**", async route => { @@ -146,6 +146,22 @@ export const researchHistoryScenario = { const selected = ["b", "c", "d"].find(id => query.get("pack_id") === id.repeat(64)) ?? "a"; if (holdOldAnalysis && selected === "a") {slowStarted(); await new Promise(resolve => {releaseOld = resolve;});} const readback = comparisonFixture ? comparisonReadback() : analysisReadback(selected); + if (continuationFixture) { + const p = readback.pack, base = selected === 'b'; + p.pack_id = (base ? 'b' : 'a').repeat(64); + p.analysis_key = (base ? '8' : '9').repeat(64); + p.scope = {...p.scope, revision: base ? 'original-capture' : 'later-evidence', window: base ? '09:00–12:00' : '09:00–14:00'}; + p.result.scope = p.scope; p.result.analysis_key = p.analysis_key; + p.title = base ? 'Synthetic original analysis' : 'Synthetic analysis with later evidence'; + if (!base) p.continuation = { + base_pack_id: 'b'.repeat(64), base_analysis_key: '8'.repeat(64), base_title: 'Synthetic original analysis', + base_scope: {...p.scope, revision: 'original-capture', window: '09:00–12:00'}, current_scope: p.scope, + sources: {added: ['follow-up'], revised: [], removed: []}, episodes: {added: [], revised: ['variant-alpha'], removed: []}, + can_reuse: false, configuration_changed: false, objective_changed: false, base_available: true, base_episode_count: 4, + reassessed_episode_ids: p.episodes.map(e => e.id), new_pair_count: 0, removed_pair_count: 0, + verdict_changes: [{episode_id: 'variant-alpha', metric: 'benefit_test', before: 'yes', after: 'unknown'}], + }; + } for (const pack of [...readback.catalog.packs, readback.pack]) pack.config = {...pack.config, output_language: language}; if (readback.pack.result) { readback.pack.result.output_language = language; @@ -154,11 +170,20 @@ export const researchHistoryScenario = { question: language === 'zh' ? `${episode.id} 是否检验原始目标？` : `Did ${episode.id} test the original objective?`})); for (const assessment of readback.pack.result.assessments) assessment.reason = language === 'zh' ? '来源只记录了实验结果，缺少干预与目标瓶颈的联系。' : 'The source records the result but not the intervention’s link to the objective bottleneck.'; + if (continuationFixture) for (const row of readback.pack.result.assessments) { + if (row.metric === 'benefit_test') row.reason = language === 'zh' ? '后续对照未覆盖原始任务收益，判断保留未知。' : 'The later control does not cover original-task utility; the judgment remains unknown.'; + if (row.metric === 'evidence_adoption') row.reason = language === 'zh' ? '后续选择记录未标明采用版本，尚不能建立采用关系。' : 'The later selection does not identify an adopted revision, so adoption remains unestablished.'; + } readback.pack.result.limitations = [language === 'zh' ? '合成展示；不是科学校准结果。' : 'Synthetic presentation; not scientific calibration.']; } } if (episodeMetadataLimit === 0) delete readback.pack.episodes; else if (episodeMetadataLimit !== null) readback.pack.episodes = readback.pack.episodes.slice(-episodeMetadataLimit); + if (equivalentContinuation && selected === 'a') { + Object.assign(readback.pack.continuation, {can_reuse: true, + sources: {added: [], revised: [], removed: []}, episodes: {added: [], revised: [], removed: []}, reassessed_episode_ids: [], verdict_changes: []}); + readback.pack.result = null; readback.pack.job = null; + } await route.fulfill(unavailableAnalysis && selected === "a" ? {status: 503, json: {error: "Synthetic analysis read unavailable"}} : {json: readback}); @@ -758,6 +783,41 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'Syntheti await page.evaluate(key => localStorage.setItem(key, 'zh-CN'), 'loopx-pw-locale'); await page.reload({waitUntil: 'networkidle'}); await chineseAnalysis.locator('.analysis-episode h4').first().filter({hasText: 'variant-alpha 是否检验原始目标？'}).waitFor(); + continuationFixture = true; + await page.reload({waitUntil: 'networkidle'}); + // The inline version summary is a semantic section, not another page. + const version = chineseAnalysis.locator('.analysis-continuation'); + await version.getByText('接续上一版', {exact: true}).waitFor(); + await version.getByText(/已复核全部 4 片段/).waitFor(); + assert.equal(await version.locator('details').getAttribute('open'), null); + await version.locator('summary').click(); + assert.match(await version.innerText(), /original-capture → later-evidence/); + assert.match(await version.innerText(), /variant-alpha.*已做收益检验.*证据不足/); + await version.locator('summary').click(); + await page.setViewportSize({width: 1440, height: 1000}); + await chineseAnalysis.evaluate(element => element.scrollIntoView({block: 'start'})); + await page.screenshot({path: resolve(outputDir, 'research-analysis-continuation-desktop.png'), animations: 'disabled'}); + const previousVersion = version.getByRole('button', {name: '查看上一版', exact: true}); + await previousVersion.focus(); await previousVersion.press('Enter'); + await chineseAnalysis.getByText('Synthetic original analysis', {exact: true}).waitFor(); + assert.equal(await chineseAnalysis.locator('.analysis-continuation').count(), 0); + await page.goBack({waitUntil: 'networkidle'}); + await version.getByText('接续上一版', {exact: true}).waitFor(); + await page.reload({waitUntil: 'networkidle'}); + await version.getByText('接续上一版', {exact: true}).waitFor(); + await page.setViewportSize({width: 390, height: 844}); + await chineseAnalysis.evaluate(element => element.scrollIntoView({block: 'start'})); + assert(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1)); + await page.screenshot({path: resolve(outputDir, 'research-analysis-continuation-mobile.png'), animations: 'disabled'}); + await page.evaluate(key => localStorage.setItem(key, 'en'), 'loopx-pw-locale'); + await page.reload({waitUntil: 'networkidle'}); + await page.getByRole('button', {name: 'Previous version', exact: true}).waitFor(); + equivalentContinuation = true; + await page.reload({waitUntil: 'networkidle'}); + await page.getByRole('button', {name: 'Reuse analysis', exact: true}).first().waitFor(); + await page.getByRole('button', {name: 'About deep analysis', exact: true}).click(); + await page.getByText('This action reuses existing judgments without a model call; original usage stays with the previous version.', {exact: true}).waitFor(); + await page.keyboard.press('Escape'); assert.equal(analysisWrites, 0, "Cached, reload, switch and keyboard reading never request execution"); assert.equal(api.turnRequests.length, 0, "Reading the graph cannot start an Agent"); return {note: "Three unchanged real checkpoint histories through production HTTP + packaged UI; per-Goal complete pagination, genuine negative evidence, no invented edges, reload/back/forward, mobile and failed refresh. Separate synthetic coincident-target regression; no benchmark qualification or model dispatch.", diff --git a/loopx/capabilities/benchmark_toolkit/trajectory_assessment.py b/loopx/capabilities/benchmark_toolkit/trajectory_assessment.py index 1ad8dce185..e2d68e2468 100644 --- a/loopx/capabilities/benchmark_toolkit/trajectory_assessment.py +++ b/loopx/capabilities/benchmark_toolkit/trajectory_assessment.py @@ -74,6 +74,7 @@ def execute( retry: bool = False, timeout_seconds: int = 900, codex: str = "codex", + reuse_from: dict[str, Any] | None = None, ) -> dict[str, Any]: # Re-derive the request at the I/O boundary; do not trust caller-supplied # paths, prompts or schemas hidden in an otherwise valid prepared object. @@ -84,6 +85,12 @@ def execute( if prepared["analysis_key"] != verified["analysis_key"]: raise ValueError("Analysis input/config key mismatch") prepared = verified + if reuse_from is not None: + comparison = effect_runtime_result( + "benchmark.trajectory.continuation", {**prepared, "previous": reuse_from} + ) + if not comparison["can_reuse"]: + raise ValueError("Changed evidence requires full reassessment") directory = output_root.resolve() / prepared["analysis_key"] directory.mkdir(parents=True, exist_ok=True, mode=0o700) lock = directory / "execution.lock" @@ -199,6 +206,26 @@ def execute( "reused": False, } _write(directory / "receipt.json", receipt) + if reuse_from is not None: + # Same declared evidence, eligibility, scope and model contract. + # Requalify against the new exact manifest; never splice labels or + # add old metric totals to new ones. This path launches no child. + output = {key: reuse_from["output"][key] for key in + ("assessments", "limitations", "episode_questions") + if key in reuse_from["output"]} + result = effect_runtime_result( + "benchmark.trajectory.qualify", {**prepared, "output": output} + ) + _write(directory / "model-output.json", output) + _write(directory / "assessment.json", result) + receipt.update( + status="succeeded", ok=True, reused=True, + reused_from_analysis_key=reuse_from["analysis_key"], + analysis_wall_seconds=time.monotonic() - started, + **_result_proof(directory, prepared), + ) + _write(directory / "receipt.json", receipt) + return receipt with ( (directory / "prompt.txt").open(encoding="utf-8") as prompt, (directory / "events.jsonl").open("w", encoding="utf-8") as events, diff --git a/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py b/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py index ddd12bbd0a..7fff8e758d 100644 --- a/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py +++ b/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py @@ -12,7 +12,7 @@ from pathlib import Path, PurePosixPath from ...control_plane.content_digest import BARE_SHA256_PATTERN -from ...control_plane.effect_runtime import effect_runtime_result +from ...control_plane.effect_runtime import EffectRuntimeRejected, effect_runtime_result from ...file_lock import exclusive_file_lock from .trajectory_assessment import _digest, _write, execute, read_job, request_cancel @@ -54,6 +54,8 @@ def _pack_id(owner_goal_id: str, record: dict) -> str: identity["solver_usage"] = record["solver_usage"] if "native_outcomes" in record: identity["native_outcomes"] = record["native_outcomes"] + if "base_pack_id" in record: + identity["base_pack_id"] = record["base_pack_id"] return _digest(identity) @@ -242,6 +244,7 @@ def register_pack( usage_jsonl: str | None = None, usage_session_id: str | None = None, native_outcomes_json: str | None = None, + base_pack_id: str | None = None, ) -> dict: """Trusted operator only; no HTTP/renderer path-registration endpoint.""" if not title.strip(): @@ -263,6 +266,14 @@ def register_pack( }, } _verify_sources(record) + if base_pack_id is not None: + _, previous = _resolve(runtime_root, owner_goal_id, base_pack_id) + # The typed owner validates Goal/case/run identity before registration. + # This merely links frozen versions; registration launches no analysis. + effect_runtime_result("benchmark.trajectory.continuation", { + **prepared, "previous": _prepared(previous), + }) + record["base_pack_id"] = _opaque(base_pack_id) if (usage_jsonl is None) != (usage_session_id is None): raise ValueError( "Usage capture and expected session identity must be supplied together" @@ -305,11 +316,15 @@ def discover(*, runtime_root: Path, owner_goal_id: str, output_language: str | N owner = _owner(runtime_root, owner_goal_id) index = _index(owner) packs = [] + unreadable = [] for pack_id in index["pack_ids"]: - _, record = _resolve(runtime_root, owner_goal_id, pack_id) - prepared = _prepared(record) - packs.append(_summary(record, _availability(record, prepared), - _prepared(record, output_language) if output_language is not None else None)) + try: + _, record = _resolve(runtime_root, owner_goal_id, pack_id) + prepared = _prepared(record) + packs.append(_summary(record, _availability(record, prepared), + _prepared(record, output_language) if output_language is not None else None)) + except (OSError, ValueError, KeyError, TypeError, EffectRuntimeRejected): + unreadable.append(pack_id) available = any(p["availability"] == "available" for p in packs) return { "ok": True, @@ -317,6 +332,7 @@ def discover(*, runtime_root: Path, owner_goal_id: str, output_language: str | N "reason_code": None if available else "no_available_pack", "default_pack_id": index["default_pack_id"], "packs": packs, + **({"unreadable_pack_ids": unreadable} if unreadable else {}), } @@ -346,6 +362,7 @@ def _public_job(job: dict) -> dict: "served_service_tier", "reused", "cancel_requested", + "reused_from_analysis_key", ) result = {k: job[k] for k in fields if k in job} if job.get("error"): @@ -358,6 +375,7 @@ def _public_job(job: dict) -> dict: def inspect_pack( *, runtime_root: Path, owner_goal_id: str, pack_id: str | None = None, output_language: str | None = None, + include_continuation: bool = True, ) -> dict: owner, record = _resolve(runtime_root, owner_goal_id, pack_id) prepared = _prepared(record) @@ -407,7 +425,7 @@ def inspect_pack( ), }, ) - return { + view = { "ok": True, **_summary(record, reason, prepared if output_language is not None else None), "objective": record["input"]["objective"], @@ -416,6 +434,51 @@ def inspect_pack( "result": result, "applicability": "historical" if reason else "current", } + if include_continuation and record.get("base_pack_id"): + continuation, _ = _continuation( + runtime_root, owner_goal_id, record, prepared, result, output_language + ) + if continuation is not None: + view["continuation"] = continuation + else: + view["continuation_unavailable"] = {"base_pack_id": record["base_pack_id"]} + return view + + +def _continuation(runtime_root, owner_goal_id, record, prepared, result, language): + """Read one explicitly linked base, without recursively loading its history.""" + base_id = record.get("base_pack_id") + if base_id is None: + return None, None + try: + _, previous = _resolve(runtime_root, owner_goal_id, base_id) + base = inspect_pack(runtime_root=runtime_root, owner_goal_id=owner_goal_id, + pack_id=base_id, output_language=language, include_continuation=False) + old = _prepared(previous, language) if language is not None else _prepared(previous) + except (OSError, ValueError, KeyError, TypeError, EffectRuntimeRejected): + # A missing/corrupt base cannot disable a valid independently qualified + # successor. It also cannot certify reuse or an unchanged judgment. + return None, None + # Changed bytes can be compared to the frozen original excerpts, but a + # removed/unreadable source must not feed reuse or continuation context. + readable = True + try: + for source in previous["input"]["sources"]: + with _source_path(Path(previous["source_root"]), source["artifact"]).open("rb") as stream: + stream.read(1) + except (OSError, ValueError): + readable = False + usable = readable and base["result"] is not None and base["reason_code"] != "method_changed" + prior = {**old, **({"output": base["result"]} if usable else {})} + projection = effect_runtime_result("benchmark.trajectory.continuation", { + **prepared, "previous": prior, **({"output": result} if result is not None else {}), + }) + return { + **projection, + "base_pack_id": base_id, "base_title": base["title"], + "base_analysis_key": base["analysis_key"], "base_available": usable, + "base_episode_count": base["episode_count"], + }, prior if projection["can_reuse"] else None def execute_pack( @@ -436,6 +499,7 @@ def execute_pack( reason = _availability(record, base) if reason: raise PackUnavailable(reason) + _, reuse = _continuation(runtime_root, owner_goal_id, record, prepared, None, output_language) return _public_job( execute( prepared, @@ -443,6 +507,7 @@ def execute_pack( retry=retry, timeout_seconds=timeout_seconds, codex=codex, + **({"reuse_from": reuse} if reuse is not None else {}), ) ) diff --git a/loopx/cli_commands/benchmark_trajectory.py b/loopx/cli_commands/benchmark_trajectory.py index d1a333964f..b57db32675 100644 --- a/loopx/cli_commands/benchmark_trajectory.py +++ b/loopx/cli_commands/benchmark_trajectory.py @@ -33,6 +33,7 @@ def register_benchmark_trajectory_commands(subparsers, add_format) -> None: register.add_argument("--input-json", type=Path, required=True) register.add_argument("--source-root", type=Path, required=True) register.add_argument("--title", required=True) + register.add_argument("--base-pack-id", help="Existing frozen pack in this owner Goal and original case/run; links a successor without running analysis.") register.add_argument( "--usage-jsonl", help="Optional complete Codex session capture, relative to source-root; read-only cost facts, not model input.", @@ -92,6 +93,7 @@ def _handle_benchmark_trajectory_command( usage_jsonl=args.usage_jsonl, usage_session_id=args.usage_session_id, native_outcomes_json=args.native_outcomes_json, + base_pack_id=args.base_pack_id, ) if args.benchmark_command == "trajectory-pack-register" else discover(**scope) diff --git a/loopx/control_plane/capabilities/trajectory_assessment_continuation.ts b/loopx/control_plane/capabilities/trajectory_assessment_continuation.ts new file mode 100644 index 0000000000..25b7e350cc --- /dev/null +++ b/loopx/control_plane/capabilities/trajectory_assessment_continuation.ts @@ -0,0 +1,55 @@ +/** Version comparison for the existing observer; never a scientific score. */ +import {EffectRuntimeRequestError} from '../effect_runtime_errors.ts'; +import {prepareTrajectoryAssessment, qualifyTrajectoryAssessment} from './trajectory_assessment.ts'; +import type {TrajectoryInput, TrajectoryContinuation, EpisodeAssessment} from './trajectory_assessment_types.ts'; + +const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b); +const ordered = (items: T[]) => [...items].sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0); +const episodeContent = (e: TrajectoryInput['episodes'][number]) => ({...e, + source_ids: [...e.source_ids].sort(), eligible_metrics: [...e.eligible_metrics].sort()}); +const sourceContent = ({artifact_sha256: _hash, ...source}: TrajectoryInput['sources'][number]) => source; +const evidenceContent = (input: TrajectoryInput) => ({scope: {...input.scope, revision: undefined}, + objective: input.objective, selection: input.selection, + sources: ordered(input.sources).map(sourceContent), episodes: ordered(input.episodes).map(episodeContent)}); +function differences(before: T[], after: T[], content: (item: T) => unknown) { + const old = new Map(before.map(item => [item.id, item])); + const current = new Map(after.map(item => [item.id, item])); + return {added: after.filter(item => !old.has(item.id)).map(item => item.id), + revised: after.filter(item => old.has(item.id) && !same(content(old.get(item.id)!), content(item))).map(item => item.id), + removed: before.filter(item => !current.has(item.id)).map(item => item.id)}; +} +const pair = (a: EpisodeAssessment) => `${a.episode_id}\0${a.metric}`; + +export function compareTrajectoryAssessments(params: Record): TrajectoryContinuation { + const current = prepareTrajectoryAssessment(params); + if (params.analysis_key !== undefined && params.analysis_key !== current.analysis_key) throw new EffectRuntimeRequestError('analysis input/config key mismatch'); + const raw = params.previous; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) throw new EffectRuntimeRequestError('previous analysis is required'); + const previous = raw as Record; + const base = prepareTrajectoryAssessment(previous); + const oldResult = previous.output === undefined ? null : qualifyTrajectoryAssessment(previous); + for (const key of ['goal_id', 'case_id', 'run_id'] as const) { + if (current.input.scope[key] !== base.input.scope[key]) throw new EffectRuntimeRequestError('continuation belongs to a different Goal/case/run'); + } + const sources = differences(base.input.sources, current.input.sources, sourceContent); + const episodes = differences(base.input.episodes, current.input.episodes, episodeContent); + // Excerpt identity includes path, source id, line range and literal text. + // A capture's other bytes/revision alone do not establish new information. + // Scope/window, selection, objective, eligibility or configuration changes + // require full reassessment. No prose heuristic certifies an unaffected row. + const can_reuse = oldResult !== null && same(base.config, current.config) && same(evidenceContent(base.input), evidenceContent(current.input)); + const oldPairs = new Map((oldResult?.assessments ?? []).map(a => [pair(a), a])); + const eligibleBefore = new Set(base.input.episodes.flatMap(e => e.eligible_metrics.map(metric => `${e.id}\0${metric}`))); + const newPairs = new Set(current.input.episodes.flatMap(e => e.eligible_metrics.map(metric => `${e.id}\0${metric}`))); + const result = params.output === undefined ? null : qualifyTrajectoryAssessment({...params, analysis_key: current.analysis_key}); + return {base_analysis_key: base.analysis_key, base_scope: base.input.scope, + current_scope: current.input.scope, sources, episodes, can_reuse, + configuration_changed: !same(base.config, current.config), objective_changed: base.input.objective !== current.input.objective, + reassessed_episode_ids: can_reuse ? [] : current.input.episodes.map(e => e.id), + new_pair_count: [...newPairs].filter(key => !eligibleBefore.has(key)).length, + removed_pair_count: [...eligibleBefore].filter(key => !newPairs.has(key)).length, + verdict_changes: (result?.assessments ?? []).flatMap(a => { + const old = oldPairs.get(pair(a)); + return old && old.verdict !== a.verdict ? [{episode_id: a.episode_id, metric: a.metric, before: old.verdict, after: a.verdict}] : []; + })}; +} diff --git a/loopx/control_plane/capabilities/trajectory_assessment_types.ts b/loopx/control_plane/capabilities/trajectory_assessment_types.ts index 6b34326bbd..519c1207f6 100644 --- a/loopx/control_plane/capabilities/trajectory_assessment_types.ts +++ b/loopx/control_plane/capabilities/trajectory_assessment_types.ts @@ -34,6 +34,16 @@ export interface TrajectoryAssessment { episode_questions?: Array<{episode_id: string; question: string}>; } +/** Changes between two frozen inputs; all substantive changes are reassessed. */ +export interface TrajectoryContinuation { + base_analysis_key: string; base_scope: TrajectoryInput['scope']; current_scope: TrajectoryInput['scope']; + sources: {added: string[]; revised: string[]; removed: string[]}; + episodes: {added: string[]; revised: string[]; removed: string[]}; + can_reuse: boolean; configuration_changed: boolean; objective_changed: boolean; + reassessed_episode_ids: string[]; new_pair_count: number; removed_pair_count: number; + verdict_changes: Array<{episode_id: string; metric: TrajectoryMetric; before: AssessmentVerdict; after: AssessmentVerdict}>; +} + /** Provider-observed facts, separate from inferred episode judgments. */ export interface TrajectorySolverUsage { session_id: string; method: 'codex_complete_session_counters_v1'; diff --git a/loopx/control_plane/effect_runtime_handlers.ts b/loopx/control_plane/effect_runtime_handlers.ts index b7376cb009..7f39a1b37a 100644 --- a/loopx/control_plane/effect_runtime_handlers.ts +++ b/loopx/control_plane/effect_runtime_handlers.ts @@ -287,6 +287,7 @@ export function createEffectRuntimeHandlers( ["benchmark.trajectory.codex-usage", lazyHandler(() => import("./capabilities/trajectory_codex_usage.ts"), ({summarizeTrajectoryCodexUsage}) => summarizeTrajectoryCodexUsage)], ["benchmark.trajectory.prepare", lazyHandler(() => import("./capabilities/trajectory_assessment.ts"), ({prepareTrajectoryAssessment}) => prepareTrajectoryAssessment)], ["benchmark.trajectory.qualify", lazyHandler(() => import("./capabilities/trajectory_assessment.ts"), ({qualifyTrajectoryAssessment}) => qualifyTrajectoryAssessment)], + ["benchmark.trajectory.continuation", lazyHandler(() => import("./capabilities/trajectory_assessment_continuation.ts"), ({compareTrajectoryAssessments}) => compareTrajectoryAssessments)], ["agent.preferences", lazyHandler(() => import("./capabilities/agent_preferences.ts"), ({agentPreferences}) => agentPreferences)], ["todo.archive.capture_dependencies", lazyHandler(() => Promise.all([import("./todos/archive_capture.ts"), import("./coordination/source_transfer.ts")]), ([{captureArchivedTodoDependencies}, {withCoordinationSourceTransfer}]) => withCoordinationSourceTransfer("todo.archive.capture_dependencies", captureArchivedTodoDependencies))], ["agent.supervisor.plan_append", lazyHandler(() => import("./agents/supervisor_event_append.ts"), ({planSupervisorEventAppend}) => planSupervisorEventAppend)], diff --git a/tests/capabilities/test_trajectory_assessment_store.py b/tests/capabilities/test_trajectory_assessment_store.py index 66a95cd21c..704fe68384 100644 --- a/tests/capabilities/test_trajectory_assessment_store.py +++ b/tests/capabilities/test_trajectory_assessment_store.py @@ -128,6 +128,7 @@ def test_scope_discovery_execute_read_citation_and_changed_source_history(pack): ) with pytest.raises(PackUnavailable, match="citation_not_in_result"): read_citation(**operation, source_id="source", first_line=1, last_line=2) + (sources / "result.txt").write_text("changed", encoding="utf-8") assert discover(**scope)["packs"][0]["reason_code"] == "source_changed" with pytest.raises(PackUnavailable, match="source_changed"): @@ -145,6 +146,93 @@ def test_scope_discovery_execute_read_citation_and_changed_source_history(pack): ) +def test_continuation_reuses_identical_excerpts_despite_other_capture_bytes(pack): + scope, operation, registered, path, sources, provider = pack + assert execute_pack(**operation, codex=provider)["status"] == "succeeded" + original = inspect_pack(**scope, pack_id=registered["pack_id"])["result"] + old_directory = next((scope["runtime_root"] / "benchmark" / "trajectory-assessments").glob("*/jobs")) / operation["analysis_key"] + old_bytes = {p.name: p.read_bytes() for p in old_directory.iterdir() if p.is_file()} + evidence = (sources / "result.txt").read_bytes() + b"Unselected capture metadata changed\n" + (sources / "result.txt").write_bytes(evidence) + data = json.loads(path.read_text()) + data["scope"]["revision"] = "capture-2" + data["sources"][0]["artifact_sha256"] = hashlib.sha256(evidence).hexdigest() + path.write_text(json.dumps(data)) + successor = register_pack(**scope, input_path=path, source_root=sources, title="Updated capture", base_pack_id=registered["pack_id"]) + assert register_pack(**scope, input_path=path, source_root=sources, title="Updated capture", base_pack_id=registered["pack_id"])["pack_id"] == successor["pack_id"] + before = inspect_pack(**scope) + assert before["job"] is None and before["result"] is None + assert before["continuation"]["can_reuse"] is True + assert before["continuation"]["sources"] == dict(added=[], revised=[], removed=[]) + new_operation = {**scope, "pack_id": successor["pack_id"], "analysis_key": successor["analysis_key"]} + completed = execute_pack(**new_operation, codex="must-not-run") + assert completed["reused"] and completed["reused_from_analysis_key"] == operation["analysis_key"] + reused_output = json.loads((old_directory.parent / successor["analysis_key"] / "model-output.json").read_text()) + assert set(reused_output) == {"assessments", "limitations"} + view = inspect_pack(**scope) + assert view["result"]["scope"]["revision"] == "capture-2" + assert view["result"]["metrics"] == original["metrics"] + assert view["result"]["evidence_manifest"][0]["artifact_sha256"] == data["sources"][0]["artifact_sha256"] + assert read_citation(**new_operation, source_id="source", first_line=2, last_line=2)["applicability"] == "current" + assert all((old_directory / name).read_bytes() == contents for name, contents in old_bytes.items()) + + +def test_new_evidence_reassesses_old_judgments_and_complete_denominator(pack, tmp_path): + scope, operation, registered, path, sources, provider = pack + execute_pack(**operation, codex=provider) + previous = inspect_pack(**scope)["result"] + data = json.loads(path.read_text()) + later = b"The earlier proxy was not representative; no native test occurred.\n" + (sources / "later.txt").write_bytes(later) + data["sources"].append(dict(id="later", artifact="later.txt", artifact_sha256=hashlib.sha256(later).hexdigest(), first_line=1, text=later.decode().strip())) + data["episodes"][0]["source_ids"].append("later") + data["episodes"].append(dict(id="follow-up", question="Did follow-up test utility?", source_ids=["later"], eligible_metrics=["benefit_test"])) + data["selection"]["population_episodes"] = 2 + data["scope"].update(revision="corrected", window="original and later evidence") + path.write_text(json.dumps(data)) + successor = register_pack(**scope, input_path=path, source_root=sources, title="Later evidence", base_pack_id=registered["pack_id"]) + before = inspect_pack(**scope) + assert before["continuation"]["can_reuse"] is False + assert before["continuation"]["reassessed_episode_ids"] == ["episode", "follow-up"] + assert before["continuation"]["new_pair_count"] == 1 + revised = {"assessments": [dict(episode_id=e, metric="benefit_test", verdict=v, reason="Later evidence corrects the earlier interpretation", citations=[dict(source_id="later", first_line=1, last_line=1)] if v != "unknown" else []) for e, v in [("episode", "no"), ("follow-up", "unknown")]], "limitations": ["Synthetic counterevidence"]} + executable = tmp_path / "reassessment" + executable.write_text(f"#!{sys.executable}\nimport sys\nfrom pathlib import Path\nprompt=sys.stdin.read()\nassert 'The earlier proxy' in prompt and '原始测量' in prompt\nPath(sys.argv[sys.argv.index('-o')+1]).write_text({json.dumps(revised)!r})\n") + executable.chmod(0o700) + new_operation = {**scope, "pack_id": successor["pack_id"], "analysis_key": successor["analysis_key"]} + execute_pack(**new_operation, codex=str(executable)) + view = inspect_pack(**scope) + benefit = next(row for row in view["result"]["metrics"] if row["metric"] == "benefit_test") + assert (benefit["yes"], benefit["no"], benefit["unknown"], benefit["eligible"]) == (0, 1, 1, 2) + assert view["continuation"]["verdict_changes"] == [dict(episode_id="episode", metric="benefit_test", before="yes", after="no")] + assert inspect_pack(**scope, pack_id=registered["pack_id"])["result"] == previous + + +def test_continuation_does_not_cross_owner_run_or_revoked_source(pack, tmp_path): + scope, operation, registered, path, sources, provider = pack + execute_pack(**operation, codex=provider) + data = json.loads(path.read_text()) + data["scope"]["run_id"] = "unrelated" + path.write_text(json.dumps(data)) + with pytest.raises(EffectRuntimeRejected, match="different Goal/case/run"): + register_pack(**scope, input_path=path, source_root=sources, title="Wrong run", base_pack_id=registered["pack_id"]) + with pytest.raises(PackUnavailable, match="pack_not_registered"): + register_pack(**{**scope, "owner_goal_id": "other-owner"}, input_path=path, source_root=sources, title="Wrong owner", base_pack_id=registered["pack_id"]) + data["scope"].update(run_id="run", revision="current") + path.write_text(json.dumps(data)) + copied = tmp_path / "current-sources" + copied.mkdir() + (copied / "result.txt").write_bytes((sources / "result.txt").read_bytes()) + successor = register_pack(**scope, input_path=path, source_root=copied, title="Current snapshot", base_pack_id=registered["pack_id"]) + (sources / "result.txt").unlink() + view = inspect_pack(**scope) + assert view["availability"] == "available" + assert not view["continuation"]["base_available"] and not view["continuation"]["can_reuse"] + assert view["continuation"]["reassessed_episode_ids"] == ["episode"] + result = execute_pack(**scope, pack_id=successor["pack_id"], analysis_key=successor["analysis_key"], codex=provider) + assert result["status"] == "succeeded" and not result["reused"] + + def test_native_results_register_through_real_cli_and_read_without_analysis(pack, tmp_path): scope, _, original, input_path, sources, _ = pack native = { @@ -164,7 +252,8 @@ def test_native_results_register_through_real_cli_and_read_without_analysis(pack result = subprocess.run([sys.executable, "-m", "loopx.cli", "--registry", str(registry), "--runtime-root", str(scope["runtime_root"]), "--format", "json", "benchmark", "trajectory-pack-register", "--goal-id", scope["owner_goal_id"], "--input-json", str(input_path), "--source-root", str(sources), - "--title", "Native outcome window", "--native-outcomes-json", native_path.name], + "--title", "Native outcome window", "--native-outcomes-json", native_path.name, + "--base-pack-id", original["pack_id"]], capture_output=True, text=True, check=True) registered = json.loads(result.stdout) assert registered["analysis_key"] == original["analysis_key"] @@ -172,6 +261,8 @@ def test_native_results_register_through_real_cli_and_read_without_analysis(pack assert registered["native_outcomes"]["projection"]["initial"]["value"] == 2 view = inspect_pack(**scope, pack_id=registered["pack_id"]) assert view["job"] is None and view["result"] is None + assert view["continuation"]["base_pack_id"] == original["pack_id"] + assert not view["continuation"]["base_available"] assert view["native_outcomes"]["projection"]["retained"]["candidate_ref"] == "candidate-a" citation = read_citation(**scope, pack_id=registered["pack_id"], analysis_key=registered["analysis_key"], source_id="source", first_line=2, last_line=2) diff --git a/tests/control_plane_ts/trajectory_assessment.test.ts b/tests/control_plane_ts/trajectory_assessment.test.ts index e9505f488f..920a47067c 100644 --- a/tests/control_plane_ts/trajectory_assessment.test.ts +++ b/tests/control_plane_ts/trajectory_assessment.test.ts @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import {prepareTrajectoryAssessment, qualifyTrajectoryAssessment, type TrajectoryInput} from '../../loopx/control_plane/capabilities/trajectory_assessment.ts'; +import {compareTrajectoryAssessments} from '../../loopx/control_plane/capabilities/trajectory_assessment_continuation.ts'; const input = (): TrajectoryInput => ({schema_version: 'trajectory_assessment_input_v1', scope: {goal_id:'goal', case_id:'case', run_id:'run', revision:'rev1', window:'0-1'}, objective:'Optimize native utility', selection:{rule:'all completed episodes in the frozen window',population_episodes:2,exhaustive:true}, @@ -14,6 +15,59 @@ const output = () => ({assessments:[ {episode_id:'e2',metric:'information_gain',verdict:'unknown',reason:'No valid measurement',citations:[]}],limitations:['Synthetic contract test, not evaluator calibration']}); const prepared = () => prepareTrajectoryAssessment({input:input(), config}); +test('capture-byte and input-order changes reuse identical declared evidence, without a new scientific judgment', () => { + const original = prepared(), next = input(); + next.scope.revision = 'capture-2'; next.sources[0]!.artifact_sha256 = 'b'.repeat(64); + next.episodes.reverse(); next.episodes[1]!.eligible_metrics.reverse(); + const current = prepareTrajectoryAssessment({input: next, config}); + const delta = compareTrajectoryAssessments({...current, previous: {...original, output: output()}}); + assert.notEqual(current.analysis_key, original.analysis_key); + assert.equal(delta.can_reuse, true); + assert.deepEqual(delta.sources, {added: [], revised: [], removed: []}); + assert.deepEqual(delta.reassessed_episode_ids, []); + assert.equal(qualifyTrajectoryAssessment({...current, output: output()}).evidence_manifest[0]!.artifact_sha256, 'b'.repeat(64)); +}); + +test('later evidence can overturn an earlier judgment; full new denominators are not summed with the old window', () => { + const original = prepared(), next = input(); + next.scope.revision = 'rev2'; next.scope.window = '0-2'; next.selection.population_episodes = 3; + next.sources.push({id: 'later', artifact: 'follow-up.txt', artifact_sha256: 'c'.repeat(64), first_line: 1, text: 'A was retried without new information; B remains untested'}); + next.episodes[0]!.source_ids.push('later'); + next.episodes.push({id: 'e3', question: 'Was later evidence useful?', source_ids: ['later'], eligible_metrics: ['information_gain']}); + const revised = output(); revised.assessments[0] = {...revised.assessments[0]!, verdict: 'no', citations: [{source_id: 'later', first_line: 1, last_line: 1}]}; + revised.assessments.push({episode_id: 'e3', metric: 'information_gain', verdict: 'yes', reason: 'A decision was resolved', citations: [{source_id: 'later', first_line: 1, last_line: 1}]}); + const current = prepareTrajectoryAssessment({input: next, config}); + const delta = compareTrajectoryAssessments({...current, previous: {...original, output: output()}, output: revised}); + assert.equal(delta.can_reuse, false); + assert.deepEqual(delta.reassessed_episode_ids, ['e1', 'e2', 'e3']); + assert.deepEqual(delta.sources.added, ['later']); assert.deepEqual(delta.episodes.added, ['e3']); + assert.equal(delta.new_pair_count, 1); assert.equal(delta.removed_pair_count, 0); + assert.deepEqual(delta.verdict_changes, [{episode_id: 'e1', metric: 'information_gain', before: 'yes', after: 'no'}]); + const metric = qualifyTrajectoryAssessment({...current, output: revised}).metrics.find(m => m.metric === 'information_gain')!; + assert.deepEqual([metric.yes, metric.no, metric.unknown, metric.eligible], [1, 1, 1, 3]); + assert.throws(() => compareTrajectoryAssessments({...current, output: output(), previous: {...original, output: output()}}), /missing eligible/); +}); + +test('corrections, removals, scope and model changes require full reassessment and identity cannot cross runs', () => { + const previous = {...prepared(), output: output()}; + for (const edit of [(x: TrajectoryInput) => {x.sources[0]!.text = 'corrected negative evidence';}, + (x: TrajectoryInput) => {x.episodes.pop(); x.selection.population_episodes = 1;}, + (x: TrajectoryInput) => {x.scope.window = 'other window';}, + (x: TrajectoryInput) => {x.objective = 'A different utility objective';}]) { + const next = input(); edit(next); + assert.equal(compareTrajectoryAssessments({input: next, config, previous}).can_reuse, false); + } + assert.equal(compareTrajectoryAssessments({input: input(), config: {...config, effort: 'high'}, previous}).can_reuse, false); + for (const key of ['goal_id', 'case_id', 'run_id'] as const) { + const next = input(); next.scope[key] = 'other'; + assert.throws(() => compareTrajectoryAssessments({input: next, config, previous}), /different Goal\/case\/run/); + } + assert.throws(() => compareTrajectoryAssessments({...prepared(), previous: {...previous, analysis_key: '0'.repeat(64)}}), /key mismatch/); + const next = input(); next.episodes.pop(); next.selection.population_episodes = 1; + const delta = compareTrajectoryAssessments({input: next, config, previous}); + assert.deepEqual(delta.episodes.removed, ['e2']); assert.equal(delta.removed_pair_count, 1); +}); + test('output language binds identity, prompt and complete analytical questions without changing evidence', () => { const original = input(), legacy = prepared(); const zh = prepareTrajectoryAssessment({input: original, config: {...config, output_language: 'zh'}}); diff --git a/tests/presentation/test_research_analysis_api.py b/tests/presentation/test_research_analysis_api.py index 2818323b62..ed2e12d77c 100644 --- a/tests/presentation/test_research_analysis_api.py +++ b/tests/presentation/test_research_analysis_api.py @@ -138,6 +138,82 @@ def test_frontend_language_scopes_process_retry_cache_and_original_citations(wor assert all(path.read_bytes() == data for path, data in w["before"].items()) +def test_continuation_http_preserves_previous_analysis_through_cancel_retry_and_equivalent_reuse(workspace): + w, read = workspace, workspace["request"] + w["mode"].write_text("ok") + first = read(output_language="zh")["pack"] + operation = {**w["operation"], "analysis_key": first["analysis_key"], "output_language": "zh"} + assert read(body=operation)["job"]["status"] == "succeeded" + previous = read(output_language="zh")["pack"]["result"] + source, path = w["sources"], w["tmp_path"] / "input.json" + data = json.loads(path.read_text()) + later = b"A late follow-up still has no representative utility test.\n" + (source / "later.txt").write_bytes(later) + data["sources"].append(dict(id="later", artifact="later.txt", artifact_sha256=hashlib.sha256(later).hexdigest(), first_line=1, text=later.decode().strip())) + data["episodes"][0]["source_ids"].append("later") + data["scope"]["revision"] = "later-evidence" + path.write_text(json.dumps(data)) + scope = dict(runtime_root=w["tmp_path"] / "runtime", owner_goal_id="research-owner") + second = register_pack(**scope, input_path=path, source_root=source, title="Later evidence", base_pack_id=first["pack_id"]) + current = read(output_language="zh")["pack"] + assert current["result"] is None and current["job"] is None + assert current["continuation"]["sources"]["added"] == ["later"] + assert current["continuation"]["reassessed_episode_ids"] == ["episode"] + assert not current["continuation"]["can_reuse"] + successor = {**operation, "pack_id": second["pack_id"], "analysis_key": current["analysis_key"]} + w["mode"].write_text("wait") + with ThreadPoolExecutor(max_workers=1) as pool: + running = pool.submit(read, body=successor) + until = time.monotonic() + 8 + while time.monotonic() < until: + view = read(output_language="zh")["pack"] + if view["job"] and view["job"]["status"] == "running": + break + time.sleep(.05) + else: + pytest.fail("successor never reached its actual process receipt") + assert read(body=successor)["job"]["status"] == "busy" + assert read(pack_id=first["pack_id"], output_language="zh")["pack"]["result"] == previous + assert read(body=successor, suffix="/cancel")["job"]["cancel_requested"] + assert running.result(timeout=8)["job"]["status"] == "cancelled" + assert read(pack_id=first["pack_id"], output_language="zh")["pack"]["result"] == previous + w["mode"].write_text("fail") + with pytest.raises(HTTPError): + read(body={**successor, "retry": True}) + w["mode"].write_text("ok") + assert read(body={**successor, "retry": True})["job"]["attempt"] == 3 + completed = read(output_language="zh")["pack"] + assert completed["result"]["metrics"][1]["eligible"] == 1 + assert completed["continuation"]["verdict_changes"] == [] + calls = w["count"].read_text() + assert read(output_language="en")["pack"]["result"] is None + assert w["count"].read_text() == calls # locale and history reads never execute + # Capture metadata outside the selected excerpt is not new evidence. + evidence = (source / "facts.txt").read_bytes() + b"Additional unselected metadata\n" + (source / "facts.txt").write_bytes(evidence) + data["sources"][0]["artifact_sha256"] = hashlib.sha256(evidence).hexdigest() + data["scope"]["revision"] = "capture-only" + path.write_text(json.dumps(data)) + third = register_pack(**scope, input_path=path, source_root=source, title="Equivalent capture", base_pack_id=second["pack_id"]) + view = read(output_language="zh")["pack"] + assert view["continuation"]["can_reuse"] and view["result"] is None + reused = read(body={**successor, "pack_id": third["pack_id"], "analysis_key": view["analysis_key"]})["job"] + assert reused["reused_from_analysis_key"] == completed["analysis_key"] + assert w["count"].read_text() == calls + assert read(output_language="zh")["pack"]["result"]["scope"]["revision"] == "capture-only" + assert read(pack_id=second["pack_id"], output_language="zh")["pack"]["result"] == completed["result"] + base_path = next((scope["runtime_root"] / "benchmark" / "trajectory-assessments").glob(f"*/packs/{second['pack_id']}.json")) + captured = base_path.read_bytes() + base_path.unlink() + unavailable_base = read(output_language="zh") + assert unavailable_base["catalog"]["unreadable_pack_ids"] == [second["pack_id"]] + assert unavailable_base["pack"]["continuation_unavailable"] == {"base_pack_id": second["pack_id"]} + assert unavailable_base["pack"]["result"]["scope"]["revision"] == "capture-only" + assert w["count"].read_text() == calls + base_path.write_bytes(captured) + assert read(output_language="zh")["pack"]["continuation"]["base_available"] + + def test_registered_http_cancel_failure_retry_reuse_and_source_return(workspace): w = workspace read, operation = w["request"], w["operation"]