diff --git a/engine/dispatch.py b/engine/dispatch.py index a4213dd..d1f79a7 100644 --- a/engine/dispatch.py +++ b/engine/dispatch.py @@ -1957,6 +1957,7 @@ def run_qualification_case(self, **params: Any) -> dict: workflow_id=workflow_id, case_id=case_id, runtime_input_bytes=runtime_input_bytes, + fault_target=params.get("fault_target"), policy_source=policy, bundle_key=self._qualification_bundle_key(workflow_id), ) diff --git a/engine/qualification.py b/engine/qualification.py index 832fdb4..6a3f20d 100644 --- a/engine/qualification.py +++ b/engine/qualification.py @@ -331,6 +331,8 @@ def _effect_view(index: int, effect, verification_policy) -> dict[str, Any]: def _qualification_controls(workflow, graph: dict[str, Any]) -> dict[str, Any]: """Project writable controls from the executable workflow and canonical project.""" + from openadapt_flow.policy import executable_actuation_paths + parameter_names = sorted( set(workflow.params) | set(workflow.param_specs) | set(workflow.secret_params) ) @@ -374,6 +376,7 @@ def _qualification_controls(workflow, graph: dict[str, Any]) -> dict[str, Any]: identity_policy = project.identity_policies.get(step.id) if project is not None else None actions[node["id"]] = { "step_id": step.id, + "execution_paths": sorted(executable_actuation_paths(step)), "classification": (classification.model_dump(mode="json") if classification else None), "identity": { "can_arm": bool(sources), @@ -1098,6 +1101,7 @@ def set_local_qualification_case_scope( workflow_id: str, case_id: str, runtime_input_bytes: bytes, + fault_target: dict[str, Any] | None = None, policy_source: str = DEFAULT_QUALIFICATION_POLICY, bundle_key: str | None = None, ) -> dict: @@ -1118,12 +1122,42 @@ def set_local_qualification_case_scope( if worklists: raise QualificationError("Desktop qualification cases do not yet support worklists") steps = {step.id: step for step in api["iter_workflow_steps"](workflow)} + selected_fault_target = None + if fault_target is not None: + if not isinstance(fault_target, dict) or set(fault_target) != { + "step_id", + "actuation_path", + }: + raise QualificationError("Fault target must name one exact action and actuation path") + try: + selected_fault_target = api["QualificationActionTarget"]( + step_id=fault_target["step_id"], + actuation_path=fault_target["actuation_path"], + ) + except (ValueError, TypeError) as exc: + raise QualificationError( + "Fault target must name one exact action and actuation path" + ) from exc + fault_step = steps.get(selected_fault_target.step_id) + if ( + fault_step is None + or selected_fault_target.actuation_path + not in executable_actuation_paths(fault_step) + ): + raise QualificationError("Fault target is outside the executable case scope") targets = [] for step_id, step in sorted(steps.items()): paths = executable_actuation_paths(step) if not paths: continue - path = "gui" if "gui" in paths else "api" if "api" in paths else None + if selected_fault_target is not None and selected_fault_target.step_id == step_id: + path = selected_fault_target.actuation_path + elif "gui" in paths: + path = "gui" + elif "api" in paths: + path = "api" + else: + path = None if path is None: raise QualificationError(f"Qualification action {step_id!r} has no executable path") targets.append(api["QualificationActionTarget"](step_id=step_id, actuation_path=path)) @@ -1133,6 +1167,7 @@ def set_local_qualification_case_scope( case_id=case_id, runtime_input_sha256=hashlib.sha256(runtime_input_bytes).hexdigest(), action_targets=targets, + fault_target=selected_fault_target, ) _save(workflow, bundle_dir, key=bundle_key) except (ValueError, TypeError) as exc: diff --git a/src/lib/types.ts b/src/lib/types.ts index a53e00f..2c6f2b7 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -199,6 +199,7 @@ export interface QualificationEditableEffect { export interface QualificationActionControls { step_id: string; + execution_paths: ("gui" | "api")[]; classification?: { step_id: string; classification: QualificationRisk | "unknown"; @@ -290,6 +291,8 @@ export interface QualificationProject { input_ref?: string | null; expected_outcome: string; required: boolean; + action_targets?: { step_id: string; actuation_path: "gui" | "api" }[]; + fault_target?: { step_id: string; actuation_path: "gui" | "api" } | null; results: { project_revision: number; runner_capabilities: string[]; diff --git a/src/screens/Qualification.tsx b/src/screens/Qualification.tsx index ed208ce..bf7056e 100644 --- a/src/screens/Qualification.tsx +++ b/src/screens/Qualification.tsx @@ -1382,7 +1382,7 @@ export function Qualification({ ? "64-character session digest" : signal.source === "application" ? "accuro or https://app.example" - : "patient-chart" + : "record-review" } spellCheck={false} /> @@ -1695,7 +1695,7 @@ export function Qualification({ {!parameters.length ? ( Add typed workflow parameters before binding a reusable effect. - OpenAdapt will not seal a patient or account literal into this form. + OpenAdapt will not seal a live identity value into this form. ) : ( <> diff --git a/src/screens/QualificationLifecycle.test.tsx b/src/screens/QualificationLifecycle.test.tsx index c106572..12e3103 100644 --- a/src/screens/QualificationLifecycle.test.tsx +++ b/src/screens/QualificationLifecycle.test.tsx @@ -75,6 +75,33 @@ function project(): QualificationProject { } as unknown as QualificationProject; } +function twoActionFaultProject(): QualificationProject { + const value = project(); + value.project!.cases.push({ + id: "fault-wrong-identity", + kind: "wrong_identity", + description: "Deterministic wrong identity refusal case", + expected_outcome: "halted", + required: true, + results: [], + }); + value.controls.actions = { + open: { + step_id: "open", + execution_paths: ["gui"], + identity: { can_arm: false, armed: false, sources: [], policy: null }, + effects: [], + }, + save: { + step_id: "save", + execution_paths: ["api", "gui"], + identity: { can_arm: false, armed: false, sources: [], policy: null }, + effects: [], + }, + }; + return value; +} + describe("Qualification lifecycle", () => { beforeEach(() => { mockedEngineInvoke.mockReset(); @@ -125,4 +152,98 @@ describe("Qualification lifecycle", () => { fireEvent.click(screen.getByRole("button", { name: "Create working version" })); await waitFor(() => expect(onOpenWorkflow).toHaveBeenCalledWith("wf-2")); }); + + it("prefills a typed fault case instead of requiring a hand-written case id", async () => { + render( + {}} + onOpenWorkflow={() => {}} + />, + ); + + fireEvent.click(screen.getByText("Add another qualification case")); + fireEvent.click(screen.getByRole("button", { name: "Use Wrong record" })); + + expect((screen.getByLabelText("Case id") as HTMLInputElement).value).toBe( + "wrong-identity-1", + ); + expect((screen.getByLabelText("Case type") as HTMLSelectElement).value).toBe( + "wrong_identity", + ); + expect((screen.getByLabelText("Description") as HTMLInputElement).value).toBe( + "The live record does not match the qualified identity; the run must halt.", + ); + + fireEvent.change(screen.getByLabelText("record id"), { + target: { value: "CASE-42" }, + }); + fireEvent.change(screen.getByLabelText("amount"), { + target: { value: "75.5" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Add case" })); + await waitFor(() => + expect(mockedEngineInvoke).toHaveBeenCalledWith( + CMD.ADD_QUALIFICATION_CASE, + expect.objectContaining({ + workflow_id: "wf-1", + case_id: "wrong-identity-1", + kind: "wrong_identity", + description: + "The live record does not match the qualified identity; the run must halt.", + parameters_json: JSON.stringify({ + record_id: "CASE-42", + amount: 75.5, + priority: "routine", + }), + }), + ), + ); + }); + + it("reuses a default fault case and sends its exact target for two actions", async () => { + const value = twoActionFaultProject(); + render( + {}} + onOpenWorkflow={() => {}} + />, + ); + + fireEvent.click(screen.getByText("Add another qualification case")); + fireEvent.click(screen.getByRole("button", { name: "Select Wrong record" })); + expect(screen.getByText("Run fault-wrong-identity")).toBeTruthy(); + expect(mockedEngineInvoke).not.toHaveBeenCalledWith( + CMD.ADD_QUALIFICATION_CASE, + expect.anything(), + ); + + fireEvent.change(screen.getByLabelText("Fault action"), { + target: { value: "save" }, + }); + fireEvent.change(screen.getByLabelText("Actuation path"), { + target: { value: "api" }, + }); + fireEvent.change(screen.getByLabelText("record id"), { + target: { value: "CASE-42" }, + }); + fireEvent.change(screen.getByLabelText("amount"), { + target: { value: "75.5" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Run and sign case" })); + + await waitFor(() => + expect(mockedEngineInvoke).toHaveBeenCalledWith( + CMD.RUN_QUALIFICATION_CASE, + expect.objectContaining({ + workflow_id: "wf-1", + case_id: "fault-wrong-identity", + fault_target: { step_id: "save", actuation_path: "api" }, + }), + ), + ); + }); }); diff --git a/src/screens/QualificationLifecycle.tsx b/src/screens/QualificationLifecycle.tsx index 9b3d8a3..0734c5e 100644 --- a/src/screens/QualificationLifecycle.tsx +++ b/src/screens/QualificationLifecycle.tsx @@ -15,6 +15,44 @@ import { Button, Callout, Card, CardHead, Field, Pill } from "../ui/primitives"; const POLICY = "clinical-write"; +const GUIDED_FAULT_CASES: { + id: string; + kind: QualificationCaseKind; + label: string; + description: string; +}[] = [ + { + id: "ambiguity-1", + kind: "ambiguity", + label: "Ambiguous target", + description: "A competing target is present; the run must halt before actuation.", + }, + { + id: "wrong-identity-1", + kind: "wrong_identity", + label: "Wrong record", + description: "The live record does not match the qualified identity; the run must halt.", + }, + { + id: "stale-identity-1", + kind: "stale_identity", + label: "Stale record", + description: "Identity changes after resolution; the run must reacquire or halt.", + }, + { + id: "weak-effect-1", + kind: "weak_effect", + label: "Weak effect evidence", + description: "The effect evidence is below the qualified tier; the run must halt.", + }, + { + id: "missing-effect-1", + kind: "missing_effect", + label: "Missing effect", + description: "The intended persisted effect is absent; the run must halt.", + }, +]; + function secretEnvironmentReference(name: string): string { return `OPENADAPT_FLOW_SECRET_${name.replace(/[^a-zA-Z0-9]/g, "_").toUpperCase()}`; } @@ -45,6 +83,10 @@ export function QualificationLifecycle({ "Representative production-shaped case", ); const [selectedCaseId, setSelectedCaseId] = useState(""); + const [faultStepId, setFaultStepId] = useState(""); + const [faultActuationPath, setFaultActuationPath] = useState<"gui" | "api" | "">( + "", + ); const [parametersJson, setParametersJson] = useState("{}"); const [parameterValues, setParameterValues] = useState({}); @@ -60,6 +102,19 @@ export function QualificationLifecycle({ () => cases.find((item) => item.id === selectedCaseId) || cases[0], [cases, selectedCaseId], ); + const actionControls = useMemo( + () => + Object.values(project.controls.actions).sort((left, right) => + left.step_id.localeCompare(right.step_id), + ), + [project.controls.actions], + ); + const selectedFaultAction = actionControls.find( + (action) => action.step_id === faultStepId, + ); + const selectedCaseNeedsFaultTarget = Boolean( + selectedCase && selectedCase.kind !== "representative", + ); const capabilityCoverageByCase = useMemo( () => new Map( @@ -92,6 +147,21 @@ export function QualificationLifecycle({ useEffect(() => setTarget(targetForProject(project)), [workflowId]); + useEffect(() => { + if (!selectedCase || selectedCase.kind === "representative") { + setFaultStepId(""); + setFaultActuationPath(""); + return; + } + const retainedTarget = + selectedCase.fault_target || + (selectedCase.action_targets?.length === 1 + ? selectedCase.action_targets[0] + : null); + setFaultStepId(retainedTarget?.step_id || ""); + setFaultActuationPath(retainedTarget?.actuation_path || ""); + }, [selectedCase?.id, workflowId]); + useEffect(() => { setParameterValues( Object.fromEntries( @@ -163,6 +233,20 @@ export function QualificationLifecycle({ ); } + function selectGuidedFaultCase( + faultCase: (typeof GUIDED_FAULT_CASES)[number], + ) { + const existing = cases.find((item) => item.kind === faultCase.kind); + if (existing) { + setSelectedCaseId(existing.id); + setNotice(`${existing.id} selected for its exact fault target.`); + return; + } + setCaseId(faultCase.id); + setCaseKind(faultCase.kind); + setDescription(faultCase.description); + } + async function runCase() { if (!selectedCase) return; const caseParametersJson = caseParameters(); @@ -173,6 +257,14 @@ export function QualificationLifecycle({ case_id: selectedCase.id, parameters_json: caseParametersJson, target, + ...(selectedCaseNeedsFaultTarget + ? { + fault_target: { + step_id: faultStepId, + actuation_path: faultActuationPath, + }, + } + : {}), ...(deploymentConfig.trim() ? { deployment_config: deploymentConfig.trim() } : {}), @@ -368,6 +460,54 @@ export function QualificationLifecycle({ idPrefix="qualification-case-target" disabled={Boolean(busy)} /> + {selectedCaseNeedsFaultTarget && ( +
+ + + + + + +
+ )} + ); + })} + +
None: + bundle = _bundle( + tmp_path / "bundle", + Step(id="open", intent="Open", action=ActionKind.CLICK), + Step( + id="save", + intent="Save", + action=ActionKind.CLICK, + api_binding=ApiBinding( + url_template="/records/{record_id}", + body_template={"record_id": "{record_id}"}, + ), + ), + params={"record_id": "example"}, + ) + initialized = _initialize(bundle) + fault_case = next( + item + for item in initialized["project"]["cases"] + if item["kind"] == "wrong_identity" + ) + assert initialized["controls"]["actions"]["open"]["execution_paths"] == ["gui"] + assert initialized["controls"]["actions"]["save"]["execution_paths"] == ["api", "gui"] + + parameters_path, _ = store_case_parameters( + tmp_path / "state", + workflow_id="wf-1", + case_id=fault_case["id"], + parameters_json='{"record_id":"case-1"}', + ) + _inputs_path, inputs = stage_case_runtime_inputs( + tmp_path / "state", + workflow_id="wf-1", + case_id=fault_case["id"], + workflow=Workflow.load(bundle), + parameters_path=parameters_path, + ) + set_local_qualification_case_scope( + bundle, + workflow_id="wf-1", + case_id=fault_case["id"], + runtime_input_bytes=inputs, + fault_target={"step_id": "save", "actuation_path": "api"}, + ) + + project = Workflow.load(bundle).qualification + assert project is not None + case = next(item for item in project.cases if item.id == fault_case["id"]) + assert [(target.step_id, target.actuation_path) for target in case.action_targets] == [ + ("open", "gui"), + ("save", "api"), + ] + assert case.fault_target is not None + assert (case.fault_target.step_id, case.fault_target.actuation_path) == ( + "save", + "api", + ) + + +def test_dispatcher_forwards_dual_path_fault_target_to_flow_scope( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + config = EngineConfig(data_dir=tmp_path / ".openadapt", log_level="WARNING") + bundle = _bundle( + config.data_dir / "bundles" / "wf-1", + Step(id="open", intent="Open", action=ActionKind.CLICK), + Step( + id="save", + intent="Save", + action=ActionKind.CLICK, + api_binding=ApiBinding( + url_template="/records/{record_id}", + body_template={"record_id": "{record_id}"}, + ), + ), + params={"record_id": "example"}, + ) + initialized = _initialize(bundle) + fault_case = next( + item + for item in initialized["project"]["cases"] + if item["kind"] == "wrong_identity" + ) + db = IndexDB(tmp_path / "index.db") + db.initialize() + db.insert_bundle("wf-1", str(bundle)) + dispatcher = EngineDispatcher(config, services=EngineServices(config, db=db)) + + def stop_after_scope(*_args, **_kwargs) -> None: + raise RuntimeError("stop after retained scope") + + monkeypatch.setattr( + "engine.qualification.prepare_local_qualification_runner", + stop_after_scope, + ) + try: + result = dispatcher.dispatch( + "run_qualification_case", + { + "workflow_id": "wf-1", + "case_id": fault_case["id"], + "parameters_json": '{"record_id":"case-1"}', + "fault_target": {"step_id": "save", "actuation_path": "api"}, + }, + ) + retained = Workflow.load(bundle).qualification + finally: + db.close() + + assert result == { + "ok": False, + "workflow_id": "wf-1", + "error": "stop after retained scope", + } + assert retained is not None + case = next(item for item in retained.cases if item.id == fault_case["id"]) + assert [(target.step_id, target.actuation_path) for target in case.action_targets] == [ + ("open", "gui"), + ("save", "api"), + ] + assert case.fault_target is not None + assert (case.fault_target.step_id, case.fault_target.actuation_path) == ( + "save", + "api", + ) + + def test_inspection_exposes_durable_target_evidence_without_flattening_to_coordinates( tmp_path: Path, ) -> None: