From c825a7dd0dbe3987f8a7c09923db01ed2065a068 Mon Sep 17 00:00:00 2001 From: Daniel Vaughn Date: Fri, 30 Oct 2026 15:51:28 -0400 Subject: [PATCH] fix(analysis-ledger): drop allOf schema rules unsupported by item registry (0.2.2) --- capabilities/analysis-ledger/capability.yaml | 2 +- capabilities/analysis-ledger/items.py | 92 +---------- .../tests/test_analysis_ledger.py | 146 +++++++++++++----- 3 files changed, 112 insertions(+), 128 deletions(-) diff --git a/capabilities/analysis-ledger/capability.yaml b/capabilities/analysis-ledger/capability.yaml index ae2c104..827f695 100644 --- a/capabilities/analysis-ledger/capability.yaml +++ b/capabilities/analysis-ledger/capability.yaml @@ -1,6 +1,6 @@ schema: 1 name: analysis-ledger -version: "0.2.1" +version: "0.2.2" description: > Persistent single-agent code analysis over a durable Agent Output worklist. The agent records analysis runs, ranked targets, and evidence-bearing claims, diff --git a/capabilities/analysis-ledger/items.py b/capabilities/analysis-ledger/items.py index 1d923cb..7f2d8df 100644 --- a/capabilities/analysis-ledger/items.py +++ b/capabilities/analysis-ledger/items.py @@ -60,29 +60,7 @@ class AnalysisRun(BaseModel): """One bounded analysis of a repository revision.""" - model_config = ConfigDict( - extra="forbid", - json_schema_extra={ - "allOf": [ - { - "if": { - "properties": { - "run_state": { - "enum": ["paused", "completed", "exhausted", "stale"] - } - }, - "required": ["run_state"], - }, - "then": { - "required": ["stop_reason"], - "properties": { - "stop_reason": {"type": "string", "minLength": 1} - }, - }, - } - ] - }, - ) + model_config = ConfigDict(extra="forbid") title: str = Field(..., min_length=1, max_length=512) run_key: RunKey = Field( @@ -157,40 +135,7 @@ def validate_run_invariants(self) -> "AnalysisRun": class AnalysisTarget(BaseModel): """One bounded code unit or path waiting on the analysis frontier.""" - model_config = ConfigDict( - extra="forbid", - json_schema_extra={ - "allOf": [ - { - "if": { - "properties": { - "target_state": {"enum": ["analyzed", "blocked", "skipped"]} - }, - "required": ["target_state"], - }, - "then": { - "required": ["summary"], - "properties": {"summary": {"type": "string", "minLength": 1}}, - }, - }, - { - "if": { - "properties": {"target_state": {"const": "analyzed"}}, - "required": ["target_state"], - }, - "then": { - "required": ["evidence_refs"], - "properties": { - "evidence_refs": { - "type": "array", - "minItems": 1, - } - }, - }, - }, - ] - }, - ) + model_config = ConfigDict(extra="forbid") title: str = Field(..., min_length=1, max_length=512) run_ref: str = Field( @@ -255,38 +200,7 @@ def require_terminal_summary(self) -> "AnalysisTarget": class AnalysisClaim(BaseModel): """An evidence-bearing hypothesis or conclusion about one analysis target.""" - model_config = ConfigDict( - extra="forbid", - json_schema_extra={ - "allOf": [ - { - "if": { - "properties": { - "disposition": {"const": "verified"}, - "claim_category": {"const": "vulnerability"}, - }, - "required": ["disposition", "claim_category"], - }, - "then": { - "required": ["weakness", "severity", "impact"], - "properties": { - "weakness": {"type": "string", "minLength": 1}, - "severity": { - "enum": [ - "critical", - "high", - "medium", - "low", - "info", - ] - }, - "impact": {"type": "string", "minLength": 1}, - }, - }, - } - ] - }, - ) + model_config = ConfigDict(extra="forbid") title: str = Field(..., min_length=1, max_length=512) run_ref: str = Field( diff --git a/capabilities/analysis-ledger/tests/test_analysis_ledger.py b/capabilities/analysis-ledger/tests/test_analysis_ledger.py index 464d4a1..a340516 100644 --- a/capabilities/analysis-ledger/tests/test_analysis_ledger.py +++ b/capabilities/analysis-ledger/tests/test_analysis_ledger.py @@ -6,7 +6,6 @@ from pathlib import Path from types import SimpleNamespace -import jsonschema import pytest import yaml @@ -185,50 +184,121 @@ def test_item_schemas_reject_unknown_fields() -> None: ) -def test_exported_json_schemas_enforce_terminal_invariants() -> None: +UNSUPPORTED_SCHEMA_KEYWORDS = {"allOf", "if", "then", "else", "not"} + + +def _schema_keywords(node: t.Any) -> set[str]: + if isinstance(node, dict): + found = set(node) + for value in node.values(): + found |= _schema_keywords(value) + return found + if isinstance(node, list): + found = set() + for value in node: + found |= _schema_keywords(value) + return found + return set() + + +@pytest.mark.parametrize( + "model_name", ["AnalysisRun", "AnalysisTarget", "AnalysisClaim"] +) +def test_exported_json_schemas_use_only_supported_keywords(model_name: str) -> None: items = _load_module("analysis_ledger_json_schemas", "items.py") - with pytest.raises(jsonschema.ValidationError): - jsonschema.validate( - { - "title": "Authentication analysis", - "run_key": "0123456789abcdef01234567", - "objective": "Find authentication bypasses", - "repository": "/repo", - "revision": "deadbeef", - "run_state": "completed", - }, - items.AnalysisRun.model_json_schema(), + schema = getattr(items, model_name).model_json_schema() + + assert not _schema_keywords(schema) & UNSUPPORTED_SCHEMA_KEYWORDS + + +def _run_payload(**overrides: t.Any) -> dict[str, t.Any]: + run_material = "/repo\ndeadbeef\nFind authentication bypasses" + return { + "title": "Authentication analysis", + "run_key": sha256(run_material.encode()).hexdigest()[:24], + "objective": "Find authentication bypasses", + "repository": "/repo", + "revision": "deadbeef", + **overrides, + } + + +def _target_payload(**overrides: t.Any) -> dict[str, t.Any]: + return { + "title": "Analyze login", + "run_ref": "run-auth", + "target_key": "src/auth.py::login", + "kind": "function", + "location": "src/auth.py:40", + "rationale": "Externally reachable authentication path", + **overrides, + } + + +@pytest.mark.parametrize("run_state", ["paused", "completed", "exhausted", "stale"]) +def test_run_validator_requires_stop_reason(run_state: str) -> None: + items = _load_module("analysis_ledger_run_invariants", "items.py") + + with pytest.raises(ValueError, match="requires stop_reason"): + items.AnalysisRun.model_validate(_run_payload(run_state=run_state)) + + run = items.AnalysisRun.model_validate( + _run_payload(run_state=run_state, stop_reason="Frontier exhausted.") + ) + assert run.stop_reason == "Frontier exhausted." + + +@pytest.mark.parametrize("target_state", ["analyzed", "blocked", "skipped"]) +def test_target_validator_requires_summary(target_state: str) -> None: + items = _load_module("analysis_ledger_target_summary", "items.py") + + with pytest.raises(ValueError, match="requires summary"): + items.AnalysisTarget.model_validate( + _target_payload(target_state=target_state, evidence_refs=["src/auth.py:42"]) ) - with pytest.raises(jsonschema.ValidationError): - jsonschema.validate( - { - "title": "Verified SQL injection", - "run_ref": "run-auth", - "target_ref": "run-auth-target-login", - "claim_category": "vulnerability", - "statement": "User input reaches the query.", - "disposition": "verified", - "evidence_refs": ["src/auth.py:42"], - }, - items.AnalysisClaim.model_json_schema(), + +def test_target_validator_requires_evidence_refs_when_analyzed() -> None: + items = _load_module("analysis_ledger_target_evidence", "items.py") + + with pytest.raises(ValueError, match="requires evidence_refs"): + items.AnalysisTarget.model_validate( + _target_payload(target_state="analyzed", summary="No bypass found.") ) - with pytest.raises(jsonschema.ValidationError): - jsonschema.validate( - { - "title": "Analyze login", - "run_ref": "run-auth", - "target_key": "src/auth.py::login", - "kind": "function", - "location": "src/auth.py:40", - "rationale": "Externally reachable authentication path", - "target_state": "analyzed", - "summary": "No bypass found.", - }, - items.AnalysisTarget.model_json_schema(), + target = items.AnalysisTarget.model_validate( + _target_payload( + target_state="analyzed", + summary="No bypass found.", + evidence_refs=["src/auth.py:42"], ) + ) + assert target.evidence_refs == ["src/auth.py:42"] + + +@pytest.mark.parametrize("missing", ["weakness", "severity", "impact"]) +def test_claim_validator_requires_verified_vulnerability_fields(missing: str) -> None: + items = _load_module("analysis_ledger_claim_invariants", "items.py") + payload = { + "title": "Verified SQL injection", + "run_ref": "run-auth", + "target_ref": "run-auth-target-login", + "claim_category": "vulnerability", + "statement": "User input reaches the query.", + "disposition": "verified", + "evidence_refs": ["src/auth.py:42"], + "weakness": "CWE-89", + "severity": "high", + "impact": "Authentication bypass.", + } + items.AnalysisClaim.model_validate(payload) + + del payload[missing] + with pytest.raises( + ValueError, match=f"verified vulnerability claim requires {missing}" + ): + items.AnalysisClaim.model_validate(payload) def test_manifest_builds_combined_report_item_schema() -> None: