From 00826b7ec290f7a9b52166fc7080b11aa2dafad3 Mon Sep 17 00:00:00 2001 From: pcvantol Date: Thu, 10 Sep 2026 19:10:10 +0200 Subject: [PATCH] Enforce strict current validation before publication --- .../engineering-platform-validation.yml | 10 +- .../CONSOLIDATION_PARKING_2026_09_10.md | 16 + .../ENGINEERING_PLATFORM_ROADMAP.md | 19 +- package-lock.json | 4 +- package.json | 2 +- pyproject.toml | 2 +- .../ENGINEERING_PLATFORM_CONFIG.json | 2 +- .../ENGINEERING_PLATFORM_VERSION.json | 10 +- src/engineering_platform/agent_state.py | 11 +- .../contracts/projection.py | 87 ++--- src/engineering_platform/execution_host.py | 216 ++++++++++- src/engineering_platform/managed_autonomy.py | 18 +- src/engineering_platform/platform_version.py | 2 +- .../qualification_runtime.py | 27 +- src/engineering_platform/storage.py | 169 ++++++++- .../templates/workspace-config.json | 2 +- .../validation_profile.py | 146 ++++++++ tests/engineering/test_contract_projection.py | 13 + .../test_deterministic_execution_e2e.py | 18 + tests/engineering/test_execution_host.py | 63 +++- tests/engineering/test_managed_autonomy.py | 12 + ...p_neutral_consumer_credential_namespace.py | 2 +- .../engineering/test_qualification_runtime.py | 25 ++ .../test_recovery_cross_surface.py | 16 +- tests/engineering/test_storage.py | 1 + .../test_strict_current_validation.py | 343 ++++++++++++++++++ tests/engineering/test_validation_profile.py | 17 +- .../p_deterministic_execution_e2e.py | 10 +- 28 files changed, 1150 insertions(+), 113 deletions(-) create mode 100644 tests/engineering/test_strict_current_validation.py diff --git a/.github/workflows/engineering-platform-validation.yml b/.github/workflows/engineering-platform-validation.yml index 0dd8ed7c..500ccb92 100644 --- a/.github/workflows/engineering-platform-validation.yml +++ b/.github/workflows/engineering-platform-validation.yml @@ -43,7 +43,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v5 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: {python-version: '3.12'} + with: {python-version: '3.14'} - name: Validate committed offline projection run: python3 docs/ai-development/validate_projection.py --profile engineering-platform --source-commit 6ec3b443c3ab3bdf76c626c2046d3778db570eb0 --extension-identity ENGINEERING_PLATFORM_DEVELOPMENT_EXTENSION validation-profile: @@ -56,6 +56,8 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v5 with: {fetch-depth: 0} + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: {python-version: '3.14'} - id: profile run: PYTHONPATH=src python3 -m engineering_platform.validation_profile --base "${{ github.event.pull_request.base.sha || github.event.before }}" --branch "${{ github.head_ref || github.ref_name }}" --github-output "$GITHUB_OUTPUT" validate: @@ -67,7 +69,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v5 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: {python-version: '3.12'} + with: {python-version: '3.14'} - name: Build and install committed canonical Engineering Platform wheel run: | python3 -m pip install build coverage @@ -169,7 +171,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v5 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: {python-version: '3.12'} + with: {python-version: '3.14'} - name: Install Engineering Platform runtime run: python3 -m pip install . - name: Install browser validation dependencies @@ -216,7 +218,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v5 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: {python-version: '3.12'} + with: {python-version: '3.14'} - name: Verify standalone qualification evidence run: | python -m pip install build . diff --git a/docs/development/CONSOLIDATION_PARKING_2026_09_10.md b/docs/development/CONSOLIDATION_PARKING_2026_09_10.md index c47bde0b..959c7c33 100644 --- a/docs/development/CONSOLIDATION_PARKING_2026_09_10.md +++ b/docs/development/CONSOLIDATION_PARKING_2026_09_10.md @@ -135,6 +135,22 @@ of QR-LOCAL-VALIDATION-PASS-001 and SEC-001 is not inferred from code/tests. Host sequencing, phase prompts and early-result rejection are not universal capability-level prevention of every HTTP/credential remote-write route. +## Minimal strict-validation extraction for first-canary readiness + +`STRICT_CURRENT_CONTROL_VALIDATION_SLICE = SOURCE_DELIVERED / QUALIFIED` for +the separately delivered minimal source increment. The normal Managed +first-publication gate now requires the current validated audit plus an exact +candidate-bound, repair-ordinal-bound persisted profile digest and explicit +terminal PASS receipts for every profile-owned required control. Missing, +stale, skipped, not-applicable, unavailable, prose-only, nonzero or malformed +receipts remain ineligible; Quality/Security PASS cannot compensate. + +This extraction does not resume or qualify the parked publication programme: +`EP #175 = OPEN_DRAFT_PARKED_NOT_QUALIFIED`, unchanged at its retained head. +Its remaining candidate-adoption and duplicate-safe publication-recovery scope +is `PARKED_NON_BLOCKER_FOR_FIRST_CANARY`. No primary runtime/CENTRAL, +configuration, governance, installer, submission or canary state was changed. + ## Parked decisions and documentary DAG | Node | Disposition and retained future acceptance | diff --git a/docs/development/ENGINEERING_PLATFORM_ROADMAP.md b/docs/development/ENGINEERING_PLATFORM_ROADMAP.md index 3f7ac6fc..0a1d440e 100644 --- a/docs/development/ENGINEERING_PLATFORM_ROADMAP.md +++ b/docs/development/ENGINEERING_PLATFORM_ROADMAP.md @@ -9,7 +9,7 @@ moved the canonical source projections forward to 2.3.2 without rewriting that published identity. The observed Mac has one 2.3.1 server process/data root and a distinct 2.3.0 PlatformIO PATH candidate. This is explicit-path investigation evidence, not proof of Mac-wide uniqueness and not authorization to remove or -cut over either installation. CENTRAL schema 56, engineering-storage schema 41 +cut over either installation. CENTRAL schema 56, engineering-storage schema 42 and repository-attachment schema 1.0 remain separate contracts. | Increment | Owning repository | Bounded result | Dependencies / acceptance | @@ -65,6 +65,23 @@ receipts remain product-specific. A production publication or an actual Mac installation/update remains outside this source order until separately authorized. +## First-canary strict current-control validation slice + +`STRICT_CURRENT_CONTROL_VALIDATION_SLICE = SOURCE_DELIVERED / QUALIFIED`. +Normal Managed first-publication eligibility now consumes one append-only, +candidate-bound validation-profile identity for the current run-wide repair +ordinal. The identity covers the selected tier, profile version/reference and +selection source, exact required-control set and launcher bindings. Every +required control must have one current terminal receipt with `EXECUTED`, +`PASS`, exit code zero and valid start/end timestamps; provider summaries and +Quality/Security passes cannot replace that evidence. + +This is only the minimal first-canary product-capability closure. It creates no +submission, configuration, grant, installation, release or canary run. PR #175 +remains `OPEN_DRAFT_PARKED_NOT_QUALIFIED`; its candidate-adoption, +publication-recovery and remaining scope are +`PARKED_NON_BLOCKER_FOR_FIRST_CANARY` and were not changed or qualified here. + ## Subagent orchestration and efficiency — retained audit and planned lane `EP_SUBAGENT_ORCHESTRATION_AND_EFFICIENCY_V1` records the diff --git a/package-lock.json b/package-lock.json index b93b112a..0d55fc27 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "engineering-platform-browser-validation", - "version": "2.3.2", + "version": "2.3.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "engineering-platform-browser-validation", - "version": "2.3.2", + "version": "2.3.4", "devDependencies": { "@playwright/test": "1.62.1" } diff --git a/package.json b/package.json index 89b2b234..aee97281 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "engineering-platform-browser-validation", "private": true, - "version": "2.3.2", + "version": "2.3.4", "scripts": { "test:engineering-dashboard": "PYTHONPATH=src python3 -m engineering_platform.dashboard_browser_validation", "test:engineering-dashboard-logic": "node --test tests/engineering/dashboard_status_store.test.mjs tests/engineering/ui_localization_contract.test.mjs", diff --git a/pyproject.toml b/pyproject.toml index dfb3d570..e3928066 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "engineering-platform" -version = "2.3.2" +version = "2.3.4" description = "Local-first Engineering Platform execution operations runtime" readme = "README.md" requires-python = ">=3.11" diff --git a/src/engineering_platform/ENGINEERING_PLATFORM_CONFIG.json b/src/engineering_platform/ENGINEERING_PLATFORM_CONFIG.json index 8d0541c9..b2e81db9 100644 --- a/src/engineering_platform/ENGINEERING_PLATFORM_CONFIG.json +++ b/src/engineering_platform/ENGINEERING_PLATFORM_CONFIG.json @@ -3,7 +3,7 @@ "platform": { "id": "engineering-platform", "name": "Engineering Platform", - "version": "2.3.2", + "version": "2.3.4", "generation": 2, "documentation_namespace": "engineering-platform", "capability_registry_version": 1 diff --git a/src/engineering_platform/ENGINEERING_PLATFORM_VERSION.json b/src/engineering_platform/ENGINEERING_PLATFORM_VERSION.json index fe639b4d..e4c40fcd 100644 --- a/src/engineering_platform/ENGINEERING_PLATFORM_VERSION.json +++ b/src/engineering_platform/ENGINEERING_PLATFORM_VERSION.json @@ -1,15 +1,15 @@ { "bootstrap_contract": "2026.12", "checkpoint_format": 1, - "dashboard_version": "2.3.2", + "dashboard_version": "2.3.4", "handoff_protocol": 1, "memory_format": 2, "minimum_codex_cli": "0.146.0", "inbox_protocol": 1, - "platform_version": "2.3.2", + "platform_version": "2.3.4", "report_format": 2, - "runner_version": "2.3.2", + "runner_version": "2.3.4", "status_model": 1, - "storage_schema": 41, - "watcher_version": "2.3.2" + "storage_schema": 42, + "watcher_version": "2.3.4" } diff --git a/src/engineering_platform/agent_state.py b/src/engineering_platform/agent_state.py index f0c46e72..7d384415 100644 --- a/src/engineering_platform/agent_state.py +++ b/src/engineering_platform/agent_state.py @@ -379,12 +379,21 @@ def from_dict(cls, raw: object) -> "TransactionState": raise StateError("checkpoint quality evidence is invalid or unsafe") profile_fields = {"version", "digest", "candidate_sha"} current_profile_fields = profile_fields | {"criteria_digest"} + validation_bound_profile_fields = current_profile_fields | {"validation_profile_digest"} if state.assurance_profile is not None and ( not isinstance(state.assurance_profile, dict) - or set(state.assurance_profile) not in (profile_fields, current_profile_fields) + or set(state.assurance_profile) not in ( + profile_fields, current_profile_fields, validation_bound_profile_fields, + ) or not all(isinstance(value, str) and value for value in state.assurance_profile.values()) or not re.fullmatch(r"sha256:[0-9a-f]{64}", state.assurance_profile["digest"]) or not re.fullmatch(r"[0-9a-f]{40}", state.assurance_profile["candidate_sha"]) + or ( + "validation_profile_digest" in state.assurance_profile + and not re.fullmatch( + r"sha256:[0-9a-f]{64}", state.assurance_profile["validation_profile_digest"], + ) + ) ): raise StateError("checkpoint assurance profile is invalid") review_fields = {"reviewer", "status", "candidate_sha", "profile_digest", "invocation_id", "findings"} diff --git a/src/engineering_platform/contracts/projection.py b/src/engineering_platform/contracts/projection.py index e3fa517a..422010ec 100644 --- a/src/engineering_platform/contracts/projection.py +++ b/src/engineering_platform/contracts/projection.py @@ -21,7 +21,8 @@ EvidenceReference, require_compatible_version, ) -from ..storage import EngineeringStorageError, database_path +from ..storage import EngineeringStorageError, database_path, load_validation_context +from ..validation_profile import strict_required_controls_pass UNAVAILABLE = "UNAVAILABLE" @@ -137,65 +138,51 @@ def _qualification_evidence(connection: sqlite3.Connection, run_id: str) -> tupl "SELECT submission_id,fresh_submission,retry_parent_run_id,resume_parent_run_id,recorded_at " "FROM execution_run_qualification_context WHERE run_id=?", (run_id,) ).fetchone() - profile = connection.execute( - "SELECT selected_validation_tier,validation_profile_version,required_validation_controls,recorded_at " - "FROM execution_validation_profiles WHERE run_id=?", (run_id,) - ).fetchone() - controls = connection.execute( - "SELECT validation_id,category,required_for_profile,execution_status,result,observed_at,currentness " - "FROM execution_validation_control_results WHERE run_id=? ORDER BY id", (run_id,) - ).fetchall() - commands = connection.execute( - "SELECT inv.validation_id,inv.category,inv.required_for_profile,'EXECUTED'," - "COALESCE(term.result,'UNAVAILABLE'),COALESCE(term.completed_at,inv.started_at),inv.currentness " - "FROM execution_validation_command_invocations inv LEFT JOIN execution_validation_command_terminals term " - "ON term.run_id=inv.run_id AND term.command_id=inv.command_id WHERE inv.run_id=? ORDER BY inv.started_at", - (run_id,), - ).fetchall() + database = Path(str(connection.execute("PRAGMA database_list").fetchone()[2])) + validation = load_validation_context(Path("."), run_id, central_database=database) except sqlite3.OperationalError: return None, None + except EngineeringStorageError: + validation = None lineage_projection = None if lineage is None else { "submission_id": lineage[0], "fresh_submission": bool(lineage[1]), "retry_parent": lineage[2], "resume_parent": lineage[3], "recorded_at": lineage[4], } - if profile is None: + if validation is None: return lineage_projection, None - try: - payload = json.loads(profile[2]) - required = payload.get("validation_ids", []) - except (AttributeError, TypeError, json.JSONDecodeError): - return lineage_projection, None - if not isinstance(required, list) or not all(isinstance(item, str) for item in required): - return lineage_projection, None - current: dict[str, sqlite3.Row] = {} - conflicts: set[str] = set() - for row in (*controls, *commands): - validation_id = str(row[0]) - existing = current.get(validation_id) - if existing is None or int(row[6]) > int(existing[6]): - current[validation_id] = row - elif int(row[6]) == int(existing[6]) and row[4] != existing[4]: - conflicts.add(validation_id) - def result_for(validation_id: str) -> object: - if validation_id in conflicts: - return "UNRESOLVED" - row = current.get(validation_id) - return row[4] if row is not None else None - results = [result_for(validation_id) for validation_id in required] - required_state = "FAIL" if any(result == "FAIL" for result in results) else ( - "PASS" if results and all(result == "PASS" for result in results) else "UNRESOLVED" + required = validation["required_validation_controls"] + controls = validation["controls"] + candidate = validation.get("candidate_sha") + currentness = validation.get("currentness") + strict_pass = ( + isinstance(candidate, str) and isinstance(currentness, int) + and strict_required_controls_pass( + validation, candidate_sha=candidate, currentness=currentness, + ) + ) + results = [controls.get(item, {}).get("result") for item in required] + required_state = ( + "PASS" if strict_pass else "FAIL" if any(result == "FAIL" for result in results) + else "UNRESOLVED" ) return lineage_projection, { - "selected_validation_tier": profile[0], "validation_profile_version": profile[1], - "profile_reference": payload.get("profile_reference", UNAVAILABLE), - "profile_selection_source": payload.get("profile_selection_source", UNAVAILABLE), - "required_validation_controls": required, "required_validation_state": required_state, - "control_bindings": payload.get("control_bindings", UNAVAILABLE), - "recorded_at": profile[3], + "selected_validation_tier": validation["selected_validation_tier"], + "validation_profile_version": validation["validation_profile_version"], + "profile_reference": validation["profile_reference"], + "profile_selection_source": validation["profile_selection_source"], + "profile_digest": validation.get("profile_digest") or UNAVAILABLE, + "candidate_sha": candidate or UNAVAILABLE, + "currentness": currentness if isinstance(currentness, int) else UNAVAILABLE, + "required_validation_controls": list(required), "required_validation_state": required_state, + "control_bindings": list(validation["control_bindings"]), + "recorded_at": validation["recorded_at"], "controls": [ - {"validation_id": validation_id, "category": row[1], "required_for_profile": bool(row[2]), - "execution_status": row[3], "result": result_for(validation_id), "observed_at": row[5]} - for validation_id, row in sorted(current.items()) + {"validation_id": validation_id, "category": row.get("category", UNAVAILABLE), + "required_for_profile": row.get("required_for_profile", False), + "execution_status": row.get("execution_status", "UNRESOLVED"), + "result": row.get("result", "UNRESOLVED"), + "observed_at": row.get("observed_at", UNAVAILABLE)} + for validation_id, row in sorted(controls.items()) ], } diff --git a/src/engineering_platform/execution_host.py b/src/engineering_platform/execution_host.py index de684999..1de2c271 100644 --- a/src/engineering_platform/execution_host.py +++ b/src/engineering_platform/execution_host.py @@ -74,7 +74,8 @@ from .validation_profile import ( VALIDATION_PROFILE_VERSION, ValidationControlLauncher, ValidationProfile, ValidationProfileResolutionError, changed_paths, classify, - profile_control_bindings, resolve_producer_profile, + matching_control_binding, profile_control_bindings, resolve_producer_profile, + strict_required_controls_pass, ) from .reviewer_evidence import ReviewerEvidence from .investigation_ledger import InvocationInvestigationLedger @@ -202,6 +203,65 @@ def invoke(self, root: Path, prompt: str) -> AgentResult: ... _format_cli_failure = executor_format_cli_failure +def _validation_profile_digest(validation_context: object) -> str | None: + """Return only a storage-verified candidate-bound profile digest.""" + if not isinstance(validation_context, dict): + return None + digest = validation_context.get("profile_digest") + candidate = validation_context.get("candidate_sha") + currentness = validation_context.get("currentness") + if ( + not isinstance(digest, str) + or re.fullmatch(r"sha256:[0-9a-f]{64}", digest) is None + or not isinstance(candidate, str) + or re.fullmatch(r"[0-9a-f]{40}", candidate) is None + or isinstance(currentness, bool) + or not isinstance(currentness, int) + or currentness < 0 + ): + return None + return digest + + +def _required_validation_controls_pass( + state: TransactionState, validation_context: object, +) -> bool: + """Require the exact current ordinal's candidate-bound terminal receipts.""" + if not isinstance(validation_context, dict): + return False + profile = state.assurance_profile + candidate = ( + profile.get("candidate_sha") if isinstance(profile, dict) + else validation_context.get("candidate_sha") + ) + if not isinstance(candidate, str): + return False + return strict_required_controls_pass( + validation_context, candidate_sha=candidate, + currentness=state.repair_iterations, + ) + + +def _has_current_local_validation_evidence( + state: TransactionState, validation_context: object, +) -> bool: + """Bind current validation audit, profile, candidate and receipts.""" + if not isinstance(validation_context, dict) or not state.local_validation_audit: + return False + if state.local_validation_audit[-1].get("outcome") != "validated": + return False + profile = state.assurance_profile + if not isinstance(profile, dict): + return False + digest = _validation_profile_digest(validation_context) + return ( + digest is not None + and profile.get("validation_profile_digest") == digest + and profile.get("candidate_sha") == validation_context.get("candidate_sha") + and _required_validation_controls_pass(state, validation_context) + ) + + def assemble_prompt( prompt_path: Path, state: TransactionState | None, @@ -739,13 +799,14 @@ def _record_validation_evidence(self, state: TransactionState, result: AgentResu required=True, currentness=state.repair_iterations, central_database=self.store.central_database, ) - validation_id = ( - "git_diff_check" if kind == "format_or_diff" else - "documentation_contract" if kind == "documentation_contract" else - self._validation_id(command, kind) if kind == "browser_e2e" else - "repository_suite" if kind == "tests" and tier == "FULL" else - "engineering_python" if kind == "tests" else f"validation_{kind}" - ) + if profile_context and profile_context.get("profile_digest"): + binding = matching_control_binding(command, profile_context["control_bindings"]) + validation_id = ( + str(binding["validation_id"]) + if isinstance(binding, dict) else self._validation_id(command, kind) + ) + else: + validation_id = self._validation_id_for_profile(command, kind, tier) record_validation_control_result( self.root, run_id=state.run_id, validation_id=validation_id, category="agent", control_identity=command[:160], required_for_profile=validation_id in required_controls, execution_status="EXECUTED", @@ -1203,6 +1264,10 @@ def _validation_kind(command: str) -> str | None: return "static_analysis" if any(tool in normalized for tool in ("bandit", "semgrep", "codeql", "pip-audit", "safety")): return "security" + if "console_route_ownership_guard.py" in normalized: + return "console_route_ownership" + if "npm run test:ui-localization" in normalized: + return "ui_localization" if "git diff --check" in normalized or "prettier" in normalized or "black --check" in normalized: return "format_or_diff" if any(tool in normalized for tool in ("npm run test:engineering-dashboard", "playwright", "selenium", "cypress", "e2e")): @@ -1218,6 +1283,17 @@ def _validation_id(command: str, kind: str) -> str: return "dashboard_browser" return f"validation_{kind}" + @classmethod + def _validation_id_for_profile(cls, command: str, kind: str, tier: object) -> str: + """Map an observed command to its persisted profile-owned control.""" + if kind == "format_or_diff": + return "git_diff_check" + if kind in {"documentation_contract", "console_route_ownership", "ui_localization"}: + return kind + if kind == "tests" and isinstance(tier, str): + return "repository_suite" if tier == "FULL" else "engineering_python" + return cls._validation_id(command, kind) + def _invoke_provider_attempt_with_timing(self, state: TransactionState, prompt: str, *, repair: bool = False, quality: bool = False, local_validation: bool = False, attempt: int | None = None) -> AgentResult: """Run one provider attempt; recovery launch authority lives in storage.""" # Baseline capture is content-free and idempotent. Recovery progress @@ -1272,12 +1348,26 @@ def command_boundary(event: str, command_id: str, command: str, exit_code: int | if kind is not None: validation_id = self._validation_id(command, kind) try: - profile = load_validation_context(self.root, state.run_id, central_database=self.store.central_database) - required = validation_id in set(profile["required_validation_controls"]) if profile else False + profile = load_validation_context( + self.root, state.run_id, currentness=state.repair_iterations, + central_database=self.store.central_database, + ) + binding = matching_control_binding( + command, profile.get("control_bindings", ()), + ) if profile else None + if isinstance(binding, dict): + validation_id = str(binding["validation_id"]) + required = validation_id in set(profile["required_validation_controls"]) + category = str(binding["category"]) + identity = str(binding["control_identity"]) + else: + required = False + category = "agent" + identity = command[:160] started_at = datetime.now(timezone.utc).isoformat() record_validation_command_invocation( self.root, run_id=state.run_id, validation_id=validation_id, command_id=command_id, - category="agent", control_identity=command[:160], required_for_profile=required, + category=category, control_identity=identity, required_for_profile=required, started_at=started_at, currentness=state.repair_iterations, central_database=self.store.central_database, ) @@ -1676,6 +1766,23 @@ def _run_local_repository_validation( return self._save_terminal( state, "BLOCKED", "local_validation_scope", "Implementation must return one branch and no pull request before local validation." ), implementation + try: + candidate = self.repository.inspect(self.root) + except RunnerError: + return self._save_terminal( + state, "BLOCKED", "local_validation_scope", + "Implementation candidate could not be inspected before local validation.", + ), implementation + if ( + not candidate.clean + or candidate.branch != branch + or re.fullmatch(r"[0-9a-f]{40}", candidate.head_sha) is None + or (implementation.commit_sha is not None and implementation.commit_sha != candidate.head_sha) + ): + return self._save_terminal( + state, "BLOCKED", "local_validation_scope", + "Local validation requires the exact clean implementation branch and candidate SHA.", + ), implementation validation = replace( state, phase="LOCAL_REPOSITORY_VALIDATION", branch=branch, pull_request=None, next_action="run_local_repository_validation", local_validation_iterations=0, @@ -1697,6 +1804,8 @@ def _run_local_repository_validation( profile_reference=f"validation-profile-registry:{profile.tier}@{VALIDATION_PROFILE_VERSION}", profile_selection_source="diff_classification", control_bindings=profile_control_bindings(profile), + candidate_sha=candidate.head_sha, + currentness=validation.repair_iterations, recorded_at=datetime.now(timezone.utc).isoformat(), central_database=self.store.central_database, ) @@ -1733,6 +1842,16 @@ def _run_local_repository_validation( description="local_repository_validation_commit_verified", ) self._persist_agent_usage(validation.run_id) + try: + validation_context = load_validation_context( + self.root, validation.run_id, + currentness=validation.repair_iterations, + central_database=self.store.central_database, + ) + after_validation = self.repository.inspect(self.root) + except (EngineeringStorageError, RunnerError): + validation_context = None + after_validation = None except ProviderReadinessBlocked as blocked: return blocked.state, implementation except CodexInvocationError as error: @@ -1755,12 +1874,41 @@ def _run_local_repository_validation( validation, "BLOCKED", "implementation_pr_before_assurance", "Read-only local validation returned a pull request before mandatory assurance.", ), implementation - if result.terminal_state == "COMPLETE" and result.validation_evidence and not self._has_failed_validation_evidence(result): + candidate_unchanged = ( + after_validation is not None + and after_validation.clean + and after_validation.branch == branch + and after_validation.head_sha == candidate.head_sha + ) + if ( + result.terminal_state == "COMPLETE" + and result.validation_evidence + and not self._has_failed_validation_evidence(result) + and candidate_unchanged + and _required_validation_controls_pass(validation, validation_context) + ): validation = self._record_local_validation_audit(validation, result=result, outcome="validated", profile=profile) return validation, replace( result, branch=branch, pull_request=None, validation_evidence=implementation.validation_evidence + result.validation_evidence, ) + if result.terminal_state == "COMPLETE" and not self._has_failed_validation_evidence(result): + controls = validation_context.get("controls", {}) if isinstance(validation_context, dict) else {} + required = validation_context.get("required_validation_controls", ()) if isinstance(validation_context, dict) else () + has_current_failure = any( + isinstance(controls.get(control), dict) + and controls[control].get("currentness") == validation.repair_iterations + and controls[control].get("result") == "FAIL" + for control in required + ) + if not has_current_failure: + validation = self._record_local_validation_audit( + validation, result=result, outcome="agent_failed", profile=profile, + ) + return self._save_terminal( + validation, "BLOCKED", "required_validation_unresolved", + "Local validation did not produce current successful terminal evidence for every required control.", + ), implementation validation = self._record_local_validation_audit(validation, result=result, outcome="validation_failed", profile=profile) if self._is_environmental_validation_instability(result): return self._save_terminal( @@ -1769,6 +1917,11 @@ def _run_local_repository_validation( "validation_infrastructure_recovery_required", "Required local validation is unstable: a failed required suite and a passing isolated rerun were recorded without an implementation correction. Preserve this run and create a separate validation-infrastructure recovery item.", ), implementation + if result.terminal_state == "COMPLETE": + result = replace( + result, terminal_state="FAILED", + diagnostic="A required current validation control did not pass.", + ) return validation, result def _run_quality_assurance( @@ -1798,15 +1951,31 @@ def _run_quality_assurance( profile_version = f"validation-profile@{VALIDATION_PROFILE_VERSION}" criteria = Path(quality.prompt_path).read_text(encoding="utf-8") criteria_digest = "sha256:" + hashlib.sha256(criteria.encode("utf-8")).hexdigest() + try: + validation_context = load_validation_context( + self.root, quality.run_id, currentness=quality.repair_iterations, + central_database=self.store.central_database, + ) + except EngineeringStorageError: + validation_context = None + validation_profile_digest = _validation_profile_digest(validation_context) # Candidate identity and policy identity are intentionally independent: # a source revision must not silently select or weaken its own policy. profile_digest = "sha256:" + hashlib.sha256( - json.dumps({"baseline": profile_version, "criteria_digest": criteria_digest}, sort_keys=True).encode("utf-8") + json.dumps({ + "baseline": profile_version, + "criteria_digest": criteria_digest, + "validation_profile_digest": validation_profile_digest, + }, sort_keys=True).encode("utf-8") ).hexdigest() - quality = replace(quality, assurance_profile={ + assurance_profile = { "version": profile_version, "digest": profile_digest, "candidate_sha": candidate.head_sha, "criteria_digest": criteria_digest, - }) + } + # Genesis deliberately has no host-owned local validation profile. + if validation_profile_digest is not None: + assurance_profile["validation_profile_digest"] = validation_profile_digest + quality = replace(quality, assurance_profile=assurance_profile) self.store.save(quality) write_live_status(self.root, quality, quality.next_action) evidence = ReviewerEvidence.from_repository(quality.run_id, quality.execution_mode, candidate) @@ -1937,6 +2106,18 @@ def _current_assurance_passes(state: TransactionState) -> bool: if isinstance(finding, dict) ) + def _validation_context(self, state: TransactionState) -> dict[str, object] | None: + try: + return load_validation_context( + self.root, state.run_id, currentness=state.repair_iterations, + central_database=self.store.central_database, + ) + except EngineeringStorageError: + return None + + def _current_local_validation_passes(self, state: TransactionState) -> bool: + return _has_current_local_validation_evidence(state, self._validation_context(state)) + def _publish_first_implementation_pull_request( self, state: TransactionState, implementation: AgentResult, ) -> tuple[TransactionState, AgentResult]: @@ -1949,6 +2130,11 @@ def _publish_first_implementation_pull_request( """ if state.execution_mode == "GENESIS" or state.pull_request or implementation.pull_request: return state, implementation + if not self._current_local_validation_passes(state): + return self._save_terminal( + state, "BLOCKED", "implementation_publication_assurance_required", + "First implementation PR publication requires current passing terminal receipts for the reviewed candidate.", + ), implementation if not self._current_assurance_passes(state): return self._save_terminal(state, "BLOCKED", "implementation_publication_assurance_required", "First implementation PR publication requires current passing quality and security assurance."), implementation try: diff --git a/src/engineering_platform/managed_autonomy.py b/src/engineering_platform/managed_autonomy.py index 9abf7af2..9dd7b28f 100644 --- a/src/engineering_platform/managed_autonomy.py +++ b/src/engineering_platform/managed_autonomy.py @@ -19,6 +19,7 @@ open_storage, record_run_qualification_snapshot, ) +from .validation_profile import strict_required_controls_pass AUTHORITIES = frozenset( { @@ -310,7 +311,22 @@ def terminal_snapshot( required = set(validation_context["required_validation_controls"]) controls = validation_context["controls"] results = [controls.get(control, {}).get("result") for control in required] - required_state = "FAIL" if any(result == "FAIL" for result in results) else "PASS" if results and all(result == "PASS" for result in results) else "UNRESOLVED" + candidate = validation_context.get("candidate_sha") + profile_currentness = validation_context.get("currentness") + strict_pass = ( + isinstance(candidate, str) + and isinstance(profile_currentness, int) + and strict_required_controls_pass( + validation_context, candidate_sha=candidate, + currentness=profile_currentness, + ) + ) + required_state = ( + "PASS" if strict_pass + else "PASS" if action_intent == "VALIDATION_ONLY" and results and all(result == "PASS" for result in results) + else "FAIL" if any(result == "FAIL" for result in results) + else "UNRESOLVED" + ) profile_projection = {key: validation_context[key] for key in ( "selected_validation_tier", "validation_profile_version", "profile_reference", "profile_selection_source", "required_validation_controls", "control_bindings", diff --git a/src/engineering_platform/platform_version.py b/src/engineering_platform/platform_version.py index d788b079..831c9287 100644 --- a/src/engineering_platform/platform_version.py +++ b/src/engineering_platform/platform_version.py @@ -12,7 +12,7 @@ SEMVER = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$") CONTRACT = re.compile(r"^(\d{4})\.(0[1-9]|1[0-2])$") -CURRENT_PLATFORM_VERSION = "2.3.2" +CURRENT_PLATFORM_VERSION = "2.3.4" MANIFEST_FIELDS = frozenset( { "platform_version", diff --git a/src/engineering_platform/qualification_runtime.py b/src/engineering_platform/qualification_runtime.py index b143d237..03988309 100644 --- a/src/engineering_platform/qualification_runtime.py +++ b/src/engineering_platform/qualification_runtime.py @@ -10,6 +10,7 @@ import os import re import subprocess +import sys import time from .capability_review import MANDATORY_REVIEW_OUTPUT_CONTRACT_VERSION, ReviewerResult @@ -19,10 +20,14 @@ class DeterministicQualificationAgent: def __init__(self) -> None: self._process_callback = None + self._command_callback = None def set_process_callback(self, callback: object) -> None: self._process_callback = callback + def set_command_callback(self, callback: object) -> None: + self._command_callback = callback + def wait_for_controlled_interruption_arm(self, _root: Path, state: object) -> None: """Offer the installed recovery E2E one bounded, non-production arm window.""" ready = os.environ.get("EP_QUALIFICATION_CONTROL_ARM_READY_FILE") @@ -160,9 +165,27 @@ def validate(self, root: Path, prompt: str) -> AgentResult: """ branch = subprocess.run(("git", "-C", str(root), "branch", "--show-current"), check=True, text=True, capture_output=True).stdout.strip() sha = subprocess.run(("git", "-C", str(root), "rev-parse", "HEAD"), check=True, text=True, capture_output=True).stdout.strip() + outcomes = [] + commands = ( + ("git diff --check", ("git", "diff", "--check")), + ("python3 -m unittest discover", (sys.executable, "-m", "unittest", "discover")), + ) + for ordinal, (identity, command) in enumerate(commands, start=1): + command_id = f"deterministic-validation-{ordinal}" + if callable(self._command_callback): + self._command_callback("started", command_id, identity) + completed = subprocess.run( + command, cwd=root, check=False, text=True, capture_output=True, + ) + if callable(self._command_callback): + self._command_callback("completed", command_id, "", completed.returncode) + outcomes.append({ + "command": identity, + "result": "passed" if completed.returncode == 0 else "failed", + }) return AgentResult( - "COMPLETE", branch=branch, commit_sha=sha, - validation_evidence=({"command": "deterministic installed validation", "result": "passed"},), + "COMPLETE" if all(item["result"] == "passed" for item in outcomes) else "FAILED", + branch=branch, commit_sha=sha, validation_evidence=tuple(outcomes), ) def review(self, _root: Path, selection: object, _objective: str, evidence: object = None) -> ReviewerResult: return ReviewerResult( diff --git a/src/engineering_platform/storage.py b/src/engineering_platform/storage.py index 8e49742c..342ef2f0 100644 --- a/src/engineering_platform/storage.py +++ b/src/engineering_platform/storage.py @@ -22,7 +22,7 @@ WORKSPACE_DIRECTORY = ".engineering" DATABASE_FILENAME = "engineering.db" CENTRAL_OPERATIONAL_DATABASE_FILENAME = "epdata.sqlite" -ENGINEERING_STORAGE_SCHEMA_VERSION = 41 +ENGINEERING_STORAGE_SCHEMA_VERSION = 42 STORE_AUTHORITY_POINTER = "store-authority.json" JOURNAL_MODES = frozenset({"DELETE", "MEMORY"}) LEGACY_DISMISSALS_PATH = Path(".engineering/status/execution_dismissals.json") @@ -1174,6 +1174,24 @@ def _schema_v41(connection: sqlite3.Connection) -> None: raise EngineeringStorageError("EP consumer credential migration source is absent.") +def _schema_v42(connection: sqlite3.Connection) -> None: + """Persist candidate-bound validation profile identities per repair ordinal.""" + connection.execute( + "CREATE TABLE IF NOT EXISTS execution_validation_profile_identities (" + "run_id TEXT NOT NULL,currentness INTEGER NOT NULL CHECK(currentness>=0)," + "candidate_sha TEXT NOT NULL CHECK(length(candidate_sha)=40)," + "profile_digest TEXT NOT NULL CHECK(length(profile_digest)=71)," + "identity_payload TEXT NOT NULL,recorded_at TEXT NOT NULL," + "PRIMARY KEY(run_id,currentness),UNIQUE(run_id,profile_digest))" + ) + for operation in ("UPDATE", "DELETE"): + connection.execute( + f"CREATE TRIGGER IF NOT EXISTS execution_validation_profile_identities_immutable_{operation.casefold()} " + f"BEFORE {operation} ON execution_validation_profile_identities BEGIN " + "SELECT RAISE(ABORT, 'Validation profile identity evidence is immutable.'); END" + ) + + def _import_legacy_execution_dismissals(root: Path, connection: sqlite3.Connection) -> None: """Copy valid legacy dismissal evidence into the canonical datastore. @@ -1262,6 +1280,7 @@ def _import_legacy_execution_dismissals(root: Path, connection: sqlite3.Connecti 39: _schema_v39, 40: _schema_v40, 41: _schema_v41, + 42: _schema_v42, } @@ -1786,8 +1805,9 @@ def record_validation_profile( required_validation_controls: tuple[str, ...], recorded_at: str, profile_reference: str | None = None, profile_selection_source: str | None = None, control_bindings: tuple[dict[str, object], ...] | None = None, + candidate_sha: str | None = None, currentness: int | None = None, central_database: Path | None = None, -) -> None: +) -> str | None: """Persist the exact mandatory controls before their execution evidence.""" if not run_id or not selected_validation_tier or not validation_profile_version or not recorded_at: raise EngineeringStorageError("Validation profile identity is invalid.") @@ -1808,18 +1828,52 @@ def record_validation_profile( binding_ids = tuple(binding.get("validation_id") for binding in control_bindings if isinstance(binding, dict)) if binding_ids != required_validation_controls: raise EngineeringStorageError("Validation profile bindings are invalid.") + resolved_reference = profile_reference or f"validation-profile-registry:{selected_validation_tier}@{validation_profile_version}" + resolved_source = profile_selection_source or "registry" payload = { "validation_ids": list(required_validation_controls), - "profile_reference": profile_reference or f"validation-profile-registry:{selected_validation_tier}@{validation_profile_version}", - "profile_selection_source": profile_selection_source or "registry", + "profile_reference": resolved_reference, + "profile_selection_source": resolved_source, "control_bindings": list(control_bindings), } connection = open_storage(root) if central_database is None else sqlite3.connect(central_database.resolve(), isolation_level=None) try: + profile_digest = None + identity_payload = None + if candidate_sha is not None or currentness is not None: + if candidate_sha is None or currentness is None: + raise EngineeringStorageError("Candidate-bound validation profile identity is incomplete.") + from .validation_profile import ValidationProfileResolutionError, validation_profile_identity + try: + identity, profile_digest = validation_profile_identity( + candidate_sha=candidate_sha, currentness=currentness, + selected_validation_tier=selected_validation_tier, + validation_profile_version=validation_profile_version, + profile_reference=resolved_reference, + profile_selection_source=resolved_source, + required_validation_controls=required_validation_controls, + control_bindings=control_bindings, + ) + except ValidationProfileResolutionError as error: + raise EngineeringStorageError("Candidate-bound validation profile identity is invalid.") from error + identity_payload = json.dumps(identity, sort_keys=True, separators=(",", ":")) connection.execute( "INSERT OR IGNORE INTO execution_validation_profiles(run_id,selected_validation_tier,validation_profile_version,required_validation_controls,recorded_at) VALUES(?,?,?,?,?)", (run_id, selected_validation_tier, validation_profile_version, _encoded_payload(payload), recorded_at), ) + if profile_digest is not None and identity_payload is not None: + connection.execute( + "INSERT OR IGNORE INTO execution_validation_profile_identities(" + "run_id,currentness,candidate_sha,profile_digest,identity_payload,recorded_at) VALUES(?,?,?,?,?,?)", + (run_id, currentness, candidate_sha, profile_digest, identity_payload, recorded_at), + ) + stored = connection.execute( + "SELECT candidate_sha,profile_digest,identity_payload FROM execution_validation_profile_identities " + "WHERE run_id=? AND currentness=?", (run_id, currentness), + ).fetchone() + if stored != (candidate_sha, profile_digest, identity_payload): + raise EngineeringStorageError("Validation profile ordinal already has a different immutable identity.") + return profile_digest finally: connection.close() @@ -1895,7 +1949,10 @@ def record_validation_command_terminal( connection.close() -def load_validation_context(root: Path, run_id: str, *, central_database: Path | None = None) -> dict[str, object] | None: +def load_validation_context( + root: Path, run_id: str, *, currentness: int | None = None, + central_database: Path | None = None, +) -> dict[str, object] | None: """Return the resolved profile and current control evidence without inference.""" connection = open_storage(root) if central_database is None else sqlite3.connect(central_database.resolve(), isolation_level=None) try: @@ -1903,6 +1960,13 @@ def load_validation_context(root: Path, run_id: str, *, central_database: Path | "SELECT selected_validation_tier,validation_profile_version,required_validation_controls,recorded_at " "FROM execution_validation_profiles WHERE run_id=?", (run_id,) ).fetchone() + strict_profile = connection.execute( + "SELECT currentness,candidate_sha,profile_digest,identity_payload,recorded_at " + "FROM execution_validation_profile_identities WHERE run_id=? " + + ("AND currentness=? " if currentness is not None else "") + + "ORDER BY currentness DESC LIMIT 1", + (run_id, currentness) if currentness is not None else (run_id,), + ).fetchone() rows = connection.execute( "SELECT validation_id,category,control_identity,required_for_profile,execution_status,result,evidence_ref,observed_at,currentness " "FROM execution_validation_control_results WHERE run_id=? ORDER BY id", (run_id,) @@ -1923,13 +1987,57 @@ def load_validation_context(root: Path, run_id: str, *, central_database: Path | } finally: connection.close() - if profile is None: + if strict_profile is None and profile is None: return None - try: - payload = json.loads(profile[2]) - required = payload.get("validation_ids", []) - except (TypeError, json.JSONDecodeError, AttributeError) as error: - raise EngineeringStorageError("Validation profile controls are corrupt.") from error + candidate_sha = profile_digest = profile_currentness = None + if strict_profile is not None: + try: + identity = json.loads(strict_profile[3]) + except (TypeError, json.JSONDecodeError) as error: + raise EngineeringStorageError("Validation profile identity is corrupt.") from error + if not isinstance(identity, dict): + raise EngineeringStorageError("Validation profile identity is invalid.") + from .validation_profile import ValidationProfileResolutionError, validation_profile_identity + try: + expected_identity, expected_digest = validation_profile_identity( + candidate_sha=identity["candidate_sha"], currentness=identity["currentness"], + selected_validation_tier=identity["selected_validation_tier"], + validation_profile_version=identity["validation_profile_version"], + profile_reference=identity["profile_reference"], + profile_selection_source=identity["profile_selection_source"], + required_validation_controls=tuple(identity["required_validation_controls"]), + control_bindings=tuple(identity["control_bindings"]), + ) + except (KeyError, TypeError, ValidationProfileResolutionError) as error: + raise EngineeringStorageError("Validation profile identity is invalid.") from error + canonical_identity = json.dumps(expected_identity, sort_keys=True, separators=(",", ":")) + if ( + strict_profile[0] != expected_identity["currentness"] + or strict_profile[1] != expected_identity["candidate_sha"] + or strict_profile[2] != expected_digest + or strict_profile[3] != canonical_identity + ): + raise EngineeringStorageError("Validation profile identity digest is invalid.") + selected_tier = expected_identity["selected_validation_tier"] + profile_version = expected_identity["validation_profile_version"] + payload = { + "validation_ids": expected_identity["required_validation_controls"], + "profile_reference": expected_identity["profile_reference"], + "profile_selection_source": expected_identity["profile_selection_source"], + "control_bindings": expected_identity["control_bindings"], + } + recorded_at = strict_profile[4] + candidate_sha = strict_profile[1] + profile_digest = strict_profile[2] + profile_currentness = strict_profile[0] + else: + assert profile is not None + try: + payload = json.loads(profile[2]) + except (TypeError, json.JSONDecodeError, AttributeError) as error: + raise EngineeringStorageError("Validation profile controls are corrupt.") from error + selected_tier, profile_version, recorded_at = profile[0], profile[1], profile[3] + required = payload.get("validation_ids", []) if not isinstance(required, list) or not all(isinstance(item, str) for item in required): raise EngineeringStorageError("Validation profile controls are invalid.") bindings = payload.get("control_bindings", []) @@ -1937,9 +2045,15 @@ def load_validation_context(root: Path, run_id: str, *, central_database: Path | raise EngineeringStorageError("Validation profile bindings are invalid.") if bindings and tuple(binding.get("validation_id") for binding in bindings) != tuple(required): raise EngineeringStorageError("Validation profile bindings are invalid.") + evidence_currentness = max( + [int(row[8]) for row in rows] + [int(row[6]) for row in command_rows], + default=profile_currentness if profile_currentness is not None else 0, + ) controls: dict[str, dict[str, object]] = {} for row in rows: validation_id, category, identity, is_required, status, result, evidence_ref, observed_at, currentness = row + if profile_currentness is not None and currentness != profile_currentness: + continue current = controls.get(validation_id) if current is None or int(currentness) > int(current["currentness"]): controls[validation_id] = {"validation_id": validation_id, "category": category, "control_identity": identity, @@ -1947,15 +2061,33 @@ def load_validation_context(root: Path, run_id: str, *, central_database: Path | "evidence_ref": evidence_ref, "observed_at": observed_at, "currentness": currentness} elif int(currentness) == int(current["currentness"]) and result != current["result"]: controls[validation_id] = {**current, "result": "UNRESOLVED", "conflict": True} + commands_by_control: dict[str, list[tuple[object, ...]]] = {} for row in command_rows: - validation_id, command_id, category, identity, is_required, started_at, currentness, completed_at, duration_ms, exit_code, result, terminal_ref = row + if profile_currentness is not None and row[6] != profile_currentness: + continue + commands_by_control.setdefault(str(row[0]), []).append(row) + for validation_id, command_records in commands_by_control.items(): + latest_currentness = max(int(row[6]) for row in command_records) + current_records = [row for row in command_records if int(row[6]) == latest_currentness] + if len(current_records) != 1: + controls[validation_id] = { + "validation_id": validation_id, "required_for_profile": True, + "execution_status": "UNRESOLVED", "result": "UNRESOLVED", + "currentness": latest_currentness, "conflict": True, + } + continue + row = current_records[0] + validation_id, command_id, category, identity, is_required, started_at, command_currentness, completed_at, duration_ms, exit_code, result, terminal_ref = row diagnostic_artifact_id = f"validation-failure-diagnostic-{command_id}" diagnostic_execution_id = diagnostic_artifacts.get(diagnostic_artifact_id) controls[validation_id] = { "validation_id": validation_id, "category": category, "control_identity": identity, - "required_for_profile": bool(is_required), "execution_status": "EXECUTED", + "required_for_profile": bool(is_required), + "execution_status": "EXECUTED" if completed_at is not None else "NOT_EXECUTED", "result": result or "UNAVAILABLE", "evidence_ref": terminal_ref if completed_at else "command_invocation", - "observed_at": completed_at or started_at, "currentness": currentness, + "evidence_authority": "command_terminal" if completed_at is not None else "command_invocation", + "observed_at": completed_at or started_at, "currentness": command_currentness, + "command_id": command_id, "started_at": started_at, "ended_at": completed_at, "duration_ms": duration_ms, "exit_code": exit_code, # The artifact id is deterministically derived from the immutable # command receipt and the writer persists that command as the @@ -1963,11 +2095,16 @@ def load_validation_context(root: Path, run_id: str, *, central_database: Path | # historical only; projections never backfill their association. "diagnostic_evidence_ref": f"artifact:{diagnostic_artifact_id}" if diagnostic_execution_id == command_id else "UNAVAILABLE", } - return {"selected_validation_tier": profile[0], "validation_profile_version": profile[1], + return {"selected_validation_tier": selected_tier, "validation_profile_version": profile_version, "profile_reference": payload.get("profile_reference", "UNAVAILABLE"), "profile_selection_source": payload.get("profile_selection_source", "UNAVAILABLE"), "required_validation_controls": tuple(required), "control_bindings": tuple(bindings), - "recorded_at": profile[3], "controls": controls} + "candidate_sha": candidate_sha, "currentness": profile_currentness, + "profile_digest": profile_digest, + "profile_currentness_conflict": ( + profile_currentness is not None and evidence_currentness > profile_currentness + ), + "recorded_at": recorded_at, "controls": controls} def record_run_qualification_snapshot(root: Path, snapshot: dict[str, object]) -> dict[str, object]: diff --git a/src/engineering_platform/templates/workspace-config.json b/src/engineering_platform/templates/workspace-config.json index dcac8c9c..b3c31a05 100644 --- a/src/engineering_platform/templates/workspace-config.json +++ b/src/engineering_platform/templates/workspace-config.json @@ -3,7 +3,7 @@ "platform": { "id": "engineering-platform", "name": "Engineering Platform", - "version": "2.3.2", + "version": "2.3.4", "generation": 2, "documentation_namespace": "engineering-platform", "capability_registry_version": 1 diff --git a/src/engineering_platform/validation_profile.py b/src/engineering_platform/validation_profile.py index 0188cc9d..fc1805ef 100644 --- a/src/engineering_platform/validation_profile.py +++ b/src/engineering_platform/validation_profile.py @@ -2,8 +2,12 @@ from __future__ import annotations import argparse from dataclasses import dataclass +from datetime import datetime +import hashlib +import json from pathlib import Path import re +import shlex import subprocess import sys @@ -45,6 +49,110 @@ class ValidationProfileResolutionError(ValueError): """The selected run profile is absent or does not match this registry.""" +def validation_profile_identity( + *, candidate_sha: str, currentness: int, selected_validation_tier: str, + validation_profile_version: str, profile_reference: str, + profile_selection_source: str, required_validation_controls: tuple[str, ...], + control_bindings: tuple[dict[str, object], ...], +) -> tuple[dict[str, object], str]: + """Build the canonical candidate-bound identity of one validation pass.""" + if ( + re.fullmatch(r"[0-9a-f]{40}", candidate_sha) is None + or isinstance(currentness, bool) or not isinstance(currentness, int) or currentness < 0 + or not all(isinstance(value, str) and value for value in ( + selected_validation_tier, validation_profile_version, + profile_reference, profile_selection_source, + )) + or not required_validation_controls + or len(set(required_validation_controls)) != len(required_validation_controls) + or any(not isinstance(control, str) or not control for control in required_validation_controls) + or len(control_bindings) != len(required_validation_controls) + or any(not isinstance(binding, dict) for binding in control_bindings) + or tuple(binding.get("validation_id") for binding in control_bindings) != required_validation_controls + ): + raise ValidationProfileResolutionError("Validation profile identity is invalid.") + identity: dict[str, object] = { + "candidate_sha": candidate_sha, + "currentness": currentness, + "selected_validation_tier": selected_validation_tier, + "validation_profile_version": validation_profile_version, + "profile_reference": profile_reference, + "profile_selection_source": profile_selection_source, + "required_validation_controls": list(required_validation_controls), + "control_bindings": list(control_bindings), + } + digest = "sha256:" + hashlib.sha256( + json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + return identity, digest + + +def _valid_terminal_times(started_at: object, ended_at: object) -> bool: + if not isinstance(started_at, str) or not started_at or not isinstance(ended_at, str) or not ended_at: + return False + try: + start = datetime.fromisoformat(started_at) + end = datetime.fromisoformat(ended_at) + except ValueError: + return False + return start.tzinfo is not None and end.tzinfo is not None and end >= start + + +def strict_required_controls_pass( + validation_context: object, *, candidate_sha: str, currentness: int, +) -> bool: + """Require exact terminal receipts for every profile-owned required control.""" + if not isinstance(validation_context, dict): + return False + try: + identity, digest = validation_profile_identity( + candidate_sha=candidate_sha, + currentness=currentness, + selected_validation_tier=validation_context["selected_validation_tier"], + validation_profile_version=validation_context["validation_profile_version"], + profile_reference=validation_context["profile_reference"], + profile_selection_source=validation_context["profile_selection_source"], + required_validation_controls=validation_context["required_validation_controls"], + control_bindings=validation_context["control_bindings"], + ) + except (KeyError, TypeError, ValidationProfileResolutionError): + return False + if ( + validation_context.get("candidate_sha") != candidate_sha + or validation_context.get("currentness") != currentness + or validation_context.get("profile_digest") != digest + or validation_context.get("profile_currentness_conflict") is True + ): + return False + required = tuple(identity["required_validation_controls"]) + bindings = tuple(identity["control_bindings"]) + controls = validation_context.get("controls") + if not isinstance(controls, dict): + return False + binding_by_id = {binding.get("validation_id"): binding for binding in bindings} + for validation_id in required: + binding = binding_by_id.get(validation_id) + control = controls.get(validation_id) + if not isinstance(binding, dict) or binding.get("required") is not True or not isinstance(control, dict): + return False + if ( + control.get("validation_id") != validation_id + or control.get("required_for_profile") is not True + or control.get("execution_status") != "EXECUTED" + or control.get("result") != "PASS" + or control.get("exit_code") != 0 + or control.get("currentness") != currentness + or not isinstance(control.get("command_id"), str) + or not control.get("command_id") + or control.get("evidence_authority") != "command_terminal" + or control.get("category") != binding.get("category") + or control.get("control_identity") != binding.get("control_identity") + or not _valid_terminal_times(control.get("started_at"), control.get("ended_at")) + ): + return False + return True + + def _python_command(*arguments: str) -> tuple[str, ...]: return (sys.executable, *arguments) @@ -145,6 +253,44 @@ def profile_control_bindings(profile: "ValidationProfile") -> tuple[dict[str, ob raise ValidationProfileResolutionError("Selected validation profile launcher is unavailable.") return tuple(binding for binding in bindings if binding is not None) + +def matching_control_binding( + command: str, control_bindings: tuple[dict[str, object], ...], +) -> dict[str, object] | None: + """Resolve only an exact, standalone profile-owned launcher command. + + Python interpreter paths are normalized because the registry snapshots the + host interpreter while provider telemetry commonly reports ``python3``. + Arguments, shell composition and environment overrides remain exact. + """ + if not isinstance(command, str) or not command or "\n" in command or "\r" in command: + return None + try: + observed = shlex.split(command) + except ValueError: + return None + if not observed or any(token in {"&&", "||", ";", "|"} for token in observed): + return None + + def normalized(tokens: list[str]) -> tuple[str, ...]: + values = list(tokens) + if values and re.fullmatch(r"python(?:3(?:\.\d+)?)?", Path(values[0]).name): + values[0] = "python" + return tuple(values) + + observed_identity = normalized(observed) + matches = [] + for binding in control_bindings: + expected = binding.get("command") if isinstance(binding, dict) else None + if ( + isinstance(expected, list) + and expected + and all(isinstance(token, str) and token for token in expected) + and normalized(expected) == observed_identity + ): + matches.append(binding) + return matches[0] if len(matches) == 1 else None + @dataclass(frozen=True) class ValidationProfile: tier: str diff --git a/tests/engineering/test_contract_projection.py b/tests/engineering/test_contract_projection.py index 4365ea4c..39e1af18 100644 --- a/tests/engineering/test_contract_projection.py +++ b/tests/engineering/test_contract_projection.py @@ -17,6 +17,8 @@ open_storage, record_run_qualification_context, record_submission, + record_validation_command_invocation, + record_validation_command_terminal, record_validation_control_result, record_validation_profile, record_run_qualification_snapshot, @@ -98,6 +100,17 @@ def test_projection_uses_explicit_lineage_and_required_validation_evidence(self) record_validation_profile( root, run_id=state.run_id, selected_validation_tier="DOCUMENTATION", validation_profile_version="1.0", required_validation_controls=("git_diff_check",), recorded_at="2026-08-28T00:00:00+00:00", + candidate_sha="a" * 40, currentness=1, + ) + record_validation_command_invocation( + root, run_id=state.run_id, validation_id="git_diff_check", + command_id="contract-diff", category="repository", + control_identity="git diff --check", required_for_profile=True, + started_at="2026-08-28T00:00:00+00:00", currentness=1, + ) + record_validation_command_terminal( + root, run_id=state.run_id, command_id="contract-diff", + completed_at="2026-08-28T00:00:01+00:00", exit_code=0, ) record_validation_control_result( root, run_id=state.run_id, validation_id="git_diff_check", category="repository", diff --git a/tests/engineering/test_deterministic_execution_e2e.py b/tests/engineering/test_deterministic_execution_e2e.py index 3215aa7c..97072db6 100644 --- a/tests/engineering/test_deterministic_execution_e2e.py +++ b/tests/engineering/test_deterministic_execution_e2e.py @@ -57,6 +57,24 @@ def test_isolated_runtime_rejects_an_unexpected_server_configuration_shape(self) with self.assertRaisesRegex(RuntimeError, "QUALIFICATION_SERVER_CONFIGURATION_INVALID"): self.module.configure_deterministic_runtime(self.data, self.root) + def test_managed_fixture_has_a_real_suite_without_dirtying_its_candidate(self) -> None: + repository = self.root / "managed" + self.module.create_repository(repository) + + completed = subprocess.run( + ("python3", "-m", "unittest", "discover"), cwd=repository, + check=False, text=True, capture_output=True, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual( + subprocess.run( + ("git", "-C", str(repository), "status", "--porcelain"), + check=True, text=True, capture_output=True, + ).stdout, + "", + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/engineering/test_execution_host.py b/tests/engineering/test_execution_host.py index 86d9c9f5..e200fbef 100644 --- a/tests/engineering/test_execution_host.py +++ b/tests/engineering/test_execution_host.py @@ -162,10 +162,24 @@ def merge(self, number: int) -> None: class FakeAgent: def __init__(self, result: AgentResult) -> None: self.result, self.prompts, self.roots = result, [], [] + self.command_callback: object | None = None + + def set_command_callback(self, callback: object) -> None: + self.command_callback = callback + + def _emit_required_validation_receipts(self, prompt: str, result: AgentResult) -> None: + if "Local repository validation gate" not in prompt or not callable(self.command_callback): + return + failed = result.terminal_state != "COMPLETE" or EngineeringRunner._has_failed_validation_evidence(result) + for ordinal, command in enumerate(("git diff --check", "python3 -m unittest discover"), start=1): + command_id = f"fake-validation-{len(self.prompts)}-{ordinal}" + self.command_callback("started", command_id, command) + self.command_callback("completed", command_id, command, 1 if failed and ordinal == 2 else 0) def invoke(self, root: Path, prompt: str) -> AgentResult: self.roots.append(root) self.prompts.append(prompt) + self._emit_required_validation_receipts(prompt, self.result) return self.result def available(self) -> bool: @@ -202,7 +216,9 @@ def __init__(self, results: list[AgentResult]) -> None: def invoke(self, root: Path, prompt: str) -> AgentResult: self.prompts.append(prompt) - return self.results.pop(0) + result = self.results.pop(0) + self._emit_required_validation_receipts(prompt, result) + return result class LiveStatusFakeAgent(FakeAgent): @@ -429,10 +445,12 @@ def invoke(self, root: Path, prompt_text: str) -> AgentResult: self.roots.append(root) self.prompts.append(prompt_text) if "Local repository validation gate" in prompt_text: - return AgentResult( + result = AgentResult( "COMPLETE", branch, commit_sha=commit, validation_evidence=({"command": "canonical suite", "result": "passed"},), ) + self._emit_required_validation_receipts(prompt_text, result) + return result if "First implementation pull-request publication gate" in prompt_text: self.pr_create_calls += 1 return AgentResult("COMPLETE", branch, 701, commit_sha=commit) @@ -1665,6 +1683,31 @@ def tearDown(self) -> None: else: os.environ[key] = value + def _record_strict_passing_profile( + self, run_id: str, *, candidate_sha: str = "a" * 40, currentness: int = 0, + ) -> str: + record_validation_profile( + self.root, run_id=run_id, selected_validation_tier="FULL", + validation_profile_version=execution_host.VALIDATION_PROFILE_VERSION, + required_validation_controls=("repository_suite",), + candidate_sha=candidate_sha, currentness=currentness, + recorded_at="2026-09-10T10:00:00+00:00", + ) + command_id = f"{run_id}-repository-suite" + record_validation_command_invocation( + self.root, run_id=run_id, validation_id="repository_suite", + command_id=command_id, category="repository", + control_identity="python3 -m unittest discover", required_for_profile=True, + started_at="2026-09-10T10:00:00+00:00", currentness=currentness, + ) + record_validation_command_terminal( + self.root, run_id=run_id, command_id=command_id, + completed_at="2026-09-10T10:00:01+00:00", exit_code=0, + ) + context = load_validation_context(self.root, run_id, currentness=currentness) + assert context is not None and isinstance(context["profile_digest"], str) + return context["profile_digest"] + def test_new_run_initializes_and_records_canonical_prompt(self) -> None: quality_evidence = ({"activity": "TEST_COVERAGE", "result": "Added focused regression coverage."},) agent = FakeAgent(AgentResult("COMPLETE", quality_evidence=quality_evidence)) @@ -2608,8 +2651,12 @@ def test_quality_assurance_does_not_create_or_replace_the_implementation_pr(self def test_first_implementation_publication_is_a_separate_post_assurance_dispatch(self) -> None: """The product gate, not provider wording, owns first PR creation.""" sha = "a" * 40 + validation_profile_digest = self._record_strict_passing_profile( + "publication-gate", candidate_sha=sha, + ) profile = {"version": "validation-profile@1.0", "digest": "sha256:" + "b" * 64, - "criteria_digest": "sha256:" + "c" * 64, "candidate_sha": sha} + "criteria_digest": "sha256:" + "c" * 64, "candidate_sha": sha, + "validation_profile_digest": validation_profile_digest} reviews = tuple({"reviewer": role, "status": "PASS", "candidate_sha": sha, "profile_digest": profile["digest"], "invocation_id": f"{role}-1", "findings": [], "contract_version": "1.0", "started_at": "now", "completed_at": "now"} @@ -2617,7 +2664,9 @@ def test_first_implementation_publication_is_a_separate_post_assurance_dispatch( agent = SequencedFakeAgent([AgentResult("COMPLETE", "main", 71, commit_sha=sha)]) runner = EngineeringRunner(self.root, self.store, FakeRepository(), FakeGitHub([]), agent, lambda _: None) state = TransactionState("publication-gate", "pcvantol/djconnect", str(self.prompt), "QUALITY_CONTROL_AGENT", - branch="main", owner_authorized=True, assurance_profile=profile, assurance_reviews=reviews) + branch="main", owner_authorized=True, + local_validation_audit=({"outcome": "validated"},), + assurance_profile=profile, assurance_reviews=reviews) published, result = runner._publish_first_implementation_pull_request( state, AgentResult("COMPLETE", "main", commit_sha=sha) ) @@ -2678,7 +2727,7 @@ def test_local_repository_validation_iterates_before_creating_the_implementation validation_evidence=({"command": "python -m unittest tests.engineering", "result": "passed"},), ), ]) - runner = EngineeringRunner(self.root, self.store, FakeRepository(), FakeGitHub([]), agent, lambda _: None) + runner = EngineeringRunner(self.root, self.store, FakeRepository(branch="codex/implementation"), FakeGitHub([]), agent, lambda _: None) state = TransactionState( "local-validation-run", "pcvantol/djconnect", str(self.prompt), "EXECUTE_AGENT", branch="codex/implementation", owner_authorized=True, @@ -2791,7 +2840,7 @@ def test_failed_local_validation_is_one_non_mutating_turn_before_shared_repair(s for _ in range(3) ] runner = EngineeringRunner( - self.root, self.store, FakeRepository(), FakeGitHub([]), SequencedFakeAgent(failures), lambda _: None + self.root, self.store, FakeRepository(branch="codex/implementation"), FakeGitHub([]), SequencedFakeAgent(failures), lambda _: None ) state = TransactionState( "local-validation-limit", "pcvantol/djconnect", str(self.prompt), "EXECUTE_AGENT", @@ -2819,7 +2868,7 @@ def test_local_repository_validation_separates_proven_environment_instability(se validation_disposition="environmental_instability", ), ]) - runner = EngineeringRunner(self.root, self.store, FakeRepository(), FakeGitHub([]), agent, lambda _: None) + runner = EngineeringRunner(self.root, self.store, FakeRepository(branch="codex/implementation"), FakeGitHub([]), agent, lambda _: None) state = TransactionState( "validation-instability-run", "pcvantol/djconnect", str(self.prompt), "EXECUTE_AGENT", branch="codex/implementation", owner_authorized=True, diff --git a/tests/engineering/test_managed_autonomy.py b/tests/engineering/test_managed_autonomy.py index 2e5657b4..37516d79 100644 --- a/tests/engineering/test_managed_autonomy.py +++ b/tests/engineering/test_managed_autonomy.py @@ -14,6 +14,8 @@ ) from engineering_platform.storage import ( record_run_qualification_context, + record_validation_command_invocation, + record_validation_command_terminal, record_validation_control_result, record_validation_profile, ) @@ -62,6 +64,16 @@ def _qualified(self, root: Path, *, retry_parent: str | None = None, resume_pare record_validation_profile( root, run_id=run, selected_validation_tier="DOCUMENTATION", validation_profile_version="1.0", required_validation_controls=("git_diff_check",), recorded_at="2026-08-28T00:00:00+00:00", + candidate_sha="a" * 40, currentness=1, + ) + record_validation_command_invocation( + root, run_id=run, validation_id="git_diff_check", command_id="managed-proof-diff", + category="repository", control_identity="git diff --check", required_for_profile=True, + started_at="2026-08-28T00:00:00+00:00", currentness=1, + ) + record_validation_command_terminal( + root, run_id=run, command_id="managed-proof-diff", + completed_at="2026-08-28T00:00:01+00:00", exit_code=0, ) record_validation_control_result( root, run_id=run, validation_id="git_diff_check", category="repository", diff --git a/tests/engineering/test_p_neutral_consumer_credential_namespace.py b/tests/engineering/test_p_neutral_consumer_credential_namespace.py index a1993506..24962e13 100644 --- a/tests/engineering/test_p_neutral_consumer_credential_namespace.py +++ b/tests/engineering/test_p_neutral_consumer_credential_namespace.py @@ -169,7 +169,7 @@ def test_historical_storage_schema_transfers_to_neutral_tables_transactionally(s ("credential", "consumer", "project", verifier("storage-token"), b"f" * 32, "now"), ) with storage.activate_storage_schema(root) as connection: - self.assertEqual(storage._schema_version(connection), 41) + self.assertEqual(storage._schema_version(connection), 42) self.assertEqual( connection.execute("SELECT verifier,fingerprint FROM ep_consumer_credentials").fetchone(), (verifier("storage-token"), b"f" * 32), diff --git a/tests/engineering/test_qualification_runtime.py b/tests/engineering/test_qualification_runtime.py index f72fce83..2b9db725 100644 --- a/tests/engineering/test_qualification_runtime.py +++ b/tests/engineering/test_qualification_runtime.py @@ -82,6 +82,31 @@ def test_controlled_interruption_window_is_explicit_and_bounded(self) -> None: with patch.dict(os.environ, {}, clear=True): self.assertIsNone(agent.wait_for_controlled_interruption_arm(self.root, SimpleNamespace(run_id="run-a"))) + def test_local_validation_emits_real_canonical_terminal_receipts(self) -> None: + agent = DeterministicQualificationAgent() + events = [] + agent.set_command_callback(lambda *event: events.append(event)) + (self.root / "test_qualification_fixture.py").write_text( + "import unittest\n\n" + "class FixtureTest(unittest.TestCase):\n" + " def test_fixture(self): self.assertTrue(True)\n", + encoding="utf-8", + ) + + result = agent.validate(self.root, "Local repository validation gate") + + self.assertEqual(result.terminal_state, "COMPLETE") + self.assertEqual( + [(event[0], event[1]) for event in events], + [ + ("started", "deterministic-validation-1"), + ("completed", "deterministic-validation-1"), + ("started", "deterministic-validation-2"), + ("completed", "deterministic-validation-2"), + ], + ) + self.assertEqual([event[3] for event in events if event[0] == "completed"], [0, 0]) + def test_external_handoff_writes_only_the_explicit_fixture_contract(self) -> None: agent = DeterministicQualificationAgent() (self.root / ".engineering-platform").mkdir() diff --git a/tests/engineering/test_recovery_cross_surface.py b/tests/engineering/test_recovery_cross_surface.py index dbd00bb6..ca1cc009 100644 --- a/tests/engineering/test_recovery_cross_surface.py +++ b/tests/engineering/test_recovery_cross_surface.py @@ -36,6 +36,8 @@ ) from engineering_platform.provider_usage import AUTHORITATIVE, ProviderInvocation, persist_provider_invocation from engineering_platform.storage import ( + record_validation_command_invocation, + record_validation_command_terminal, record_run_qualification_context, record_submission, record_validation_control_result, @@ -89,8 +91,20 @@ def _fixture(self, *, qualified: bool) -> TransactionState: record_validation_profile( self.root, run_id=self.run_id, selected_validation_tier="DOCUMENTATION", validation_profile_version="1.0", required_validation_controls=("git_diff_check",), + candidate_sha=self.commit, currentness=2, recorded_at="2026-08-30T00:00:00+00:00", ) + record_validation_command_invocation( + self.root, run_id=self.run_id, validation_id="git_diff_check", + command_id="validation-current-profile", category="repository", + control_identity="git diff --check", required_for_profile=True, + started_at="2026-08-30T00:00:00+00:00", currentness=2, + ) + record_validation_command_terminal( + self.root, run_id=self.run_id, command_id="validation-current-profile", + completed_at="2026-08-30T00:00:01+00:00", + exit_code=0 if qualified else 1, + ) record_validation_control_result( self.root, run_id=self.run_id, validation_id="git_diff_check", category="repository", control_identity="git diff --check", required_for_profile=True, execution_status="EXECUTED", @@ -178,7 +192,7 @@ def test_recovered_terminal_run_has_one_consistent_cross_surface_projection(self self.assertEqual(history["history"]["submission_id"], self.submission_id) self.assertEqual(history["history"]["producer_id"], self.producer_id) self.assertIsNone(history["history"]["retry_of"]) - self.assertEqual(history["history"]["execution_activity_summary"]["activity"]["overall_activity_total"], 3) + self.assertEqual(history["history"]["execution_activity_summary"]["activity"]["overall_activity_total"], 4) self.assertEqual(lifecycle_projection(self.root, self.run_id)["terminal_state"], "COMPLETE") history_rows = json.loads(dashboard._prompt_history(self.root))["runs"] self.assertEqual(len(history_rows), 1) diff --git a/tests/engineering/test_storage.py b/tests/engineering/test_storage.py index 1900cd8f..c9fa8d7c 100644 --- a/tests/engineering/test_storage.py +++ b/tests/engineering/test_storage.py @@ -552,6 +552,7 @@ def test_schema_twenty_five_repairs_early_usage_snapshot_table(self) -> None: connection.execute("DELETE FROM engineering_schema_migrations WHERE version=39") connection.execute("DELETE FROM engineering_schema_migrations WHERE version=40") connection.execute("DELETE FROM engineering_schema_migrations WHERE version=41") + connection.execute("DELETE FROM engineering_schema_migrations WHERE version=42") with activate_storage_schema(root) as connection: columns = { row[1] diff --git a/tests/engineering/test_strict_current_validation.py b/tests/engineering/test_strict_current_validation.py new file mode 100644 index 00000000..21bc97ed --- /dev/null +++ b/tests/engineering/test_strict_current_validation.py @@ -0,0 +1,343 @@ +"""Product closure for candidate-bound current validation publication evidence.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import replace +from pathlib import Path +import sqlite3 +import tempfile +import unittest +from unittest.mock import patch + +from engineering_platform.agent_state import StateStore, TransactionState +from engineering_platform.execution_host import ( + EngineeringRunner, + _has_current_local_validation_evidence, + _required_validation_controls_pass, + _validation_profile_digest, +) +from engineering_platform.execution_errors import RunnerError +from engineering_platform.execution_models import AgentResult, RepositoryEvidence +from engineering_platform.storage import ( + EngineeringStorageError, + load_validation_context, + open_storage, + record_validation_command_invocation, + record_validation_command_terminal, + record_validation_profile, +) +from engineering_platform.validation_profile import ( + strict_required_controls_pass, + validation_profile_identity, +) + + +CANDIDATE = "a" * 40 +BINDING = { + "validation_id": "required_suite", + "required": True, + "category": "repository", + "control_identity": "canonical required suite", + "command": ["python3", "-m", "unittest", "discover"], +} + + +def _context() -> dict[str, object]: + identity, digest = validation_profile_identity( + candidate_sha=CANDIDATE, currentness=1, + selected_validation_tier="FULL", validation_profile_version="1.0", + profile_reference="validation-profile-registry:FULL@1.0", + profile_selection_source="diff_classification", + required_validation_controls=("required_suite",), + control_bindings=(BINDING,), + ) + return { + **identity, + "required_validation_controls": tuple(identity["required_validation_controls"]), + "control_bindings": tuple(identity["control_bindings"]), + "profile_digest": digest, + "profile_currentness_conflict": False, + "controls": { + "required_suite": { + "validation_id": "required_suite", + "command_id": "required-suite-1", + "category": "repository", + "control_identity": "canonical required suite", + "required_for_profile": True, + "execution_status": "EXECUTED", + "result": "PASS", + "exit_code": 0, + "currentness": 1, + "evidence_authority": "command_terminal", + "started_at": "2026-09-10T10:00:00+00:00", + "ended_at": "2026-09-10T10:00:01+00:00", + }, + "provider_optional_claim": { + "required_for_profile": True, + "execution_status": "NOT_EXECUTED", + "result": "FAIL", + }, + }, + } + + +class StrictValidationContractTest(unittest.TestCase): + def test_every_current_required_control_needs_an_exact_passing_terminal_receipt(self) -> None: + context = _context() + self.assertTrue(strict_required_controls_pass(context, candidate_sha=CANDIDATE, currentness=1)) + + mutations = { + "missing_required_control": lambda value: value["controls"].pop("required_suite"), + "missing_receipt": lambda value: value["controls"]["required_suite"].pop("command_id"), + "not_executed": lambda value: value["controls"]["required_suite"].update(execution_status="NOT_EXECUTED"), + "failed": lambda value: value["controls"]["required_suite"].update(result="FAIL"), + "skipped": lambda value: value["controls"]["required_suite"].update(result="SKIPPED"), + "not_applicable": lambda value: value["controls"]["required_suite"].update(result="NOT_APPLICABLE"), + "unavailable": lambda value: value["controls"]["required_suite"].update(result="UNAVAILABLE"), + "timeout": lambda value: value["controls"]["required_suite"].update(result="TIMEOUT"), + "nonzero": lambda value: value["controls"]["required_suite"].update(exit_code=7), + "stale_ordinal": lambda value: value["controls"]["required_suite"].update(currentness=0), + "wrong_digest": lambda value: value.update(profile_digest="sha256:" + "f" * 64), + "wrong_required_set": lambda value: value.update(required_validation_controls=("other",)), + "missing_started_at": lambda value: value["controls"]["required_suite"].update(started_at=None), + "missing_ended_at": lambda value: value["controls"]["required_suite"].update(ended_at=None), + "invalid_time_order": lambda value: value["controls"]["required_suite"].update(ended_at="2026-09-10T09:59:59+00:00"), + "prose_only": lambda value: value["controls"]["required_suite"].update( + command_id=None, evidence_authority="agent_result", exit_code=None, + ), + } + for name, mutate in mutations.items(): + with self.subTest(name=name): + changed = deepcopy(context) + mutate(changed) + self.assertFalse(strict_required_controls_pass(changed, candidate_sha=CANDIDATE, currentness=1)) + + self.assertFalse(strict_required_controls_pass(context, candidate_sha="b" * 40, currentness=1)) + self.assertFalse(strict_required_controls_pass(context, candidate_sha=CANDIDATE, currentness=2)) + + def test_profile_and_receipts_are_append_only_and_recover_by_exact_ordinal(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + digest = record_validation_profile( + root, run_id="strict-restart", selected_validation_tier="FULL", + validation_profile_version="1.0", required_validation_controls=("required_suite",), + profile_reference="validation-profile-registry:FULL@1.0", + profile_selection_source="diff_classification", control_bindings=(BINDING,), + candidate_sha=CANDIDATE, currentness=1, + recorded_at="2026-09-10T10:00:00+00:00", + ) + record_validation_command_invocation( + root, run_id="strict-restart", validation_id="required_suite", + command_id="required-suite-1", category="repository", + control_identity="canonical required suite", required_for_profile=True, + started_at="2026-09-10T10:00:00+00:00", currentness=1, + ) + record_validation_command_terminal( + root, run_id="strict-restart", command_id="required-suite-1", + completed_at="2026-09-10T10:00:01+00:00", exit_code=0, + ) + recovered = load_validation_context(root, "strict-restart", currentness=1) + self.assertEqual(recovered["profile_digest"], digest) + self.assertTrue(strict_required_controls_pass( + recovered, candidate_sha=CANDIDATE, currentness=1, + )) + + with self.assertRaisesRegex(EngineeringStorageError, "different immutable identity"): + record_validation_profile( + root, run_id="strict-restart", selected_validation_tier="FULL", + validation_profile_version="1.0", required_validation_controls=("required_suite",), + profile_reference="validation-profile-registry:FULL@1.0", + profile_selection_source="diff_classification", control_bindings=(BINDING,), + candidate_sha="b" * 40, currentness=1, + recorded_at="2026-09-10T10:01:00+00:00", + ) + with open_storage(root) as connection, self.assertRaises(sqlite3.DatabaseError): + connection.execute( + "UPDATE execution_validation_profile_identities SET candidate_sha=? WHERE run_id=?", + ("b" * 40, "strict-restart"), + ) + + def test_quality_and_security_passes_cannot_compensate_for_missing_validation_receipt(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prompt = root / "prompt.md" + prompt.write_text("bounded implementation", encoding="utf-8") + digest = record_validation_profile( + root, run_id="qs-cannot-override", selected_validation_tier="FULL", + validation_profile_version="1.0", required_validation_controls=("required_suite",), + profile_reference="validation-profile-registry:FULL@1.0", + profile_selection_source="diff_classification", control_bindings=(BINDING,), + candidate_sha=CANDIDATE, currentness=1, + recorded_at="2026-09-10T10:00:00+00:00", + ) + assurance_digest = "sha256:" + "c" * 64 + profile = { + "version": "validation-profile@1.0", "digest": assurance_digest, + "candidate_sha": CANDIDATE, "criteria_digest": "sha256:" + "d" * 64, + "validation_profile_digest": digest, + } + reviews = tuple({ + "reviewer": role, "status": "PASS", "candidate_sha": CANDIDATE, + "profile_digest": assurance_digest, "invocation_id": f"{role}-1", + "findings": [], "contract_version": "1.0", + "started_at": "2026-09-10T10:00:02+00:00", + "completed_at": "2026-09-10T10:00:03+00:00", + } for role in ("quality", "security")) + audit = ({ + "iteration": "1", "observed_at": "2026-09-10T10:00:01+00:00", + "failed_checks": "No failed controls.", "proposed_action": "FULL profile.", + "agent_summary": "Provider claimed completion.", "commit_sha": "not_recorded", + "outcome": "validated", + },) + state = TransactionState( + "qs-cannot-override", "pcvantol/djconnect", str(prompt), "QUALITY_CONTROL_AGENT", + branch="codex/strict", owner_authorized=True, repair_iterations=1, + local_validation_iterations=1, local_validation_audit=audit, + assurance_profile=profile, assurance_reviews=reviews, + ) + + class Repository: + @staticmethod + def inspect(_: Path) -> RepositoryEvidence: + return RepositoryEvidence("pcvantol/djconnect", "codex/strict", CANDIDATE, True) + + class Agent: + called = False + + @staticmethod + def available() -> bool: + return True + + @staticmethod + def version() -> str: + return "0.146.0" + + def invoke(self, _: Path, __: str) -> AgentResult: + self.called = True + return AgentResult("COMPLETE", "codex/strict", 71, commit_sha=CANDIDATE) + + agent = Agent() + runner = EngineeringRunner( + root, StateStore(root / ".engineering" / "engineering-runs"), + Repository(), None, agent, lambda _: None, + ) + blocked, _ = runner._publish_first_implementation_pull_request( + state, AgentResult("COMPLETE", "codex/strict", commit_sha=CANDIDATE), + ) + self.assertTrue(blocked.terminal) + self.assertEqual(blocked.next_action, "implementation_publication_assurance_required") + self.assertFalse(agent.called) + + def test_candidate_bound_profile_and_validated_audit_make_current_validation_eligible(self) -> None: + context = _context() + digest = context["profile_digest"] + state = TransactionState( + "eligible", "pcvantol/djconnect", "prompt.md", "QUALITY_CONTROL_AGENT", + repair_iterations=1, + local_validation_audit=({"outcome": "validated"},), + assurance_profile={ + "version": "validation-profile@1.0", "digest": "sha256:" + "c" * 64, + "candidate_sha": CANDIDATE, "criteria_digest": "sha256:" + "d" * 64, + "validation_profile_digest": digest, + }, + ) + self.assertTrue(_has_current_local_validation_evidence(state, context)) + + def test_host_adapters_fail_closed_and_classify_exact_profile_launchers(self) -> None: + context = _context() + state = TransactionState( + "adapter-fail-closed", "pcvantol/djconnect", "prompt.md", "QUALITY_CONTROL_AGENT", + repair_iterations=1, + local_validation_audit=({"outcome": "validated"},), + assurance_profile={ + "version": "validation-profile@1.0", "digest": "sha256:" + "c" * 64, + "candidate_sha": CANDIDATE, "criteria_digest": "sha256:" + "d" * 64, + "validation_profile_digest": context["profile_digest"], + }, + ) + self.assertIsNone(_validation_profile_digest({ + "profile_digest": "invalid", "candidate_sha": CANDIDATE, "currentness": 1, + })) + self.assertFalse(_required_validation_controls_pass(state, None)) + missing_candidate = deepcopy(context) + missing_candidate["candidate_sha"] = None + self.assertFalse(_required_validation_controls_pass( + replace(state, assurance_profile=None), missing_candidate, + )) + self.assertFalse(_has_current_local_validation_evidence( + replace(state, local_validation_audit=()), context, + )) + self.assertFalse(_has_current_local_validation_evidence( + replace(state, local_validation_audit=({"outcome": "failed"},)), context, + )) + self.assertFalse(_has_current_local_validation_evidence( + replace(state, assurance_profile=None), context, + )) + self.assertEqual( + EngineeringRunner._validation_kind("python3 tools/qualification/console_route_ownership_guard.py"), + "console_route_ownership", + ) + self.assertEqual( + EngineeringRunner._validation_kind("npm run test:ui-localization"), + "ui_localization", + ) + self.assertEqual( + EngineeringRunner._validation_id_for_profile("python3 -m unittest discover", "tests", "FULL"), + "repository_suite", + ) + + def test_publication_adapters_block_missing_assurance_and_repository_evidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prompt = root / "prompt.md" + prompt.write_text("bounded implementation", encoding="utf-8") + profile = { + "version": "validation-profile@1.0", "digest": "sha256:" + "c" * 64, + "candidate_sha": CANDIDATE, "criteria_digest": "sha256:" + "d" * 64, + "validation_profile_digest": "sha256:" + "e" * 64, + } + state = TransactionState( + "publication-adapters", "pcvantol/djconnect", str(prompt), "QUALITY_CONTROL_AGENT", + branch="codex/strict", owner_authorized=True, repair_iterations=1, + assurance_profile=profile, + ) + + class MissingRepository: + @staticmethod + def inspect(_: Path) -> RepositoryEvidence: + raise RunnerError("candidate unavailable") + + runner = EngineeringRunner( + root, StateStore(root / ".engineering" / "engineering-runs"), + MissingRepository(), None, object(), lambda _: None, + ) + implementation = AgentResult("COMPLETE", "codex/strict", commit_sha=CANDIDATE) + with patch.object(runner, "_current_local_validation_passes", return_value=True): + missing_reviews, _ = runner._publish_first_implementation_pull_request(state, implementation) + self.assertEqual(missing_reviews.next_action, "implementation_publication_assurance_required") + + reviews = tuple({ + "reviewer": role, "status": "PASS", "candidate_sha": CANDIDATE, + "profile_digest": profile["digest"], "invocation_id": f"{role}-1", + "findings": [], "contract_version": "1.0", + "started_at": "2026-09-10T10:00:02+00:00", + "completed_at": "2026-09-10T10:00:03+00:00", + } for role in ("quality", "security")) + with patch.object(runner, "_current_local_validation_passes", return_value=True): + missing_candidate, _ = runner._publish_first_implementation_pull_request( + replace(state, assurance_reviews=reviews), implementation, + ) + self.assertEqual( + missing_candidate.next_action, "implementation_publication_candidate_unavailable", + ) + + with patch( + "engineering_platform.execution_host.load_validation_context", + side_effect=EngineeringStorageError("corrupt profile"), + ): + self.assertIsNone(runner._validation_context(state)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/engineering/test_validation_profile.py b/tests/engineering/test_validation_profile.py index 02d521a0..1c6b636b 100644 --- a/tests/engineering/test_validation_profile.py +++ b/tests/engineering/test_validation_profile.py @@ -1,7 +1,7 @@ from __future__ import annotations import unittest from engineering_platform.validation_profile import ( - ValidationProfileResolutionError, browser_dashboard_required, classify, localization_required, phase_for_branch, profile_control_bindings, + ValidationProfileResolutionError, browser_dashboard_required, classify, localization_required, matching_control_binding, phase_for_branch, profile_control_bindings, producer_profile_payload, resolve_producer_profile, ) @@ -61,3 +61,18 @@ def test_producer_payload_is_derived_only_from_the_canonical_registry(self) -> N }) with self.assertRaises(ValidationProfileResolutionError): producer_profile_payload("DASHBOARD ") + + def test_required_control_binding_matches_only_its_exact_standalone_command(self) -> None: + bindings = profile_control_bindings(classify(["custom/mixed-scope.txt"])) + full = bindings[-1] + self.assertEqual( + matching_control_binding("python3 -m unittest discover", bindings), full, + ) + for command in ( + "python3 -m unittest tests.engineering.test_storage", + "PYTHONPATH=src python3 -m unittest discover", + "python3 -m unittest discover && true", + "python3 -m unittest discover\ngit diff --check", + ): + with self.subTest(command=command): + self.assertIsNone(matching_control_binding(command, bindings)) diff --git a/tools/qualification/p_deterministic_execution_e2e.py b/tools/qualification/p_deterministic_execution_e2e.py index 7b794d19..5b6ef4bf 100644 --- a/tools/qualification/p_deterministic_execution_e2e.py +++ b/tools/qualification/p_deterministic_execution_e2e.py @@ -79,8 +79,16 @@ def create_repository(path: Path, *, origin: Path | None = None) -> None: subprocess.run(("git", "init", "-q", "-b", "main", str(path)), check=True) # nosec B603 git(path, "config", "user.email", "qualification@example.invalid") git(path, "config", "user.name", "Installed qualification") + (path / ".gitignore").write_text("__pycache__/\n", encoding="utf-8") (path / "BOOTSTRAP.md").write_text("# Installed qualification\n", encoding="utf-8") - git(path, "add", "BOOTSTRAP.md") + (path / "test_qualification_fixture.py").write_text( + "import unittest\n\n" + "class QualificationFixtureTest(unittest.TestCase):\n" + " def test_fixture_is_executable(self):\n" + " self.assertTrue(True)\n", + encoding="utf-8", + ) + git(path, "add", ".gitignore", "BOOTSTRAP.md", "test_qualification_fixture.py") git(path, "commit", "-qm", "initial qualification repository") if origin is not None: subprocess.run(("git", "init", "-q", "--bare", str(origin)), check=True) # nosec B603