diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c1bff885..aeda0bcc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,50 @@ and qualification evidence remains in `docs/REVIVAL_TEST_RESULTS.md`. ### Added +- Gate 14 hardware acceptance now has a fail-closed, source-bound verifier and durable GCP + lifecycle controller. They pin the exact Gate 13 packaged lifecycle and Windows/Qwen plus + Linux/Gemma Gate 9 envelopes, enforce the USD 100 aggregate ledger ceiling, serialize fresh + L4 hosts, bind host-reported evidence before collection, and require cleanup proof for the + exact authorization bytes, controller source, provider plan, project, zone, resources, + and successful terminal state while excluding the protected bootstrap from cleanup targets. + The focused 49-test suite covers deadline, rollback, orphan/returned-resource inventory, forged-state, + evidence-substitution, cross-platform, hidden/excess spend, and cleanup-binding failures. Exact source + `c0f2342e15aa7e12ca7c2980deca64d613204143` passed independent adversarial review, CodeQL, + style, Linux/Windows tests, and production-package provenance verification. A native read-only GCP + preflight stopped before inventory or quota inspection because the configured account requires + interactive reauthentication; no reservation or resource was created, so the current-epoch USD 44 + remainder is intact. This is software and package evidence only: no Gate 14 hardware pass is claimed yet. +- Gate 13's successful manual desktop flow now has a bounded automated replay. The + production package can open its real Qt window in a hidden qualification mode, perform + localhost inference, save the actual sharing-policy dialog, click Start, exit, relaunch, + prove sharing resumed, click Pause, and infer again without retaining prompt, response, + credential, endpoint, or path data. A standard-library outer runner verifies the exact + production archive, runs all four packaged self-tests, executes both sessions, validates + canonical evidence, and removes its run temporaries. The durable host-job boundary now + accepts the Python replay entrypoint on Windows and Linux. Local real-window and contract + tests pass; no paid clean-host replay or new release artifact is claimed by this change. +- The owner raised the current combined GCP/Fly public-alpha accounting epoch to USD 500 on + 2026-08-31. The existing USD 52 committed maximum remains charged. Run + `gate13-20260831-b` reserved USD 56 but failed before VM creation when its two IAP tags reached + gcloud as one value; exact cleanup passed. Fresh run `gate13-20260831-c` reserves USD 56 with + corrected explicit tag arguments, leaving USD 336. Per-run preflight, hard deadlines, exact + cleanup, protected-resource, and evidence requirements are unchanged. +- Gate 13 paid qualification now has durable source-bound native host jobs: an exact-current-user + Windows Scheduled Task and a non-root transient Linux systemd service persist one attempt across + operator disconnects, bound output, terminate the complete process tree on timeout or overflow, + revalidate canonical lifecycle evidence before collection, and never re-arm consumed route or + client intents. Exact Windows lifecycle-config co-location and full Linux `ExecStart` structure + matching close the final independent-review gaps. Source `0e16ac2` passes the 217-test Gate 13 and + desktop matrix independently. This is a software prerequisite only: it created no cloud resources, + authorizes no paid run, and does not claim a completed clean-host lifecycle or Gate 13 pass. +- Gate 13 paid qualification now has a persisted authorization-bound run-state contract that + inventories exact resources before every transition, accepts the product route before any + client, runs Windows/Qwen before Linux/Gemma, permanently consumes a failed or ambiguous + lifecycle host, rejects stale/foreign/deadline-expired observations, validates digest-bound + canonical 16-phase records, and permits a pass only after both records and exact provider + absence. The failed `gate13-20260831-a` attempt is cleanup-proved: its route, clients, disks, + and firewalls are absent while the protected bootstrap remains running. No lifecycle pass is + claimed, and the run's USD 52 maximum remains committed in the current budget epoch. - Gate 13 packaged-lifecycle prerequisites now emit deterministic self-contained Windows ZIP and Linux tar.gz archives, preserve or reject platform filesystem semantics fail-closed, and bind the archive plus strict desktop metrics into exact-type release provenance. The local diff --git a/desktop/build_desktop.py b/desktop/build_desktop.py index 2dac1013d..7ed0aa638 100644 --- a/desktop/build_desktop.py +++ b/desktop/build_desktop.py @@ -1269,6 +1269,8 @@ def main() -> int: str(build_root / "spec"), "--hidden-import", "communityai_desktop.pyside_shell", + "--hidden-import", + "communityai_desktop.gate13_playthrough", "--add-data", f"{icon_path}{os.pathsep}communityai_desktop/assets", ] diff --git a/desktop/src/communityai_desktop/acceptance.py b/desktop/src/communityai_desktop/acceptance.py index 35009ad6f..567d757d2 100644 --- a/desktop/src/communityai_desktop/acceptance.py +++ b/desktop/src/communityai_desktop/acceptance.py @@ -308,8 +308,10 @@ def do_DELETE(self): # noqa: N802 @contextmanager -def fake_node() -> Iterator[Tuple[str, str]]: +def fake_node(*, all_workers_paused: bool = False) -> Iterator[Tuple[str, str]]: state = _FakeNodeState() + if all_workers_paused: + state.worker_states = {worker_id: (model, "paused") for worker_id, (model, _) in state.worker_states.items()} server = ThreadingHTTPServer(("127.0.0.1", 0), _handler(state)) thread = threading.Thread(target=server.serve_forever, name="desktop-acceptance-node", daemon=True) thread.start() diff --git a/desktop/src/communityai_desktop/app.py b/desktop/src/communityai_desktop/app.py index 91c56717a..63b50ac21 100644 --- a/desktop/src/communityai_desktop/app.py +++ b/desktop/src/communityai_desktop/app.py @@ -44,6 +44,8 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--no-manage-node", action="store_true", help=argparse.SUPPRESS) parser.add_argument(LOGIN_STARTUP_FLAG, action="store_true", help=argparse.SUPPRESS) parser.add_argument("--capture-page", type=int, default=0, help=argparse.SUPPRESS) + parser.add_argument("--gate13-ui-evidence", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--gate13-ui-screenshot", type=Path, help=argparse.SUPPRESS) action = parser.add_mutually_exclusive_group() action.add_argument("--store-control-key", action="store_true") action.add_argument("--delete-control-key", action="store_true") @@ -53,6 +55,7 @@ def build_parser() -> argparse.ArgumentParser: action.add_argument("--onboarding-ui-self-test", action="store_true", help=argparse.SUPPRESS) action.add_argument("--capture-ui", type=Path, help=argparse.SUPPRESS) action.add_argument("--probe-only", action="store_true", help=argparse.SUPPRESS) + action.add_argument("--gate13-ui-playthrough", type=Path, help=argparse.SUPPRESS) return parser @@ -69,6 +72,11 @@ def _write_json(value: Any) -> None: def main(argv: Optional[Sequence[str]] = None) -> int: parser = build_parser() args = parser.parse_args(argv) + if args.gate13_ui_playthrough is None: + if args.gate13_ui_evidence is not None or args.gate13_ui_screenshot is not None: + parser.error("Gate 13 evidence options require --gate13-ui-playthrough") + elif args.gate13_ui_evidence is None: + parser.error("--gate13-ui-playthrough requires --gate13-ui-evidence") try: if args.self_test: _write_json(run_self_test()) @@ -150,6 +158,16 @@ def connect() -> DesktopController: token = credential_store.get_or_migrate() return DesktopController(NodeClient(node_url, token, timeout=args.timeout)) + qualification_automation = None + if args.gate13_ui_playthrough is not None: + from communityai_desktop.gate13_playthrough import Gate13Playthrough, PlaythroughPlan + + qualification_automation = Gate13Playthrough( + PlaythroughPlan.load(args.gate13_ui_playthrough), + args.gate13_ui_evidence, + screenshot_path=args.gate13_ui_screenshot, + ) + if args.probe_only: try: _write_json(connect().snapshot()) @@ -169,6 +187,7 @@ def connect() -> DesktopController: start_minimized=args.started_at_login, activate_existing_instance=not args.started_at_login, before_termination_restore=None if lifecycle is None else lifecycle.close, + qualification_automation=qualification_automation, ) or 0 ) diff --git a/desktop/src/communityai_desktop/gate13_playthrough.py b/desktop/src/communityai_desktop/gate13_playthrough.py new file mode 100644 index 000000000..89c4c4a7d --- /dev/null +++ b/desktop/src/communityai_desktop/gate13_playthrough.py @@ -0,0 +1,826 @@ +"""Automate the real packaged Gate 13 desktop playthrough. + +This module is deliberately part of the frozen desktop rather than an external UI +mock. Qualification invocations reproduce the platform-specific sequence from the +accepted manual run using the normal window, real sharing-policy dialog, literal +Start/Pause buttons, and bounded localhost inference with an ephemeral client key. +It retains only bounded acceptance facts. +""" + +from __future__ import annotations + +import json +import math +import os +import re +import stat +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence +from urllib.error import HTTPError, URLError +from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener + +from communityai_desktop.client import normalize_loopback_url + +SCHEMA_VERSION = 2 +SCOPE = "gate13-packaged-desktop-playthrough" +MAX_CONFIG_BYTES = 65_536 +MAX_RESPONSE_BYTES = 1_048_576 +QUALIFICATION_KEY_LABEL = "Gate 13 automated qualification" +MANUAL_ROUTE_WAIT_SECONDS = 450.0 +MANUAL_ROUTE_POLL_SECONDS = 5.0 + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_MODEL_RE = re.compile(r"[ -~]{1,128}") +_POLICY_FIELDS = { + "sharing_enabled", + "allowed_models", + "preferred_models", + "denied_models", + "max_disk_space", + "max_vram", + "max_bandwidth_mbps", + "max_power_watts", + "pause_timeout", + "schedule", +} + + +def _manual_schedule() -> dict[str, Any]: + return { + "timezone": "UTC", + "windows": [ + { + "days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"], + "start": "00:00", + "end": "23:59", + } + ], + } + + +_CONFIG_FIELDS = { + "schema_version", + "run_id", + "platform", + "stage", + "model_id", + "manifest_digest", + "total_blocks", + "policy", + "timeout_seconds", + "inference_timeout_seconds", +} + + +class PlaythroughError(ValueError): + """A qualification plan or observed desktop state failed closed.""" + + +def _reject_constant(_value: str) -> None: + raise PlaythroughError("configuration contains a non-finite value") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise PlaythroughError("configuration contains a duplicate field") + value[key] = item + return value + + +def _regular_bytes(path: Path, maximum: int) -> bytes: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise PlaythroughError("configuration is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= maximum: + raise PlaythroughError("configuration is not a bounded regular file") + try: + return path.read_bytes() + except OSError as exc: + raise PlaythroughError("configuration is unreadable") from exc + + +def _bounded_number(value: Any, label: str, *, minimum: float, maximum: float) -> float: + if type(value) not in (int, float): + raise PlaythroughError(f"{label} is invalid") + rendered = float(value) + if not math.isfinite(rendered) or not minimum <= rendered <= maximum: + raise PlaythroughError(f"{label} is invalid") + return rendered + + +def _selectors(value: Any, label: str) -> tuple[str, ...]: + if not isinstance(value, list) or len(value) > 8: + raise PlaythroughError(f"{label} is invalid") + clean: list[str] = [] + folded: set[str] = set() + for item in value: + if not isinstance(item, str) or _MODEL_RE.fullmatch(item) is None or item != item.strip(): + raise PlaythroughError(f"{label} is invalid") + canonical = item.casefold() + if canonical in folded: + raise PlaythroughError(f"{label} contains a duplicate selector") + folded.add(canonical) + clean.append(item) + return tuple(clean) + + +def _policy(value: Any, model_id: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != _POLICY_FIELDS: + raise PlaythroughError("sharing policy schema is invalid") + allowed = _selectors(value["allowed_models"], "allowed models") + preferred = _selectors(value["preferred_models"], "preferred models") + denied = _selectors(value["denied_models"], "denied models") + if value["sharing_enabled"] is not True: + raise PlaythroughError("sharing must be enabled for the start stage") + if model_id not in allowed or model_id not in preferred or denied: + raise PlaythroughError("sharing policy does not select the qualification model") + if value["max_disk_space"] != "32GB": + raise PlaythroughError("storage ceiling does not match the proven manual replay") + if value["max_vram"] != "20GB": + raise PlaythroughError("memory ceiling does not match the proven manual replay") + bandwidth = _bounded_number(value["max_bandwidth_mbps"], "bandwidth ceiling", minimum=0.001, maximum=1_000_000) + if bandwidth != 100.0: + raise PlaythroughError("bandwidth ceiling does not match the proven manual replay") + if value["max_power_watts"] is not None: + raise PlaythroughError("the manual CPU-host replay requires an unset power ceiling") + pause = _bounded_number(value["pause_timeout"], "pause timeout", minimum=1, maximum=300) + if pause != 120.0: + raise PlaythroughError("pause timeout does not match the proven manual replay") + if value["schedule"] != _manual_schedule(): + raise PlaythroughError("sharing schedule does not match the proven manual replay") + return { + "sharing_enabled": True, + "allowed_models": list(allowed), + "preferred_models": list(preferred), + "denied_models": list(denied), + "max_disk_space": value["max_disk_space"], + "max_vram": value["max_vram"], + "max_bandwidth_mbps": bandwidth, + "max_power_watts": None, + "pause_timeout": pause, + "schedule": _manual_schedule(), + } + + +@dataclass(frozen=True) +class PlaythroughPlan: + run_id: str + platform: str + stage: str + model_id: str + manifest_digest: str + total_blocks: int + policy: Mapping[str, Any] + timeout_seconds: float + inference_timeout_seconds: float + + @classmethod + def load(cls, path: Path) -> "PlaythroughPlan": + payload = _regular_bytes(path, MAX_CONFIG_BYTES) + try: + raw = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PlaythroughError("configuration is invalid JSON") from exc + if not isinstance(raw, dict) or set(raw) != _CONFIG_FIELDS or raw.get("schema_version") != SCHEMA_VERSION: + raise PlaythroughError("configuration schema is invalid") + run_id = raw["run_id"] + platform = raw["platform"] + stage = raw["stage"] + model_id = raw["model_id"] + digest = raw["manifest_digest"] + total_blocks = raw["total_blocks"] + if not isinstance(run_id, str) or _RUN_RE.fullmatch(run_id) is None: + raise PlaythroughError("run id is invalid") + if platform not in ("windows", "linux"): + raise PlaythroughError("playthrough platform is invalid") + if stage not in ("initial", "restart"): + raise PlaythroughError("playthrough stage is invalid") + if not isinstance(model_id, str) or _MODEL_RE.fullmatch(model_id) is None or model_id != model_id.strip(): + raise PlaythroughError("model id is invalid") + if not isinstance(digest, str) or _DIGEST_RE.fullmatch(digest) is None: + raise PlaythroughError("manifest digest is invalid") + if type(total_blocks) is not int or not 1 <= total_blocks <= 512: + raise PlaythroughError("block count is invalid") + timeout = _bounded_number(raw["timeout_seconds"], "playthrough timeout", minimum=30, maximum=3_600) + inference_timeout = _bounded_number( + raw["inference_timeout_seconds"], "inference timeout", minimum=10, maximum=600 + ) + return cls( + run_id=run_id, + platform=platform, + stage=stage, + model_id=model_id, + manifest_digest=digest, + total_blocks=total_blocks, + policy=_policy(raw["policy"], model_id), + timeout_seconds=timeout, + inference_timeout_seconds=inference_timeout, + ) + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ARG002 + return None + + +def _completion_request(url: str, secret: str, timeout: float) -> Mapping[str, Any]: + body = json.dumps( + { + "model": "auto", + "messages": [{"role": "user", "content": "Reply with one word."}], + "max_tokens": 1, + "stream": False, + }, + separators=(",", ":"), + ).encode("utf-8") + request = Request( + url, + data=body, + method="POST", + headers={ + "Authorization": f"Bearer {secret}", + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) + opener = build_opener(ProxyHandler({}), _RejectRedirects()) + try: + with opener.open(request, timeout=timeout) as response: + if response.status != 200 or response.headers.get_content_type() != "application/json": + raise PlaythroughError("localhost inference was rejected") + payload = response.read(MAX_RESPONSE_BYTES + 1) + except (HTTPError, URLError, OSError, TimeoutError) as exc: + raise PlaythroughError("localhost inference failed") from exc + if not 1 <= len(payload) <= MAX_RESPONSE_BYTES: + raise PlaythroughError("localhost inference response is invalid") + try: + value = json.loads(payload.decode("utf-8"), parse_constant=_reject_constant) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PlaythroughError("localhost inference response is invalid") from exc + if not isinstance(value, dict): + raise PlaythroughError("localhost inference response is invalid") + return value + + +def _manual_route_ready(status: Mapping[str, Any], plan: PlaythroughPlan) -> bool: + selection = status.get("auto_selection") + models = status.get("models") + if not isinstance(selection, dict) or not isinstance(models, list): + return False + selected = next( + ( + item + for item in models + if isinstance(item, dict) + and item.get("id") == plan.model_id + and item.get("manifest_digest") == plan.manifest_digest + ), + None, + ) + return bool( + selection.get("status") == "selected" + and selection.get("model") == plan.model_id + and selection.get("manifest_digest") == plan.manifest_digest + and selection.get("covered_blocks") == plan.total_blocks + and selection.get("total_blocks") == plan.total_blocks + and isinstance(selection.get("peer_count"), int) + and selection["peer_count"] > 0 + and selected is not None + and selected.get("route_complete") is True + and selected.get("covered_blocks") == plan.total_blocks + and selected.get("total_blocks") == plan.total_blocks + ) + + +def _completion_after_manual_readiness_wait( + controller: Any, + plan: PlaythroughPlan, + url: str, + secret: str, +) -> Mapping[str, Any]: + """Replay the manual Model-unavailable -> wait-for-complete -> retry sequence.""" + + try: + return _completion_request(url, secret, plan.inference_timeout_seconds) + except PlaythroughError as first_error: + deadline = time.monotonic() + min(MANUAL_ROUTE_WAIT_SECONDS, plan.inference_timeout_seconds) + while time.monotonic() < deadline: + time.sleep(min(MANUAL_ROUTE_POLL_SECONDS, max(0.0, deadline - time.monotonic()))) + try: + status = controller.client.status() + except BaseException: + continue + if _manual_route_ready(status, plan): + try: + return _completion_request(url, secret, plan.inference_timeout_seconds) + except PlaythroughError: + raise first_error + raise first_error + + +def qualify_localhost_inference(controller: Any, plan: PlaythroughPlan) -> dict[str, Any]: + """Run one response-content-free localhost inference and restore the API-key baseline.""" + + baseline_items = [item for item in controller.client.list_keys() if item.get("revoked_at") is None] + baseline = {item["id"] for item in baseline_items} + if not baseline: + raise PlaythroughError("a preexisting client key is required") + if any(item.get("label") == QUALIFICATION_KEY_LABEL for item in baseline_items): + raise PlaythroughError("a prior qualification key remains active") + created_id = "" + secret = "" + failed = False + cleanup_failed = False + completion_count = 0 + generated_token_count = 0 + try: + status = controller.client.status() + selection = status["auto_selection"] + if ( + selection.get("status") != "selected" + or selection.get("model") != plan.model_id + or selection.get("manifest_digest") != plan.manifest_digest + ): + raise PlaythroughError("automatic selection changed before inference") + created = controller.client.create_key(QUALIFICATION_KEY_LABEL) + created_id = created.get("key", {}).get("id", "") + secret = created.get("secret", "") + if ( + not isinstance(created_id, str) + or not created_id + or not isinstance(secret, str) + or not 1 <= len(secret) <= 512 + ): + raise PlaythroughError("temporary client key response is invalid") + base = normalize_loopback_url(status["openai_base_url"]) + completion = _completion_after_manual_readiness_wait( + controller, + plan, + f"{base}/v1/chat/completions", + secret, + ) + if completion.get("model") != plan.model_id: + raise PlaythroughError("localhost inference identity is invalid") + usage = completion.get("usage") + generated = usage.get("completion_tokens") if isinstance(usage, dict) else None + if type(generated) is not int or generated != 1: + raise PlaythroughError("localhost inference token count is invalid") + completion_count = 1 + generated_token_count = generated + except BaseException: + failed = True + finally: + secret = "" + try: + active = {item["id"] for item in controller.client.list_keys() if item.get("revoked_at") is None} + candidates = active - baseline + if created_id and created_id in active: + candidates.add(created_id) + if len(candidates) != 1: + raise PlaythroughError("temporary client key identity is ambiguous") + controller.client.revoke_key(next(iter(candidates))) + after = {item["id"] for item in controller.client.list_keys() if item.get("revoked_at") is None} + if after != baseline: + raise PlaythroughError("temporary client key cleanup failed") + except BaseException: + cleanup_failed = True + if failed or cleanup_failed: + raise PlaythroughError("localhost inference or key cleanup failed") + return { + "passed": True, + "model_id": plan.model_id, + "manifest_digest": plan.manifest_digest, + "completion_count": completion_count, + "generated_token_count": generated_token_count, + "response_content_retained": False, + "token_identifiers_retained": False, + "temporary_key_removed": True, + } + + +def _atomic_json(path: Path, value: Mapping[str, Any]) -> None: + destination = Path(path).absolute() + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists() and (destination.is_symlink() or not destination.is_file()): + raise PlaythroughError("evidence destination is unsafe") + payload = (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + temporary_name = "" + try: + with tempfile.NamedTemporaryFile( + prefix=".gate13-playthrough-", suffix=".tmp", dir=destination.parent, delete=False + ) as out: + temporary_name = out.name + out.write(payload) + out.flush() + os.fsync(out.fileno()) + os.replace(temporary_name, destination) + temporary_name = "" + except OSError as exc: + raise PlaythroughError("evidence could not be persisted") from exc + finally: + if temporary_name: + try: + Path(temporary_name).unlink() + except OSError: + pass + + +class Gate13Playthrough: + """A bounded Qt state machine that drives the real packaged controls.""" + + def __init__( + self, + plan: PlaythroughPlan, + evidence_path: Path, + *, + screenshot_path: Path | None = None, + inference_runner: Callable[[Any, PlaythroughPlan], Mapping[str, Any]] = qualify_localhost_inference, + clock: Callable[[], float] = time.monotonic, + start_observation_seconds: float | None = None, + restart_observation_seconds: float | None = None, + ): + self.plan = plan + self.evidence_path = Path(evidence_path) + self.screenshot_path = None if screenshot_path is None else Path(screenshot_path) + self._inference_runner = inference_runner + self._clock = clock + default_start_observation = 25.0 if plan.platform == "windows" else 20.0 + self._start_observation_seconds = ( + default_start_observation + if start_observation_seconds is None + else _bounded_number(start_observation_seconds, "start observation", minimum=0.05, maximum=60) + ) + self._restart_observation_seconds = ( + 15.0 + if restart_observation_seconds is None + else _bounded_number(restart_observation_seconds, "restart observation", minimum=0.05, maximum=60) + ) + self._started = clock() + self._state = "wait_ready" + self._done = False + self._inference: Mapping[str, Any] | None = None + self._window = None + self._application = None + self._qt: Mapping[str, Any] = {} + self._timer = None + self._observation_deadline: float | None = None + self._ui = { + "real_window_opened": True, + "policy_dialog_saved": False, + "start_clicked": False, + "pause_control_observed": False, + "pause_clicked": False, + "restart_resume_observed": False, + "sharing_intent_enabled_observed": False, + "sharing_intent_disabled_observed": False, + } + + def install(self, window: Any, application: Any, qt: Mapping[str, Any]) -> None: + self._window = window + self._application = application + self._qt = qt + timer_type = qt["QTimer"] + self._timer = timer_type(window) + self._timer.setInterval(200) + self._timer.timeout.connect(self._tick) + self._timer.start() + timer_type.singleShot(max(1, int(self.plan.timeout_seconds * 1_000)), self._timeout) + + def _timeout(self) -> None: + if not self._done: + self._fail() + + def _ready(self) -> bool: + window = self._window + if window is None or window._controller is None or window._busy: + return False + snapshot = window._snapshot + selection = snapshot.get("auto_selection", {}) + models = snapshot.get("models", []) + selected = next((item for item in models if item.get("id") == self.plan.model_id), None) + return bool( + selection.get("status") == "selected" + and selection.get("model") == self.plan.model_id + and selection.get("manifest_digest") == self.plan.manifest_digest + and selection.get("covered_blocks") == self.plan.total_blocks + and selection.get("total_blocks") == self.plan.total_blocks + and isinstance(selection.get("peer_count"), int) + and selection["peer_count"] > 0 + and selected is not None + and selected.get("route_complete") is True + and selected.get("covered_blocks") == self.plan.total_blocks + and selected.get("total_blocks") == self.plan.total_blocks + ) + + def _tick(self) -> None: + if self._done or self._window is None: + return + try: + if self._state == "wait_ready": + if not self._ready(): + return + if self.plan.stage == "initial": + self._begin_inference("after_initial_inference") + elif self.plan.platform == "windows": + self._begin_policy_edit() + else: + if not self._show_sharing_page(): + self._fail() + return + self._state = "wait_resumed" + elif self._state == "wait_policy": + contribution = self._window._snapshot.get("contribution", {}) + if not self._window._busy and contribution.get("policy") == self.plan.policy: + self._ui["policy_dialog_saved"] = True + # The manual Windows run toggled the selected model before + # using the master Start control. Automatic placement can + # race the policy refresh and start that model first; in + # that state the master control already says Pause and the + # old replay waited forever for a Start button. Restore a + # paused baseline through the literal per-model control, + # then replay the literal master Start action. + if contribution.get("intent_enabled"): + self._click_model_toggle_to_pause() + else: + self._click_start() + elif self._state == "wait_prestart_paused": + contribution = self._window._snapshot.get("contribution", {}) + if ( + not self._window._busy + and not contribution.get("intent_enabled") + and self._start_control_available() + ): + self._click_start() + elif self._state == "wait_started_intent": + contribution = self._window._snapshot.get("contribution", {}) + if contribution.get("intent_enabled") and self._pause_control_available(): + self._ui["sharing_intent_enabled_observed"] = True + self._ui["pause_control_observed"] = True + if self._observation_deadline is None: + self._observation_deadline = self._clock() + self._start_observation_seconds + if self._clock() >= self._observation_deadline: + if self.plan.platform == "windows": + self._click_pause() + else: + self._pass() + elif self._state == "wait_resumed": + contribution = self._window._snapshot.get("contribution", {}) + workers = self._window._snapshot.get("workers", []) + desired = any( + item.get("model") == self.plan.model_id and item.get("desired_running") for item in workers + ) + if contribution.get("intent_enabled") and desired and self._pause_control_available(): + self._ui["sharing_intent_enabled_observed"] = True + self._ui["pause_control_observed"] = True + self._ui["restart_resume_observed"] = True + if self._observation_deadline is None: + self._observation_deadline = self._clock() + self._restart_observation_seconds + if self._clock() >= self._observation_deadline: + self._click_pause() + elif self._state == "wait_paused_intent": + contribution = self._window._snapshot.get("contribution", {}) + if ( + not self._window._busy + and not contribution.get("intent_enabled") + and self._start_control_available() + ): + self._ui["sharing_intent_disabled_observed"] = True + if self.plan.platform == "linux": + self._begin_inference("after_restart_inference") + else: + self._pass() + except BaseException: + self._fail() + + def _begin_inference(self, waiting_state: str) -> None: + self._state = waiting_state + controller = self._window._controller + + def finished(result: Mapping[str, Any]) -> None: + self._inference = dict(result) + if waiting_state == "after_initial_inference" and self.plan.platform == "linux": + self._begin_policy_edit() + else: + self._pass() + + self._window._submit( + lambda: self._inference_runner(controller, self.plan), + finished, + lambda _message: self._fail(), + ) + + def _begin_policy_edit(self) -> None: + if not self._show_sharing_page(): + self._fail() + return + if self._window.edit_policy_button.isEnabled() is False: + self._fail() + return + self._state = "editing_policy" + self._qt["QTimer"].singleShot(100, self._fill_policy_dialog) + self._window.edit_policy_button.click() + + def _fill_policy_dialog(self) -> None: + try: + dialog = self._window.findChild(self._qt["QDialog"], "sharingPolicyDialog") + if dialog is None: + raise PlaythroughError("sharing policy dialog did not open") + checkbox = dialog.findChild(self._qt["QCheckBox"], "policy_sharing_enabled") + checkbox.setChecked(True) + for field in ("allowed_models", "preferred_models", "denied_models"): + editor = dialog.findChild(self._qt["QPlainTextEdit"], f"policy_{field}") + editor.setPlainText("\n".join(self.plan.policy[field])) + for field in ( + "max_disk_space", + "max_vram", + "max_bandwidth_mbps", + "max_power_watts", + "pause_timeout", + ): + editor = dialog.findChild(self._qt["QLineEdit"], f"policy_{field}") + value = self.plan.policy[field] + editor.setText("" if value is None else f"{value:g}" if isinstance(value, float) else str(value)) + schedule = dialog.findChild(self._qt["QPlainTextEdit"], "policy_schedule") + schedule.setPlainText(json.dumps(self.plan.policy["schedule"], separators=(",", ":"))) + buttons = dialog.findChild(self._qt["QDialogButtonBox"], "sharingPolicyButtons") + save = buttons.button(self._qt["QDialogButtonBox"].StandardButton.Save) + self._state = "wait_policy" + save.click() + except BaseException: + self._fail() + + def _show_sharing_page(self) -> bool: + window = self._window + buttons = None if window is None else getattr(window, "_page_buttons", None) + if not isinstance(buttons, list) or len(buttons) != 4: + return False + button = buttons[2] + if button.text() != "Sharing" or not button.isEnabled(): + return False + if not button.isChecked(): + button.click() + return bool(button.isChecked()) + + def _click_start(self) -> None: + if not self._show_sharing_page(): + self._fail() + return + button = self._window.master_share_button + if button.text() != "Start sharing" or not button.isEnabled(): + return + self._state = "wait_started_intent" + self._observation_deadline = None + self._ui["start_clicked"] = True + button.click() + + def _click_model_toggle_to_pause(self) -> None: + if not self._show_sharing_page(): + self._fail() + return + checkbox_type = self._qt.get("QCheckBox") + if checkbox_type is None: + self._fail() + return + expected_name = f"Share compute with {self.plan.model_id}" + matches = [ + checkbox + for checkbox in self._window.findChildren(checkbox_type) + if checkbox.accessibleName() == expected_name + ] + desired = any( + worker.get("model") == self.plan.model_id and worker.get("desired_running") + for worker in self._window._snapshot.get("workers", []) + ) + if len(matches) != 1 or not desired or not matches[0].isChecked(): + self._fail() + return + if not matches[0].isEnabled(): + return + self._state = "wait_prestart_paused" + matches[0].click() + + def _click_pause(self) -> None: + if not self._show_sharing_page(): + self._fail() + return + button = self._window.master_share_button + if button.text() != "Pause sharing" or not button.isEnabled(): + return + self._state = "wait_paused_intent" + self._observation_deadline = None + self._ui["pause_clicked"] = True + button.click() + + def _pause_control_available(self) -> bool: + button = self._window.master_share_button + return bool(button.text() == "Pause sharing" and button.isEnabled()) + + def _start_control_available(self) -> bool: + button = self._window.master_share_button + # A just-paused worker may remain temporarily resource-suspended while its + # process exits, which legitimately leaves Start disabled. The manual run + # accepted the intent transition and literal control text at this boundary. + return bool(button.text() == "Start sharing") + + def _base_result(self, result: str) -> dict[str, Any]: + duration = max(0.0, self._clock() - self._started) + return { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "run_id": self.plan.run_id, + "platform": self.plan.platform, + "stage": self.plan.stage, + "result": result, + "model_id": self.plan.model_id, + "manifest_digest": self.plan.manifest_digest, + "duration_seconds": round(duration, 6), + } + + def _pass(self) -> None: + inference_required = (self.plan.platform, self.plan.stage) in { + ("windows", "initial"), + ("linux", "initial"), + ("linux", "restart"), + } + if self._done or (inference_required and self._inference is None): + self._fail() + return + value = self._base_result("passed") + value.update( + { + "route": { + "rendered_in_real_window": True, + "complete": True, + "covered_blocks": self.plan.total_blocks, + "total_blocks": self.plan.total_blocks, + }, + "inference": None if self._inference is None else dict(self._inference), + "ui": dict(self._ui), + "limits": { + "storage": self._ui["policy_dialog_saved"], + "memory_or_vram": self._ui["policy_dialog_saved"], + "bandwidth": self._ui["policy_dialog_saved"], + "power": False, + "pause_timeout": self._ui["policy_dialog_saved"], + "schedule": self._ui["policy_dialog_saved"], + }, + "timing": { + "start_observation_seconds": ( + self._start_observation_seconds if self._ui["start_clicked"] else 0.0 + ), + "restart_observation_seconds": ( + self._restart_observation_seconds if self._ui["restart_resume_observed"] else 0.0 + ), + }, + "privacy": { + "prompt_retained": False, + "response_content_retained": False, + "token_identifiers_retained": False, + "credentials_retained": False, + "paths_retained": False, + "endpoints_retained": False, + }, + } + ) + self._finish(value) + + def _fail(self) -> None: + if self._done: + return + value = self._base_result("failed") + value["failure_code"] = "playthrough_failed" + self._finish(value) + + def _finish(self, value: Mapping[str, Any]) -> None: + self._done = True + if self._timer is not None: + self._timer.stop() + try: + if self.screenshot_path is not None and self._window is not None: + self.screenshot_path.parent.mkdir(parents=True, exist_ok=True) + if not self._window.grab().save(str(self.screenshot_path)): + raise PlaythroughError("playthrough screenshot failed") + _atomic_json(self.evidence_path, value) + except BaseException: + fallback = self._base_result("failed") + fallback["failure_code"] = "evidence_write_failed" + try: + _atomic_json(self.evidence_path, fallback) + except BaseException: + pass + finally: + if self._application is not None: + self._application.quit() diff --git a/desktop/src/communityai_desktop/pyside_shell.py b/desktop/src/communityai_desktop/pyside_shell.py index d354e704b..db59231cf 100644 --- a/desktop/src/communityai_desktop/pyside_shell.py +++ b/desktop/src/communityai_desktop/pyside_shell.py @@ -188,6 +188,7 @@ def run( activate_existing_instance: bool = True, instance_name: str | None = None, before_termination_restore: Callable[[], None] | None = None, + qualification_automation=None, # noqa: ANN001 ) -> int: if controller is None and connect is None: raise ValueError("the desktop requires an initial controller or connector") @@ -1060,6 +1061,7 @@ def _edit_contribution_policy(self) -> None: return dialog = QDialog(self) + dialog.setObjectName("sharingPolicyDialog") dialog.setWindowTitle("Edit sharing limits") dialog.setMinimumWidth(620) layout = QVBoxLayout(dialog) @@ -1072,6 +1074,7 @@ def _edit_contribution_policy(self) -> None: form = QFormLayout() sharing_enabled = QCheckBox("Allow this node to share compute") + sharing_enabled.setObjectName("policy_sharing_enabled") sharing_enabled.setChecked(policy["sharing_enabled"]) form.addRow("Sharing", sharing_enabled) @@ -1082,6 +1085,7 @@ def _edit_contribution_policy(self) -> None: ("denied_models", "Denied models"), ): editor = QPlainTextEdit() + editor.setObjectName(f"policy_{field}") editor.setPlainText("\n".join(policy[field])) editor.setPlaceholderText("One exact model selector per line") editor.setFixedHeight(64) @@ -1098,6 +1102,7 @@ def _edit_contribution_policy(self) -> None: ("pause_timeout", "Pause timeout (seconds)", "10"), ): editor = QLineEdit() + editor.setObjectName(f"policy_{field}") value = policy[field] editor.setText("" if value is None else f"{value:g}" if isinstance(value, float) else str(value)) editor.setPlaceholderText(placeholder) @@ -1106,6 +1111,7 @@ def _edit_contribution_policy(self) -> None: form.addRow(title, editor) schedule = QPlainTextEdit() + schedule.setObjectName("policy_schedule") schedule.setPlainText("" if policy["schedule"] is None else json.dumps(policy["schedule"], indent=2)) schedule.setPlaceholderText( '{"timezone":"local","windows":[{"days":["mon"],"start":"22:00","end":"06:00"}]}' @@ -1116,6 +1122,7 @@ def _edit_contribution_policy(self) -> None: layout.addLayout(form) buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Save | QDialogButtonBox.StandardButton.Cancel) + buttons.setObjectName("sharingPolicyButtons") buttons.accepted.connect(dialog.accept) buttons.rejected.connect(dialog.reject) layout.addWidget(buttons) @@ -1311,6 +1318,20 @@ def notify_existing_instance(timeout_ms: int) -> bool: else: window.show() + if qualification_automation is not None: + qualification_automation.install( + window, + application, + { + "QTimer": QTimer, + "QDialog": QDialog, + "QDialogButtonBox": QDialogButtonBox, + "QCheckBox": QCheckBox, + "QPlainTextEdit": QPlainTextEdit, + "QLineEdit": QLineEdit, + }, + ) + if instance_server is not None: def activate_window() -> None: diff --git a/desktop/tests/test_gate13_playthrough.py b/desktop/tests/test_gate13_playthrough.py new file mode 100644 index 000000000..7a3d5028a --- /dev/null +++ b/desktop/tests/test_gate13_playthrough.py @@ -0,0 +1,520 @@ +from __future__ import annotations + +import json +import os +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from communityai_desktop.acceptance import fake_node +from communityai_desktop.app import main +from communityai_desktop.client import NodeClient +from communityai_desktop.controller import DesktopController +from communityai_desktop.gate13_playthrough import ( + Gate13Playthrough, + PlaythroughError, + PlaythroughPlan, + qualify_localhost_inference, +) + +MODEL_ID = "Qwen 3 8B" +MANIFEST_DIGEST = "sha256:" + "b" * 64 + + +def _config(stage: str, platform: str = "windows") -> dict: + return { + "schema_version": 2, + "run_id": "gate13-automated-test", + "platform": platform, + "stage": stage, + "model_id": MODEL_ID, + "manifest_digest": MANIFEST_DIGEST, + "total_blocks": 36, + "policy": { + "sharing_enabled": True, + "allowed_models": [MODEL_ID], + "preferred_models": [MODEL_ID], + "denied_models": [], + "max_disk_space": "32GB", + "max_vram": "20GB", + "max_bandwidth_mbps": 100.0, + "max_power_watts": None, + "pause_timeout": 120.0, + "schedule": { + "timezone": "UTC", + "windows": [ + { + "days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"], + "start": "00:00", + "end": "23:59", + } + ], + }, + }, + "timeout_seconds": 30.0, + "inference_timeout_seconds": 10.0, + } + + +def _write_plan(path: Path, stage: str, platform: str = "windows") -> PlaythroughPlan: + path.write_text(json.dumps(_config(stage, platform)), encoding="utf-8") + return PlaythroughPlan.load(path) + + +def _inference(_controller, plan): # noqa: ANN001 + return { + "passed": True, + "model_id": plan.model_id, + "manifest_digest": plan.manifest_digest, + "completion_count": 1, + "generated_token_count": 1, + "response_content_retained": False, + "token_identifiers_retained": False, + "temporary_key_removed": True, + } + + +class PlaythroughPlanTests(unittest.TestCase): + def test_plan_is_strict_and_bounded(self): + from tempfile import TemporaryDirectory + + with TemporaryDirectory() as directory: + root = Path(directory) + plan = _write_plan(root / "plan.json", "initial") + self.assertEqual(plan.model_id, MODEL_ID) + self.assertEqual(plan.policy["allowed_models"], [MODEL_ID]) + self.assertIsNone(plan.policy["max_power_watts"]) + + invalid = _config("initial") + invalid["policy"]["denied_models"] = [MODEL_ID] + (root / "invalid.json").write_text(json.dumps(invalid), encoding="utf-8") + with self.assertRaises(PlaythroughError): + PlaythroughPlan.load(root / "invalid.json") + + invalid_power = _config("initial") + invalid_power["policy"]["max_power_watts"] = 250.0 + (root / "invalid-power.json").write_text(json.dumps(invalid_power), encoding="utf-8") + with self.assertRaises(PlaythroughError): + PlaythroughPlan.load(root / "invalid-power.json") + + (root / "duplicate.json").write_text('{"schema_version":2,"schema_version":2}', encoding="utf-8") + with self.assertRaises(PlaythroughError): + PlaythroughPlan.load(root / "duplicate.json") + + def test_localhost_inference_restores_key_baseline_and_retains_only_counts(self): + plan = PlaythroughPlan( + run_id="gate13-automated-test", + platform="windows", + stage="initial", + model_id=MODEL_ID, + manifest_digest=MANIFEST_DIGEST, + total_blocks=36, + policy=_config("initial")["policy"], + timeout_seconds=30, + inference_timeout_seconds=10, + ) + + class Client: + def __init__(self): + self.active = { + "baseline": { + "id": "baseline", + "label": "baseline", + "revoked_at": None, + } + } + + def list_keys(self): + return list(self.active.values()) + + def status(self): + return { + "openai_base_url": "http://127.0.0.1:8080/v1", + "auto_selection": { + "status": "selected", + "model": MODEL_ID, + "manifest_digest": MANIFEST_DIGEST, + "covered_blocks": plan.total_blocks, + "total_blocks": plan.total_blocks, + "peer_count": 1, + }, + "models": [ + { + "id": MODEL_ID, + "manifest_digest": MANIFEST_DIGEST, + "route_complete": True, + "covered_blocks": plan.total_blocks, + "total_blocks": plan.total_blocks, + } + ], + } + + def create_key(self, label): + self.active["temporary"] = {"id": "temporary", "label": label, "revoked_at": None} + return {"key": self.active["temporary"], "secret": "temporary-secret"} + + def revoke_key(self, key_id): + self.active[key_id]["revoked_at"] = 1 + return {"key": self.active[key_id]} + + client = Client() + completion = { + "model": MODEL_ID, + # The manual Gate 13 command deliberately retained no generated + # content. A one-token response is qualified by identity and the + # server-reported token count, not by decoded visible text. + "choices": [{"message": {"role": "assistant", "content": ""}}], + "usage": {"completion_tokens": 1}, + } + with patch("communityai_desktop.gate13_playthrough._completion_request", return_value=completion): + result = qualify_localhost_inference(SimpleNamespace(client=client), plan) + + self.assertEqual(result["completion_count"], 1) + self.assertFalse(result["response_content_retained"]) + self.assertEqual({item["id"] for item in client.list_keys() if item["revoked_at"] is None}, {"baseline"}) + + unavailable_then_ready = MagicMock(side_effect=[PlaythroughError("localhost inference failed"), completion]) + with ( + patch( + "communityai_desktop.gate13_playthrough._completion_request", + unavailable_then_ready, + ), + patch("communityai_desktop.gate13_playthrough.time.sleep") as readiness_sleep, + ): + retried = qualify_localhost_inference(SimpleNamespace(client=client), plan) + + self.assertTrue(retried["passed"]) + self.assertEqual(unavailable_then_ready.call_count, 2) + readiness_sleep.assert_called_once_with(5.0) + self.assertEqual({item["id"] for item in client.list_keys() if item["revoked_at"] is None}, {"baseline"}) + + def test_localhost_inference_requests_exactly_one_token(self): + from communityai_desktop.gate13_playthrough import _completion_request + + response = MagicMock() + response.__enter__.return_value = response + response.status = 200 + response.headers.get_content_type.return_value = "application/json" + response.read.return_value = b'{"result":"bounded"}' + opener = MagicMock() + opener.open.return_value = response + + with patch("communityai_desktop.gate13_playthrough.build_opener", return_value=opener): + result = _completion_request("http://127.0.0.1:8080/v1/chat/completions", "secret", 10) + + self.assertEqual(result, {"result": "bounded"}) + request = opener.open.call_args.args[0] + self.assertEqual( + json.loads(request.data), + { + "model": "auto", + "messages": [{"role": "user", "content": "Reply with one word."}], + "max_tokens": 1, + "stream": False, + }, + ) + + def test_hidden_packaged_cli_installs_the_qualification_automation(self): + lifecycle = SimpleNamespace(close=lambda: None) + loaded_plan = SimpleNamespace(stage="initial") + automation = SimpleNamespace() + with ( + patch("communityai_desktop.app.NodeLifecycleSupervisor", return_value=lifecycle), + patch("communityai_desktop.gate13_playthrough.PlaythroughPlan.load", return_value=loaded_plan), + patch("communityai_desktop.gate13_playthrough.Gate13Playthrough", return_value=automation), + patch("communityai_desktop.pyside_shell.run", return_value=0) as run, + ): + result = main( + [ + "--gate13-ui-playthrough", + "plan.json", + "--gate13-ui-evidence", + "evidence.json", + ] + ) + + self.assertEqual(result, 0) + self.assertIs(run.call_args.kwargs["qualification_automation"], automation) + + with self.assertRaises(SystemExit): + main(["--self-test", "--gate13-ui-evidence", "orphan.json"]) + + +class PackagedUiPlaythroughTests(unittest.TestCase): + def test_policy_auto_start_is_normalized_through_model_toggle_before_master_start(self): + from tempfile import TemporaryDirectory + + class PageButton: + def __init__(self, label): + self.label = label + self.enabled = True + self.checked = label == "Home" + + def text(self): + return self.label + + def isEnabled(self): + return self.enabled + + def isChecked(self): + return self.checked + + def click(self): + self.checked = True + + class Window: + def __init__(self, policy): + self._busy = 0 + self._page_buttons = [PageButton(label) for label in ("Home", "Models", "Sharing", "API access")] + self._snapshot = { + "contribution": {"intent_enabled": True, "enabled": True, "policy": policy}, + "workers": [{"model": MODEL_ID, "desired_running": True}], + } + self.master_share_button = MasterButton(self) + self.model_toggle = ModelToggle(self) + + def findChildren(self, _kind): + return [self.model_toggle] + + class ModelToggle: + def __init__(self, window): + self.window = window + self.checked = True + self.clicks = 0 + + def accessibleName(self): + return f"Share compute with {MODEL_ID}" + + def isChecked(self): + return self.checked + + def isEnabled(self): + return True + + def click(self): + self.clicks += 1 + self.checked = False + self.window._snapshot["contribution"]["intent_enabled"] = False + self.window._snapshot["contribution"]["enabled"] = False + self.window._snapshot["workers"][0]["desired_running"] = False + self.window.master_share_button.label = "Start sharing" + + class MasterButton: + def __init__(self, window): + self.window = window + self.label = "Pause sharing" + self.clicks = [] + + def text(self): + return self.label + + def isEnabled(self): + return True + + def click(self): + if not self.window._page_buttons[2].isChecked(): + raise AssertionError("sharing action was invoked off the Sharing page") + self.clicks.append(self.label) + enabled = self.label == "Start sharing" + self.window._snapshot["contribution"]["intent_enabled"] = enabled + self.window._snapshot["contribution"]["enabled"] = enabled + self.window._snapshot["workers"][0]["desired_running"] = enabled + self.label = "Pause sharing" if enabled else "Start sharing" + + with TemporaryDirectory() as directory: + root = Path(directory) + now = [10.0] + plan = _write_plan(root / "windows-restart-plan.json", "restart", "windows") + window = Window(plan.policy) + application = SimpleNamespace(quit=MagicMock()) + automation = Gate13Playthrough( + plan, + root / "windows-restart-evidence.json", + clock=lambda: now[0], + start_observation_seconds=0.05, + ) + automation._application = application + automation._window = window + automation._qt = {"QCheckBox": object} + automation._state = "wait_policy" + + automation._tick() + self.assertEqual(automation._state, "wait_prestart_paused") + self.assertEqual(window.model_toggle.clicks, 1) + self.assertEqual(window.master_share_button.clicks, []) + + automation._tick() + self.assertEqual(window.master_share_button.clicks, ["Start sharing"]) + automation._tick() + now[0] += 0.1 + automation._tick() + self.assertEqual(window.master_share_button.clicks, ["Start sharing", "Pause sharing"]) + automation._tick() + + evidence = json.loads((root / "windows-restart-evidence.json").read_text(encoding="utf-8")) + self.assertEqual(evidence["result"], "passed") + self.assertTrue(evidence["ui"]["start_clicked"]) + self.assertTrue(evidence["ui"]["pause_clicked"]) + application.quit.assert_called_once() + + def test_bandwidth_suspension_and_async_worker_exit_do_not_reintroduce_manual_false_failure(self): + from tempfile import TemporaryDirectory + + class Button: + def __init__(self, navigation): + self.label = "Pause sharing" + self.enabled = True + self.clicks = 0 + self.navigation = navigation + + def text(self): + return self.label + + def isEnabled(self): + return self.enabled + + def click(self): + if not self.navigation.isChecked(): + raise AssertionError("sharing action was invoked off the Sharing page") + self.clicks += 1 + + class PageButton: + def __init__(self, label): + self.label = label + self.enabled = True + self.checked = label == "Home" + self.clicks = 0 + + def text(self): + return self.label + + def isEnabled(self): + return self.enabled + + def isChecked(self): + return self.checked + + def click(self): + self.clicks += 1 + self.checked = True + + with TemporaryDirectory() as directory: + root = Path(directory) + now = [10.0] + pages = [PageButton(label) for label in ("Home", "Models", "Sharing", "API access")] + button = Button(pages[2]) + application = SimpleNamespace(quit=MagicMock()) + automation = Gate13Playthrough( + _write_plan(root / "windows-restart-plan.json", "restart", "windows"), + root / "windows-restart-evidence.json", + clock=lambda: now[0], + start_observation_seconds=0.05, + restart_observation_seconds=0.05, + ) + automation._application = application + automation._window = SimpleNamespace( + _busy=0, + _page_buttons=pages, + master_share_button=button, + _snapshot={ + "contribution": {"intent_enabled": True, "enabled": False}, + "workers": [ + { + "model": MODEL_ID, + "desired_running": True, + "sharing_active": False, + "resource_admitted": False, + "resource_reason": "bandwidth usage exceeds contribution budget", + } + ], + }, + ) + automation._state = "wait_started_intent" + + automation._tick() + now[0] += 0.1 + automation._tick() + + self.assertEqual(button.clicks, 1) + self.assertEqual(pages[2].clicks, 1) + self.assertEqual(automation._state, "wait_paused_intent") + button.label = "Start sharing" + button.enabled = False + automation._window._snapshot = { + "contribution": {"intent_enabled": False, "enabled": False}, + # The manual trace still counted workers after Pause. Their + # asynchronous exit is deliberately not this UI gate's boundary. + "workers": [{"model": MODEL_ID, "desired_running": False, "sharing_active": True}], + } + automation._tick() + + evidence = json.loads((root / "windows-restart-evidence.json").read_text(encoding="utf-8")) + self.assertEqual(evidence["result"], "passed") + self.assertTrue(evidence["ui"]["sharing_intent_disabled_observed"]) + application.quit.assert_called_once() + + def test_real_window_replays_manual_platform_sequences(self): + os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + try: + from PySide6.QtWidgets import QApplication + except ModuleNotFoundError as exc: + self.skipTest(f"PySide6 is unavailable: {exc}") + + from tempfile import TemporaryDirectory + + from communityai_desktop.pyside_shell import run + + QApplication.instance() or QApplication([]) + with TemporaryDirectory() as directory: + root = Path(directory) + for platform in ("windows", "linux"): + with self.subTest(platform=platform), fake_node(all_workers_paused=True) as (url, token): + controller = DesktopController(NodeClient(url, token)) + initial = Gate13Playthrough( + _write_plan(root / f"{platform}-initial-plan.json", "initial", platform), + root / f"{platform}-initial-evidence.json", + inference_runner=_inference, + start_observation_seconds=0.05, + restart_observation_seconds=0.05, + ) + with patch("communityai_desktop.pyside_shell.login_startup_enabled", return_value=False): + self.assertEqual( + run(controller, single_instance=False, qualification_automation=initial), + 0, + ) + initial_evidence = json.loads( + (root / f"{platform}-initial-evidence.json").read_text(encoding="utf-8") + ) + self.assertEqual(initial_evidence["result"], "passed") + self.assertEqual(initial_evidence["platform"], platform) + self.assertEqual(initial_evidence["ui"]["policy_dialog_saved"], platform == "linux") + self.assertEqual(initial_evidence["ui"]["start_clicked"], platform == "linux") + + restart = Gate13Playthrough( + _write_plan(root / f"{platform}-restart-plan.json", "restart", platform), + root / f"{platform}-restart-evidence.json", + inference_runner=_inference, + start_observation_seconds=0.05, + restart_observation_seconds=0.05, + ) + with patch("communityai_desktop.pyside_shell.login_startup_enabled", return_value=False): + self.assertEqual( + run(controller, single_instance=False, qualification_automation=restart), + 0, + ) + restart_evidence = json.loads( + (root / f"{platform}-restart-evidence.json").read_text(encoding="utf-8") + ) + self.assertEqual(restart_evidence["result"], "passed") + self.assertTrue(restart_evidence["ui"]["pause_control_observed"]) + self.assertTrue(restart_evidence["ui"]["pause_clicked"]) + self.assertTrue(restart_evidence["ui"]["sharing_intent_disabled_observed"]) + self.assertEqual( + restart_evidence["ui"]["restart_resume_observed"], + platform == "linux", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/PACKAGED_ALPHA_OPERATIONS.md b/docs/PACKAGED_ALPHA_OPERATIONS.md index 3b3b84bf5..861d7bad6 100644 --- a/docs/PACKAGED_ALPHA_OPERATIONS.md +++ b/docs/PACKAGED_ALPHA_OPERATIONS.md @@ -4,9 +4,16 @@ This runbook defines the Gate 13 clean-host lifecycle for the unsigned Community public-alpha packages. It applies to Windows and Linux. It does not apply to macOS, does not test credits, and does not authorize cloud creation. -Passing the controller tests in this repository does **not** pass Gate 13. Gate 13 -requires one complete real packaged lifecycle on each supported platform against the -published signed bootstrap and a live product route. +Passing the controller tests in this repository does **not** constitute a fresh live +qualification. A replay requires the exact packaged desktop on each supported platform +against the published signed bootstrap and a live product route. + +Gate 13 and Gate 15 are now separate release gates. Gate 13 covers verified package +startup, real-window inference, sharing-policy editing, Start, full application restart, +automatic sharing resume, Pause, and post-restart inference. Manual replacement, +retain/delete uninstall choices, and retained-data reinstall are Gate 15. The older +16-phase contract later in this document remains a useful combined Gate 13/15 release +exercise; it is not the shortest Gate 13 replay. ## Release boundary @@ -41,6 +48,9 @@ Resolve these before touching a clean host: 7. One privacy-safe run ID for each host. 8. A copy of [gate13_packaged_lifecycle.py](../scripts/gate13_packaged_lifecycle.py). +9. A copy of + [gate13_automated_playthrough.py](../scripts/gate13_automated_playthrough.py) + when replaying the current Gate 13 boundary. The controller is a standard-library qualification tool. It may be copied separately to the host, but it does not install or import CommunityAI source. The product runtime @@ -48,7 +58,118 @@ must consist only of the unpacked release executables. A source checkout, editab install, repository PYTHONPATH, developer virtual environment, or invocation of python -m drift invalidates the run. -## Evidence contract +## Current automated Gate 13 replay + +The production desktop contains a hidden qualification mode that drives the real Qt +window. It does not call the controller in place of UI actions. The first process opens +the normal window, verifies the exact selected route, performs one localhost inference, +opens and saves **Edit sharing limits**, clicks **Start sharing**, and observes the +selected worker running. The process then exits normally so the desktop-owned node is +stopped. A second fresh desktop process proves sharing resumed after restart, clicks +**Pause sharing**, proves the worker stopped, and performs another localhost inference. + +The replay preserves the literal manual control order. It opens **Sharing** before using +page-scoped controls. After saving policy, it checks whether automatic placement already +enabled the selected model. If so, it clicks the exact checked **Share compute with +<model>** control to restore a paused baseline before exercising literal **Start +sharing** and **Pause sharing**. This normalization prevents policy-save reconciliation +from bypassing the manual Start step. If the first inference reports `Model unavailable`, +the replay polls the exact model in `/v1/models` every five seconds for at most 90 seconds +and retries the same `model:auto`, one-token request once. + +Each inference creates one in-memory temporary client key, retains only completion and +token counts, revokes the key, and proves the active-key baseline was restored. It requests +and requires exactly one generated token, matching the manual Gate 13 procedure. Session +timeouts are bounded to one hour each. The outer runner verifies the production archive +digest and byte size, runs the four packaged self-tests, executes both window sessions, +validates their strict privacy-safe evidence, and removes its exact run-scoped temporary +root. + +Prepare one absolute-path config beside the staged runner. `work_root` must not exist and +its leaf must be exactly `.gate13-playthrough-`: + +~~~json +{ + "schema_version": 1, + "run_id": "gate13-replay-a", + "platform": "windows", + "source_commit": "<40 lowercase hex>", + "package_archive": "", + "package_sha256": "sha256:<64 lowercase hex>", + "package_bytes": 1, + "desktop_executable": "", + "work_root": "/.gate13-playthrough-gate13-replay-a", + "model_id": "Qwen3.5 2B", + "manifest_digest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "total_blocks": 24, + "policy": { + "sharing_enabled": true, + "allowed_models": ["Qwen3.5 2B"], + "preferred_models": ["Qwen3.5 2B"], + "denied_models": [], + "max_disk_space": "32GB", + "max_vram": "20GB", + "max_bandwidth_mbps": 100.0, + "max_power_watts": null, + "pause_timeout": 120.0, + "schedule": { + "timezone": "UTC", + "windows": [{ + "days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"], + "start": "00:00", + "end": "23:59" + }] + } + }, + "session_timeout_seconds": 3600, + "inference_timeout_seconds": 600 +} +~~~ + +Run it as the ordinary qualification user with tracing disabled: + +This policy is the exact CPU-host policy proven by the manual Gate 13 playthrough: +the power field stays blank because the e2 hosts have no power telemetry, while the +storage, memory, bandwidth, pause, and explicit UTC schedule fields are exercised. + +~~~text +python gate13_automated_playthrough.py --config gate13-windows-run.json > gate13-windows-evidence.json +~~~ + +The replay is desktop automation, not a headless smoke test. On Windows, provision with +[gate13_windows_client_startup.ps1](../scripts/gate13_windows_client_startup.ps1), wait +for the ordinary `M` account to own a real console session, and let the privileged host +adapter register the bound task with `Interactive` logon and `Limited` run level. `S4U`, +service-session, and SSH-session launches are invalid because Qt may start without an +actual user desktop or access to that user's Credential Manager. + +On Linux, provision with +[gate13_linux_client_startup.sh](../scripts/gate13_linux_client_startup.sh). It installs +the package's complete XCB runtime closure, starts a TCP-disabled Xvfb display, and +prepares the ordinary `gate13` account. The host adapter runs the replay inside a private +`dbus-run-session`, starts GNOME Keyring's Secret Service, and passes only the fixed +display, home, and runtime-directory values into the bounded service. Do not substitute +`QT_QPA_PLATFORM=offscreen`: the qualification requires two real X11 windows and the +same native credential session across restart. + +Use `platform: linux` and the exact Linux executable/archive for Linux. The durable +Gate 13 host-job adapter accepts this Python entrypoint on both platforms, binds the +config and source commit, and validates the aggregate before collection. A cloud replay +still requires a fresh cost authorization, route acceptance, exact clean clients, and +provider cleanup; prior Gate 13 reservations must not be reused. + +[Paid-cloud run `gate13-20260901-a`](evidence/gate13-20260901-a-automated-qualification-and-cleanup.json) +proves this automation from production packages without manual UI recovery. Windows/Qwen +passed two real-window sessions in 260.828 and 66.328 seconds; Linux/Gemma passed in +229.270 and 44.956 seconds, including automatic sharing resume and the second inference. +Both formal client jobs passed as attempt ordinal 1. The route preflight now allows a +bounded 180 seconds for GPU service actions and keeps polling when a stale complete DHT +advertisement expires during restart. On a fresh Linux host, `LoadState=not-found` is +accepted with systemd's exact field set even though Ubuntu omits `ExecStart`; loaded units +still require the complete strict binding. All run resources were deleted after evidence +collection, L4 usage returned to zero, and the protected bootstrap remained running. + +## Combined 16-phase Gate 13/15 evidence contract Platform startup scripts perform product actions and write one local JSON phase result after each action. After final cleanup they place the ordered phase objects in one @@ -487,7 +608,7 @@ Always finish exact product cleanup. A failed action is not permission to leave worker, node, desktop process, persistent test data, credential, or phase temporary behind. -## Publication checklist +## Combined 16-phase publication checklist A Gate 13 evidence record is publishable only when: @@ -500,5 +621,8 @@ A Gate 13 evidence record is publishable only when: checksum. Archive the Windows and Linux records separately, then aggregate their bounded facts in -release readiness. Do not mark Gate 13 passed from controller unit tests, build-job -smokes, or only one supported platform. +release readiness. For a current-scope Gate 13 replay, the automated aggregate replaces +the combined 16-phase record only for the open/infer/share/restart/resume/pause boundary; +Gate 15 still requires separate replacement and uninstall evidence. Do not claim a fresh +live qualification from controller tests, build-job smokes, or only one supported +platform. diff --git a/docs/RELEASE_READINESS.md b/docs/RELEASE_READINESS.md index 0891abce9..a3c6103f5 100644 --- a/docs/RELEASE_READINESS.md +++ b/docs/RELEASE_READINESS.md @@ -1,6 +1,6 @@ # Public inference alpha release readiness -Last verified: 2026-08-31 +Last verified: 2026-09-02 This is the live source of truth for public-alpha implementation. Update it whenever a gate changes state. `docs/REVIVAL.md` defines the execution contract and long-term design; @@ -40,20 +40,36 @@ gate changes state. `docs/REVIVAL.md` defines the execution contract and long-te ## Critical path Work from top to bottom while prerequisites are satisfied. Gate V and Gates 5–6 have passed. -The current mandatory sequence is **Gates 13–16 → Gate 17**. The visible +The current mandatory sequence is **Gates 14–16 → Gate 17**. The visible vertical slice proved real Qwen3.5 2B inference through a public GCP L4 worker, and the strict four-profile Qwen and Gemma matrices now pass, and Gate 7 passed the generic five-Machine provider recovery mechanism with TinyLlama. Per-model repetition of the same provider recovery gate is not required. -As of 2026-08-31, Gate 13 is `BLOCKED`: cleanup proved the Gate 11 product route, -its run-scoped firewall rules, and every Gate 13 client and disk absent, so the owner -explicitly authorized a cleanup-backed reset for the next run. The new combined -authorization epoch starts at USD 100 with no reservation recorded. The selected native -GCP account still requires interactive reauthentication, and every paid create still -requires a fresh exact source-bound conservative ledger reservation plus fail-closed -preflight. No later mandatory gate is unblocked until the live packaged route and -completed Gate 13 lifecycle evidence exist. +As of 2026-09-02, Gate 13 is `PASSED`. [Run `gate13-20260831-i`](evidence/gate13-20260831-i-manual-qualification-and-cleanup.json) +replaced the opaque wrapper-first approach with a literal clean-host desktop playthrough. +The route first passed Qwen primary, automatic Gemma fallback, and Qwen restoration. +Windows and Linux then ran sequentially as ordinary users from exact verified production +archives: the app opened, public inference passed, sharing was configured and started, +the app was restarted, sharing resumed, Pause sharing worked, and post-restart inference +passed on Linux. The Windows playthrough exposed and fixed the actual product blocker: +legacy MAX_PATH on a manifest-artifact lock path under the normal per-user data root. +Source `f1dc3a0` passed the regression test, rebuilt-package self-tests, and default-root +Qwen inference. Every run instance, disk, and firewall is absent, global L4 usage is zero, +and the protected bootstrap remains running. Gate 14 is now `READY`; Gate 15 owns +publication of the source-fixed Windows archive plus reinstall/uninstall release work. + +The follow-up [automated paid-cloud replay `gate13-20260901-a`](evidence/gate13-20260901-a-automated-qualification-and-cleanup.json) +now proves that the manual sequence is repeatable without UI assistance. Exact production +packages from source `e904d36` passed archive verification and four packaged self-tests. +Windows/Qwen passed two real-window sessions in 260.828 and 66.328 seconds; Linux/Gemma +passed in 229.270 and 44.956 seconds, including restart resume and a second inference. +Both formal client jobs passed as attempt ordinal 1. The replay now preserves the manual +Sharing-page and per-model-toggle order, handles the transient `Model unavailable` case, +fences slow GPU restarts and stale DHT advertisements, and accepts the exact fresh Ubuntu +systemd inventory. All run instances, disks, and firewalls are absent, regional L4 usage +is zero, and the protected bootstrap remains running. A future Gate 13 replay still needs +a new source-bound package, authorization, route, clean clients, and cleanup evidence. Do not work on the post-alpha items in the deferred table while an alpha gate can progress. Missing Docker, snapshots, local GPU hardware, or local host capacity is not an external @@ -83,11 +99,11 @@ longer consume the new authorization; later billing should still be recorded for | 8 | Per-model duplicate separate-machine recovery | DEFERRED | Gates 5 and 6 already qualify Qwen and Gemma across the supported platform/device matrix; Gate 7 proves the model-independent provider recovery mechanism. Repeating the same Fly topology for each catalog model would test artifact transport rather than a new release property. | No public-alpha action. Model admission uses manifest/artifact and resource-envelope checks; product-level recovery is covered after automatic placement and catalog publication. | | 9 | Publish edge resource envelopes for selectable profiles | PASSED | [Run `gate9-20260830-e`](evidence/gate9-20260830-e-edge-resource-envelopes.json) publishes all four privacy-safe acquisition and schema-v3 steady-state records at exact runtime source `ba410f7`. Qwen selected 4,571,197,320 bytes and Gemma 10,278,818,149 bytes from empty caches on both Windows Server 2022 and Ubuntu 24.04; every artifact SHA-256 passed with zero resumptions. Qwen measured load/first-token/decode at 25.896 s/1.785 s/1.392 tok/s on Windows and 17.115 s/2.973 s/0.800 tok/s on Linux, with process-tree RSS peaks of 1,883,205,632 and 2,832,244,736 bytes. Gemma measured 64.286 s/1.386 s/1.949 tok/s on Windows and 38.808 s/2.025 s/0.868 tok/s on Linux, with peaks of 1,728,995,328 and 2,818,523,136 bytes. Every workload generated eight tokens without retaining prompts or outputs; Windows Job Objects and Linux process groups were empty, and route/DHT, accelerator, runtime-close, and provider cleanup all passed. All four temporary instances and auto-delete disks are absent; the protected bootstrap and separately authorized Gate 11 route remain running. The Windows Gemma cache-preserving in-place memory retry created no new resource and did not raise the USD 46 Gate 9 ceiling. | Proceed immediately to Gate 13 clean packaged install/inference on Windows and Linux using these envelopes and the live bounded product route. | | 10 | Implement automatic contributor model and block placement | PASSED | Signed bootstrap now installs one bounded `auto` worker. The local planner filters exact manifested candidates through owner policy and local resource ceilings, requires fresh authenticated replica coverage, targets the least-covered contiguous range with per-node jitter, reconciles exact-manifest launches through the existing artifact-verifying server and `WorkerSupervisor`, applies residency/cooldown/switch hysteresis, exposes placement reasons, and preserves an explicit operator pause across ineligibility or placement changes. A new or migrated worker must sign an expiring exact-manifest/range intent with fixed numeric resource claims and receive a remote DHT store acknowledgement (`exclude_self=True`) before entering the artifact path; invalid, rejected, or failed publication is fail-closed and cannot advance planner state, while a previously admitted placement is retained. Actual completed local generations feed exact-manifest demand, useful-throughput, and reliability through two bounded five-minute aggregate windows; no prompt, output, token ID, key, request ID, address, path, error, or per-request event is retained. Only a closed window with at least four completed routes may be signed by the separate router identity and published under the manifest-bound `demand-v1` DHT key with a 90-second lifetime and `exclude_self=True`. Consumers verify signature, exact schema/digest, lifetime, revocation, and replay ordering. The threshold-signed catalog may authorize 2–32 sorted RSA observer roots; missing or empty roots disable remote demand. Discovery discards unlisted identities before signature/replay work, excludes local and duplicate roots, isolates malformed records, requires two authorized roots, and medians at most 32 quantized observations. Observer keys are never generated or bundled: only a separately provisioned `route-demand.key` matching a signed root may publish, while ordinary nodes can consume without one. Any hot-edited root-list mismatch disables both publication and consumption until restart. Local utility is capped at 6 points and signed remote utility at 2, keeping the combined hint below the 10-point migration margin and 100-point replica step. Verified announcement and route-demand replay watermarks now survive restarts in one Windows-safe journal per raw manifest digest under the node data directory. Each strict journal is capped at 256 active identity scopes and 256 KiB, retains only public record kind, key ID, ordering tuple, record digest, and the bounded replay deadline, and is fsync-written through atomic replacement; malformed, duplicate, oversized, symlinked, non-regular, or unwritable state fails closed. The retained deadline prevents an older still-live record from returning after a short-lived newer record expires. The replay slice's 99-test focused protocol/discovery/planner/node-configuration matrix and 209-pass, 2-skip catalog/node/API superset pass. The Sybil slice's 122-test focused catalog/bootstrap/config/discovery matrix proves that 30 valid attacker keys plus one authorized root cannot reach threshold, two authorized roots aggregate without attacker weight, one high authorized vote cannot inflate a lower second vote, old catalogs remain signature-verifiable with remote demand disabled, and trust-epoch reload mismatches fail closed. A 190-pass, 1-skip catalog/protocol/planner/discovery/node/API superset also passes. Independent verification passed 146 focused tests and a 255-pass, 2-skip broader node/API superset, plus a native-Windows publication-boundary probe; formatting, import-order, import-smoke, and diff checks pass. The [explicit privacy review](AUTOMATIC_PLACEMENT_PRIVACY_V1.md) inventories collection, retention, public-key linkability, DHT/journal/API/log exposure, secure-deletion limits, and residual governance/host risks. Three executable privacy-contract tests fix the aggregate, intent, demand, replay, forbidden-field, and path-free warning schemas; the focused privacy/protocol/planner/discovery/node matrix passes 108 tests and the broader catalog/node/API matrix passes 258 tests with 2 skips. Independent privacy review passed 108 tests with 1 skip and a 225-pass, 2-skip broader subset; every caught observer-key exception and an unauthorized key produced no path, key ID, or exception detail, while prompt and identity-path schema injections failed closed. The [deterministic convergence and load acceptance](AUTOMATIC_PLACEMENT_ACCEPTANCE_V1.md) closes the remaining software gate: equal snapshots use node-specific 32-point model dispersion and range rendezvous ranks; a fixed 512-node cold cohort selects both models and every range below the 85% concentration boundary; two 4,096-node fresh-arrival cohorts remain below that boundary under maximum priority-aligned or standby demand; maximum demand causes zero incumbent migrations; one-replica loss migrates after residency without early reversal; rolling arrivals keep every model/block populated and repair an abrupt block loss. The alpha fails closed above 32 candidates or 512 blocks, permits one `auto` worker, clamps reconciliation to at least one second, and scans each candidate in one bounded pass. The focused planner/convergence/configuration matrix passes 78 tests and the broader catalog/protocol/discovery/node/API matrix passes 214 with 2 skips. A real Windows DHT round trip exposed and fixed a durable-replay multiprocessing regression: replay guards now omit/recreate their thread lock across serialization and reload persistent state; its 15-test protocol/network matrix passes. Independent verification reproduced the 78-test focus, passed an expanded 235-test matrix with 2 skips and the 15-test real-DHT probe, and exercised adversarial score, timing, 32-by-512 load, 1,000-case range-equivalence, and persistent replay-reload boundaries. This slice used no cloud resources and spent USD 0. | Gates 9–11 are passed. Gates 13–14 must now prove the packaged flow and real hardware ceilings using the published envelopes. | -| 11 | Operate initial public alpha routes | PASSED | [Product-node run `route-20260830-j`](evidence/gate11node-20260830-a-lifecycle.json) installed the generic CommunityAI wheel on a bounded G2/L4 VM, verified the signed catalog, downloaded both exact manifested models directly from Hugging Face into one persistent shared cache, and used the product node's automatic workers to expose complete Qwen 24/24 primary and Gemma 35/35 standby routes. No model-specific image, cache mirror, or operator-transferred model artifact was used. The privacy-safe acceptance passed one-token primary inference, deliberate primary pause, automatic Gemma selection in 58.073 seconds, standby inference, Qwen restoration in 32.042 seconds, and restored inference. Both workers were stable before the drill. After Gate 13 released the L4, the preserved route was restored without changing its model cache or source, its ephemeral endpoint was rebound, both product-node services became active, and a fresh acceptance reproved Qwen 24/24 primary inference, automatic Gemma 35/35 fallback/inference, Qwen restoration, and restored inference. The protected bootstrap remains running. A corrected 4,800-second provider DELETE backstop was set for `2026-08-31T05:28:16.516Z`, earlier than the original deadline. [Post-backstop cleanup evidence](evidence/gate11route-20260830-j-backstop-cleanup.json) and an independent recheck prove the route instance, named disk, and both exact run-scoped firewall rules absent, all Gate 13 clients/disks absent, zero remaining route availability, and the protected bootstrap still running. The same-host standby is a bounded alpha fallback, not independent infrastructure redundancy; independent redundancy remains post-alpha. | Gate 11 acceptance evidence remains complete, but no product route is live after the corrected DELETE backstop. The 2026-08-31 reset supplies a new USD 100 epoch, but any replacement route still requires refreshed native authentication, a fresh exact source-bound conservative reservation, and fail-closed preflight before provisioning. | -| 12 | Create, publish, and bundle the minimal signed alpha catalog/bootstrap | PASSED | [Run `gate12-20260829-a`](evidence/gate12-20260829-alpha-catalog-publication.json) published the deterministic [`communityai-public-alpha-v1` bundle](../public-alpha/catalog-v1/bundle.json) from source `26be579`. Its threshold-one Ed25519 root signs sequence 1 with the exact qualified Qwen primary and Gemma standby manifests, one pinned public HTTPS mirror, one public seed, a one-route best-effort policy, and no unprovisioned route-demand roots. The canonical bundle binds five members and retains `complete_release_qualification=false`. All three public objects returned HTTP 200 with exact sizes, and a fresh empty consumer fetched them remotely, verified the signature/digests, and created the two-model `auto` node configuration. The private signing key remained ignored and uncommitted. The focused publication suite passes 32 tests, the catalog/bootstrap/model/desktop superset passes 92, and the run spent USD 0. | Preserve the branch-scoped mirror until a newly signed catalog sequence and packaged bootstrap migrate it. The Gate 11 acceptance and Gate 9 envelopes exist, but no product route is currently live; Gate 13 awaits native reauthentication and fresh per-run reservations under the new epoch. Independent threshold holders and interchangeable mirror/seed governance are post-alpha. | -| 13 | Pass packaged clean-install inference on Windows and Linux | BLOCKED | [Prerequisite run `gate13-20260830-a-prerequisites`](evidence/gate13-20260830-a-prerequisites.json) established deterministic install archives, exact first-use bytes, strict provenance, and the canonical lifecycle contract. [Native-harness and production-package run `gate13-20260830-b`](evidence/gate13-20260830-b-native-harness-and-packages.json) now completes the native Windows Credential Manager/Job Object and Linux Secret Service/systemd-cgroup 16-phase adapters, exact worker and descendant cleanup proofs, 3,600-second acquisition bounds, and package/runtime/catalog cross-binding. Independent software review passed 134 focused tests plus a 113-pass broader matrix with 3 platform skips; the production-discovery correction passes 73 unittests, 4 pytest checks, self-test, formatting, and import checks. [Exact-source production run 33338872342](https://github.com/flujo-app/CommunityAI/actions/runs/33338872342) passed both jobs at source `1971f10` and published independently audited CUDA 12.4 archives: Windows `sha256:45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6` (2,695,065,068 bytes) and Linux `sha256:f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00` (3,360,717,934 bytes). Pushed source `6787272` adds the fixed stdin-only artifact downloader and exact platform configs; its 42-test adversarial suite and independent race/special-member/live-wrapper audit pass. No cloud resource was created for these prerequisites. Real completed clean-host lifecycle evidence remains absent. Provider cleanup and the temporary Gate 11 restoration are proved, but the corrected backstop has since removed that route. | [Run `gate13-20260830-c` revision 13](evidence/gate13-20260830-c-cost-authorization.json) is stopped clean. The latest Windows host passed exact package audit, clean install, four desktop self-tests, and the packaged-node self-test, then failed before model acquisition because child stderr diagnostics contaminated strict JSON captured on stdout. [Attempt, cleanup, correction, and route-restoration evidence](evidence/gate13-20260830-c-windows-attempt-and-route-restore.json) proves zero cache bytes, no retained credential or product process, all four exact client instances/disks absent, the bootstrap running, and the temporary restored Qwen/Gemma product route. [Post-backstop cleanup evidence](evidence/gate11route-20260830-j-backstop-cleanup.json) now proves that route, its named disk, and both exact run-scoped firewall rules absent while every Gate 13 target remains absent and the protected bootstrap remains running. Pushed source `4818da3` separates captured stdout from a dedicated NUL stderr sink and passes 15 native tests plus independent high-volume, handle-leak, descendant, timeout, and Job Object probes, but it has not completed a paid clean-host lifecycle. The cleanup-backed 2026-08-31 owner reset releases the USD 98 historical maxima and opens a new USD 100 epoch for the next run; it does not authorize any particular resource or reuse the stopped record. A read-only native-auth check on 2026-08-31 found an active account selection, but provider requests could not refresh its token without interactive reauthentication; no provider mutation or resource creation occurred. Complete both 16-phase fresh-host lifecycles only after refreshing native authentication and recording fresh exact source-bound conservative reservations for the replacement route and Gate 13 clients; do not provision, restart FLUJO, or mark Gate 13 passed before then. | -| 14 | Pass automatic-contribution and resource-control hardware checks | WAITING | [PR #11](https://github.com/flujo-app/CommunityAI/pull/11) and [PR #12](https://github.com/flujo-app/CommunityAI/pull/12) implemented the authenticated node-authoritative Sharing UI and atomic policy editing, but cross-model automatic placement and real packaged hardware evidence are absent. | After Gates 9–13, follow the [recovery runbook](RECOVERY_TEST_RUNBOOK.md) once for the clean-install product flow while validating model/block choice, exact selected-shard bytes, shared-cache affinity, download authorization, VRAM/storage/bandwidth/power limits, suspension, pause timing, cleanup, restart persistence, and unsupported telemetry on real packaged Windows/Linux hardware. | -| 15 | Complete minimal alpha release engineering | WAITING | The desktop builder now emits a stable sorted `SHA256SUMS` inventory of exact regular-file bytes and safe relative in-bundle file symlinks, source/build/catalog-bound `provenance.json`, and `release-metadata.json` with explicit unsigned public-alpha, no-publisher-signature, no-authenticated-update, Windows/Linux-only, no-credits, and incomplete-qualification claims. Structural verification binds each safe file symlink to its canonical in-bundle target, digest, and size while rejecting changed, missing, extra, absolute, external, broken, cyclic, directory-linked/junction, special, traversal, or case-colliding payloads plus unsupported or noncanonical metadata. Exact-source builds also reject dirty relevant inputs, and the expected-input fresh-process check rejects rewritten commit/tree, workflow, platform, Python, PyInstaller, or catalog evidence. Production desktop CI is configured to verify and bundle the Gate 12 inputs, bind the exact clean Git commit/tree and workflow, revalidate every expected input separately, and upload all evidence on Windows/Linux. The focused release-input/artifact suite passes 15 tests, including fresh-process CLI, dirty-source, and canonical-rewrite checks, and the broader catalog/bootstrap/model/desktop subset passes 134. Independent verification reproduced all 134, passed 58 desktop unittests with two environment skips, formatting/import-order/YAML/diff checks, an expected Gate 12/workflow fresh-process probe, and real Windows junction rejection; no cloud was used. [The first PR #22 production-desktop run](https://github.com/flujo-app/CommunityAI/actions/runs/33273518744) reached packaging on both hosts and exposed two exact cross-platform defects: PyInstaller's legitimate relative internal Qt file symlink on Ubuntu and CRLF-transformed signed Gate 12 JSON on Windows. The follow-up binds safe internal file symlinks without accepting external or directory links, forces `public-alpha/**` to LF at checkout, and includes `.gitattributes` in the clean-source boundary. [The second run](https://github.com/flujo-app/CommunityAI/actions/runs/33274432423) proved the Ubuntu package and the Windows signed-bundle/provenance path, then exposed a stale desktop contribution-status schema 2 contract when the packaged node emitted schema 3 automatic-placement evidence. Source `fcd1f41` now strictly validates schema 3 placement and rejects stale schema 2 plus missing, extra, secret-bearing, or inconsistent placement data; its 50-test node/client/lifecycle/build focus and all 59 desktop unittests passed with two environment skips. [The final run](https://github.com/flujo-app/CommunityAI/actions/runs/33275216332) bound exact source `fcd1f417d1435557addb2d6cded9dac0827c7d8c` and completed both Windows and Ubuntu package jobs, including bundle build/smoke, independent checksum/provenance verification, the Windows packaged-node/native-credential/public-seed smoke, and artifact uploads; every PR style, test, and package check is green. Source `36d85d2` makes generic release-artifact fixtures select the supported Linux archive explicitly instead of inheriting the CI host platform; the 21-test local artifact suite and [PR #22 test run 33372581439](https://github.com/flujo-app/CommunityAI/actions/runs/33372581439) pass, without expanding the supported platform matrix. Clean-install lifecycle evidence remains absent. | Retain the verified Windows/Linux artifacts as engineering evidence, then test clean install, manual upgrade/reinstall, uninstall, retained-data choice for the persistent verified model cache, and recovery instructions on both platforms against a newly authorized live product-node route and the published Gate 9 envelopes. Do not mark passed from metadata/unit tests alone. Publisher signing and automatic authenticated update/rollback are post-alpha. | +| 11 | Operate initial public alpha routes | PASSED | [Product-node run `route-20260830-j`](evidence/gate11node-20260830-a-lifecycle.json) installed the generic CommunityAI wheel on a bounded G2/L4 VM, verified the signed catalog, downloaded both exact manifested models directly from Hugging Face into one persistent shared cache, and used the product node's automatic workers to expose complete Qwen 24/24 primary and Gemma 35/35 standby routes. No model-specific image, cache mirror, or operator-transferred model artifact was used. The privacy-safe acceptance passed one-token primary inference, deliberate primary pause, automatic Gemma selection in 58.073 seconds, standby inference, Qwen restoration in 32.042 seconds, and restored inference. Both workers were stable before the drill. After Gate 13 released the L4, the preserved route was restored without changing its model cache or source, its ephemeral endpoint was rebound, both product-node services became active, and a fresh acceptance reproved Qwen 24/24 primary inference, automatic Gemma 35/35 fallback/inference, Qwen restoration, and restored inference. The protected bootstrap remains running. A corrected 4,800-second provider DELETE backstop was set for `2026-08-31T05:28:16.516Z`, earlier than the original deadline. [Post-backstop cleanup evidence](evidence/gate11route-20260830-j-backstop-cleanup.json) and an independent recheck prove the route instance, named disk, and both exact run-scoped firewall rules absent, all Gate 13 clients/disks absent, zero remaining route availability, and the protected bootstrap still running. The same-host standby is a bounded alpha fallback, not independent infrastructure redundancy; independent redundancy remains post-alpha. | Gate 11 acceptance evidence remains complete, but no product route is live after the corrected DELETE backstop. [Gate 13 run `gate13-20260831-a`](evidence/gate13-20260831-a-cost-authorization.json) now binds refreshed native authentication, fail-closed preflight, and a fresh USD 52 reservation for the replacement route and packaged clients. | +| 12 | Create, publish, and bundle the minimal signed alpha catalog/bootstrap | PASSED | [Run `gate12-20260829-a`](evidence/gate12-20260829-alpha-catalog-publication.json) published the deterministic [`communityai-public-alpha-v1` bundle](../public-alpha/catalog-v1/bundle.json) from source `26be579`. Its threshold-one Ed25519 root signs sequence 1 with the exact qualified Qwen primary and Gemma standby manifests, one pinned public HTTPS mirror, one public seed, a one-route best-effort policy, and no unprovisioned route-demand roots. The canonical bundle binds five members and retains `complete_release_qualification=false`. All three public objects returned HTTP 200 with exact sizes, and a fresh empty consumer fetched them remotely, verified the signature/digests, and created the two-model `auto` node configuration. The private signing key remained ignored and uncommitted. The focused publication suite passes 32 tests, the catalog/bootstrap/model/desktop superset passes 92, and the run spent USD 0. | Preserve the branch-scoped mirror until a newly signed catalog sequence and packaged bootstrap migrate it. The Gate 11 acceptance and Gate 9 envelopes exist; [Gate 13 run `gate13-20260831-a`](evidence/gate13-20260831-a-cost-authorization.json) now authorizes the bounded replacement route and fresh packaged clients under the new epoch. Independent threshold holders and interchangeable mirror/seed governance are post-alpha. | +| 13 | Pass packaged clean-install inference on Windows and Linux | PASSED | [Manual run `gate13-20260831-i`](evidence/gate13-20260831-i-manual-qualification-and-cleanup.json) found and fixed the Windows legacy-MAX_PATH blocker while proving the literal clean-host flow. [Automated paid-cloud run `gate13-20260901-a`](evidence/gate13-20260901-a-automated-qualification-and-cleanup.json) translates that flow into two real-window sessions per platform from exact production packages. Formal attempt 1 passed unattended on Windows/Qwen and Linux/Gemma: package verification, four self-tests, inference, policy save, per-model normalization, literal Start, observation, full process restart, Pause, final paused intent, and Linux restart-resume plus second inference. The route passed exact 24/24 and 35/35 stable fences. All run instances, disks, and firewalls are absent, L4 usage is zero, and the protected bootstrap is running. | Proceed to Gate 14 without replaying Gate 13 discovery. Gate 15 owns reinstall/uninstall/retained-data release engineering. | +| 14 | Pass automatic-contribution and resource-control hardware checks | IN PROGRESS | [PR #11](https://github.com/flujo-app/CommunityAI/pull/11) and [PR #12](https://github.com/flujo-app/CommunityAI/pull/12) implemented authenticated node-authoritative sharing controls. [Automated Gate 13 run `gate13-20260901-a`](evidence/gate13-20260901-a-automated-qualification-and-cleanup.json) proves the real packaged Sharing UI starts, survives restart, resumes on Linux, and pauses on both supported platforms without manual UI recovery. The new strict Gate 14 verifier pins that lifecycle evidence plus the exact Windows/Qwen and Linux/Gemma Gate 9 envelopes, platform/OS pair, production package, L4 device profile, all five configured resource classes, suspension/resume, automatic block placement, recovery, unsupported CPU power telemetry, privacy, and exact GCP cleanup. Its durable controller recomputes the reset current-epoch spend ledger under the USD 100 ceiling, rejects expired/foreign/overlapping resources, serializes fresh Windows then Linux hosts, binds reported evidence bytes before collection, rejects rolled-back completed jobs, hidden active reservations, orphan planned disks before every fresh start, and resources that return during teardown, and requires cleanup to match the exact authorization file, controller source, plan, project, zone, resources, and successful terminal state while excluding the protected bootstrap. Exact source [`c0f2342e15aa7e12ca7c2980deca64d613204143`](https://github.com/flujo-app/CommunityAI/commit/c0f2342e15aa7e12ca7c2980deca64d613204143) passed independent adversarial review, the 49-test focused suite, formatting, import-order, compilation, and diff checks; [CodeQL](https://github.com/flujo-app/CommunityAI/actions/runs/33659218953), [style](https://github.com/flujo-app/CommunityAI/actions/runs/33659223681), and [Linux/Windows tests](https://github.com/flujo-app/CommunityAI/actions/runs/33659223645) are green. Its [production desktop run](https://github.com/flujo-app/CommunityAI/actions/runs/33659223622) passed exact-source build, smoke, independent checksum/provenance verification, packaged-node/native-credential/public-seed exercise on Windows, and all four uploads. The retained artifact digests are Windows install `sha256:44ab9faa5dae4537bae60e005ac790ce2b4f3636a2f5ad5327de713302a37b2c` (2,695,084,318 bytes), Windows audit `sha256:55e924e4a36f6deeb0e37bd64979f20debc36227503616f71c1f40622241ccc7` (468,605 bytes), Linux install `sha256:1494cb0bb4c37de1c8825c20d5400e76b4bbc62eefbd4010a161427a85f164c8` (3,360,751,567 bytes), and Linux audit `sha256:5ac2ce726235ef6c1ae3f1dea6faf89c9e42d61a2d0d7cd0f905ef5fe92deb91` (514,731 bytes), expiring 2026-09-09. A 2026-09-02 native read-only GCP preflight stopped before inventory or quota inspection because the configured account requires interactive reauthentication. No reservation or provider resource was created, and the current-epoch USD 44 remainder is intact. No Gate 14 hardware pass is claimed by this evidence. | Implement the thin source-bound Windows/Linux host probes and GCP action executor, including calibrated physical power and bandwidth triggers. Then refresh interactive GCP authentication, repeat exact inventory/quota/pricing/cleanup preflight, reserve at most USD 44, and run the remaining Gate 14 checks sequentially on fresh Windows/Linux L4 hosts while reusing the cited Gate 13 lifecycle evidence. | +| 15 | Complete minimal alpha release engineering | WAITING | The desktop builder now emits a stable sorted `SHA256SUMS` inventory of exact regular-file bytes and safe relative in-bundle file symlinks, source/build/catalog-bound `provenance.json`, and `release-metadata.json` with explicit unsigned public-alpha, no-publisher-signature, no-authenticated-update, Windows/Linux-only, no-credits, and incomplete-qualification claims. Structural verification binds each safe file symlink to its canonical in-bundle target, digest, and size while rejecting changed, missing, extra, absolute, external, broken, cyclic, directory-linked/junction, special, traversal, or case-colliding payloads plus unsupported or noncanonical metadata. Exact-source builds also reject dirty relevant inputs, and the expected-input fresh-process check rejects rewritten commit/tree, workflow, platform, Python, PyInstaller, or catalog evidence. Production desktop CI is configured to verify and bundle the Gate 12 inputs, bind the exact clean Git commit/tree and workflow, revalidate every expected input separately, and upload all evidence on Windows/Linux. The focused release-input/artifact suite passes 15 tests, including fresh-process CLI, dirty-source, and canonical-rewrite checks, and the broader catalog/bootstrap/model/desktop subset passes 134. Independent verification reproduced all 134, passed 58 desktop unittests with two environment skips, formatting/import-order/YAML/diff checks, an expected Gate 12/workflow fresh-process probe, and real Windows junction rejection; no cloud was used. [The first PR #22 production-desktop run](https://github.com/flujo-app/CommunityAI/actions/runs/33273518744) reached packaging on both hosts and exposed two exact cross-platform defects: PyInstaller's legitimate relative internal Qt file symlink on Ubuntu and CRLF-transformed signed Gate 12 JSON on Windows. The follow-up binds safe internal file symlinks without accepting external or directory links, forces `public-alpha/**` to LF at checkout, and includes `.gitattributes` in the clean-source boundary. [The second run](https://github.com/flujo-app/CommunityAI/actions/runs/33274432423) proved the Ubuntu package and the Windows signed-bundle/provenance path, then exposed a stale desktop contribution-status schema 2 contract when the packaged node emitted schema 3 automatic-placement evidence. Source `fcd1f41` now strictly validates schema 3 placement and rejects stale schema 2 plus missing, extra, secret-bearing, or inconsistent placement data; its 50-test node/client/lifecycle/build focus and all 59 desktop unittests passed with two environment skips. [The final run](https://github.com/flujo-app/CommunityAI/actions/runs/33275216332) bound exact source `fcd1f417d1435557addb2d6cded9dac0827c7d8c` and completed both Windows and Ubuntu package jobs, including bundle build/smoke, independent checksum/provenance verification, the Windows packaged-node/native-credential/public-seed smoke, and artifact uploads; every PR style, test, and package check is green. Source `36d85d2` makes generic release-artifact fixtures select the supported Linux archive explicitly instead of inheriting the CI host platform; the 21-test local artifact suite and [PR #22 test run 33372581439](https://github.com/flujo-app/CommunityAI/actions/runs/33372581439) pass, without expanding the supported platform matrix. Gate 13 manual clean-install evidence now exists; Gate 15 still lacks its upgrade/reinstall/uninstall and retained-data release evidence. | Retain the verified Windows/Linux artifacts as engineering evidence, then test clean install, manual upgrade/reinstall, uninstall, retained-data choice for the persistent verified model cache, and recovery instructions on both platforms against a newly authorized live product-node route and the published Gate 9 envelopes. Do not mark passed from metadata/unit tests alone. Publisher signing and automatic authenticated update/rollback are post-alpha. | | 16 | Complete the bounded public-alpha safety canary | WAITING | [PR #13](https://github.com/flujo-app/CommunityAI/pull/13) and [PR #14](https://github.com/flujo-app/CommunityAI/pull/14) implemented bounded admission, privacy-safe aggregate health, training-off defaults, rollback procedures, and bounded routine rejection logs; no public canary has run. | After Gates 11–15, run a small monitored canary proving finite admission/timeouts, malformed-peer rejection, health reconstruction, privacy disclosure, route/catalog disable, and clean rollback. Exhaustive hostile-load, Sybil/collusion, partition, and long-soak campaigns are post-alpha. | | 17 | Publish and observe the public alpha | TODO | Owner has authorized a public inference alpha, but preceding mandatory alpha gates are open. | After Gate V and Gates 1–16 pass, publish with explicit best-effort availability, unsigned-package, support, and prompt-privacy limitations; preserve the disable path and monitor real route/worker failures. | @@ -108,19 +124,35 @@ longer consume the new authorization; later billing should still be recorded for ## Cloud authorization and spend ledger Authorization applies only to CommunityAI qualification and public-alpha infrastructure. -The ceiling is USD 100 combined across new temporary GCP and Fly resources in the current -owner-authorized accounting epoch. The existing -GCP bootstrap's ordinary baseline cost is tracked separately; never delete it as test cleanup. +On 2026-09-01 the owner explicitly reset the cloud accounting epoch after reporting that the +prior real-world cloud charge was approximately USD 10 rather than the conservative reserved +maximums. Historical rows and their then-current `CLEANED-COMMITTED` labels remain unchanged +for auditability but consume USD 0 in the reset epoch. The reset epoch has a USD 100 accounting +ceiling; only the exact USD 56 `gate13-20260901-a` reservation is authorized, leaving USD 44 +unreserved and unauthorized for any other run. USD 56 remains a maximum-lifetime safety bound, +not a bill forecast. The existing GCP bootstrap's ordinary baseline cost is tracked separately; +never delete it as test cleanup. Before every paid run, add an entry with a conservative maximum. After cleanup, replace the estimate with observed cost when available. If provider billing is delayed, retain the maximum estimate until actual cost is known unless the owner explicitly resets the budget -after complete cleanup. On reset, keep historical rows, mark them `CLEANED-RELEASED`, and -continue recording later observed charges for information; released rows do not consume the -new epoch. +after complete cleanup. On reset, keep historical rows and continue recording later observed +charges for information. `CLEANED-COMMITTED` means resources were absence-proved while the +conservative maximum still consumed the accounting epoch then in force; a later explicit reset +starts a new epoch without rewriting that historical state. | Run | Provider | Purpose | Maximum estimate | Observed cost | Cleanup proof | State | | --- | --- | --- | ---: | ---: | --- | --- | +| gate13-20260901-a | GCP | Automated Gate 13 real-window replay, finalized against production packages from `e904d36416a4f186c0bec05ff20210df9ca19848`: one bounded L4 route, then sequential ordinary-user Windows/Qwen and Linux/Gemma clients [original plan `sha256:6687b9ba098b3f6676f48f4bf03ebb92bdc6a1278bf5bc1c227819b3a3e7cbb0`] | USD 56.00 | — | [Passed automation and cleanup](evidence/gate13-20260901-a-automated-qualification-and-cleanup.json) records both formal attempt-1 passes, exact evidence digests, all instances/disks/firewalls absent, L4 usage zero, and the protected bootstrap running. Provider billing was not yet available; the owner reports comparable real-world use at approximately USD 10. | CLEANED-COMMITTED | +| gate13-20260831-i | GCP | Final Gate 13 manual clean-host playthrough: Gate 11 route acceptance first, then sequential ordinary-user Windows/Qwen and Linux/Gemma desktop qualification with literal UI controls and post-restart inference [plan `sha256:8525c3099f273c099aba26de57c1f610a0c74cac65ed2640589d51e874bd0c44`] | USD 56.00 | — | [Passed qualification and cleanup](evidence/gate13-20260831-i-manual-qualification-and-cleanup.json) proves both exact archives, packaged self-tests, real desktop start/share/restart/pause flows, Qwen and Gemma inference, the Windows long-path product fix, all exact resources absent, L4 usage zero, and the protected bootstrap running. | CLEANED-COMMITTED | +| gate13-20260831-h | GCP | Final corrected Gate 13 route-first lifecycle with both four-file release-audit bundles pinned and staged, the bounded Windows user-runtime environment, exact archive preflight, and sequential ordinary-user Windows/Qwen then Linux/Gemma clients [plan `sha256:f243254cc5fb65f44d0c9e707be36feb3284fd6e15b15620882843798fb456b1`] | USD 56.00 | — | [Failed attempt and cleanup](evidence/gate13-20260831-h-failed-attempt-and-cleanup.json) records passed route acceptance and exact Windows archive verification, one Windows failure at `signed_bootstrap/product_readiness`, no Linux create, exact instance/disk/firewall absence, L4 usage zero, and protected-bootstrap health. | CLEANED-COMMITTED | +| gate13-20260831-g | GCP | Corrected Gate 13 route-first lifecycle with a bounded standard Windows user-runtime environment, one durable foreground host-adapter execution as each ordinary OS user, exact archive preflight, and sequential Windows/Qwen then Linux/Gemma clients [plan `sha256:f27f36158f2ad16019578555023cc854cb1e6e3b10ebae8cd3ed24d757b8e032`] | USD 56.00 | — | [Failed attempt and cleanup](evidence/gate13-20260831-g-failed-attempt-and-cleanup.json) records passed route acceptance, the exact Windows archive, a two-second `package_verification` failure caused by four omitted existing audit inputs, no Linux create, and exact instance/disk/firewall cleanup with L4 usage zero. | CLEANED-COMMITTED | +| gate13-20260831-f | GCP | Fresh Gate 13 route-first lifecycle using one durable foreground host-adapter execution over IAP SSH as each ordinary OS user, exact archive preflight, and sequential Windows/Qwen then Linux/Gemma clients [plan `sha256:c9a2aafc84940df901a7db1755af2e684f845b78dcdfac04332cfed36388ba25`] | USD 56.00 | — | [Failed attempt and cleanup](evidence/gate13-20260831-f-failed-attempt-and-cleanup.json) records passed route acceptance, exact Windows archive and staged-input verification, the same bounded `signed_bootstrap` failure under a direct ordinary-user launch as under S4U, no Linux create, and exact instance/disk/firewall cleanup with L4 usage zero. | CLEANED-COMMITTED | +| gate13-20260831-e | GCP | Fresh Gate 13 route-first lifecycle with pinned reusable route setup, corrected S4U/SID Windows host job, explicit archive download-and-hash prerequisite, and sequential Windows/Qwen then Linux/Gemma clients [plan `sha256:9ca0fa516017c4a3709a467752f779bcb3bbc0a7c790f9bc61de56d385804c62`] | USD 56.00 | — | [Failed attempt and cleanup](evidence/gate13-20260831-e-failed-attempt-and-cleanup.json) records passed route acceptance, the exact Windows archive preflight, ordinary-user SSH repair, a durable S4U/Limited lifecycle failure with opaque phase output, no Linux create, and exact instance/disk/firewall cleanup with L4 usage zero. | CLEANED-COMMITTED | +| gate13-20260831-d | GCP | Fresh Gate 13 route-first lifecycle using the durable controller and host jobs, one bounded route and sequential clients [plan `sha256:d32050a51b8f696aa224fc7e748c9113e174e3c3069c1f8b2bc769b0c5ecea18`] | USD 56.00 | — | [Failed attempt and cleanup](evidence/gate13-20260831-d-failed-attempt-and-cleanup.json) records passed route acceptance, the corrected headless S4U supervisor, one consumed Windows attempt that failed because its archive had not been downloaded, no Linux create, and exact instance/disk/firewall cleanup with L4 usage zero. | CLEANED-COMMITTED | +| gate13-20260831-c | GCP | Gate 13 durable route-first lifecycle with the same bounded 16-hour route and sequential 6-hour clients, new exact resources, and corrected explicit IAP target-tag arguments [plan `sha256:07b6cd399ef7a9733602dfc19a741feddec8d15e5f4b5bac7347a192675f6d9c`] | USD 56.00 | — | [Terminal-state and cleanup proof](evidence/gate13-20260831-c-terminal-state-and-cleanup.json) records a local terminal absence state without a durable provider execution record, retires the run ID without reset or reuse, proves every exact target absent, global GPU usage zero, and the protected bootstrap running. | CLEANED-COMMITTED | +| gate13-20260831-b | GCP | Gate 13 durable route-first lifecycle: one 16-hour G2/L4 product route, then sequential fresh 6-hour Windows/Qwen and Linux/Gemma CPU clients [plan `sha256:3f3f921ded6eed1729aff175f5c91b4effe1966a31c82bdbe41ed69075442d64`] | USD 56.00 | — | [Failed start and cleanup](evidence/gate13-20260831-b-failed-start-and-cleanup.json) records passed preflight and persisted intent, one transient DHT firewall, IAP-tag argument rejection before VM creation, exact firewall cleanup, all run resources absent, and protected-bootstrap health. | CLEANED-COMMITTED | +| gate13-20260831-a | GCP | Gate 13 replacement product-node route plus fresh CPU Windows/Linux packaged lifecycles at route source `f64a388a47b098ac7f69d2affc59816376b43bb1` and exact package source `1971f106cc5bf90724d938c986a719ce2744f3e7` [plan sha256:313f5d34eefd64c71e265bdb7044d8ef5f56550360a7e9a7104265434292fd69] | USD 52.00 | — | [Failed attempt and cleanup proof](evidence/gate13-20260831-a-failed-attempt-and-cleanup.json) records verified archive downloads but no completed lifecycle, the non-durable orchestration failure, consumed-client semantics, and exact absence of the route, both clients, all three disks, and both firewalls while the protected bootstrap remains running. The USD 52 maximum remains committed; after the owner raised the epoch ceiling to USD 500, USD 448 remains before a new reservation. | CLEANED-COMMITTED | | gate13-20260830-c | GCP | Gate 13 sequential clean packaged Qwen Windows and Gemma Linux lifecycles at exact package source `1971f106cc5bf90724d938c986a719ce2744f3e7`, temporarily suspending and later restoring the Gate 11 route while reusing its sole global L4 allocation on uniquely named fresh Windows and Linux clients [plan sha256:427bc1ed8a6645ad0650d91aaba7aa753d398fa84f56d57b50aca04c4e0cc955] | USD 26.00 | — | [Cost authorization](evidence/gate13-20260830-c-cost-authorization.json) binds the passed production archives/audits, pushed download-helper/config identities, exact Actions wrapper/inner archives, exact Qwen/Gemma manifests, no service accounts/scopes, direct model transfer, native credential stores, whole-tree containment, all 16 phases, exact cleanup targets, and zero Fly/image/mirror/credits/macOS work. Revision 13 records the final Windows pre-acquisition failure, pushed correction `4818da3`, complete native cleanup, all four exact client instance/disk absences, and successful Gate 11 route restoration. [Privacy-safe final state](evidence/gate13-20260830-c-windows-attempt-and-route-restore.json) proves the package audit and install boundary, zero model-cache bytes, no retained credential/process/path/endpoint/provider output, protected-bootstrap health, active Qwen/Gemma route services, and fresh primary/fallback/restoration inference. The two required 16-phase lifecycles remain incomplete. Complete cleanup permits the explicit owner reset on 2026-08-31; its USD 26 maximum is historical and delayed billing remains informational. This record authorizes no later provisioning. | CLEANED-RELEASED | | gate9-20260830-e | GCP | Gate 9 concurrent Qwen/Gemma Windows/Linux acquisition records and schema-v3 envelopes at pushed source `ba410f74f1cf625f1e1c34734b53e4514fa7c5ec`, reusing the separately authorized product route and using bounded isolated clients [plan sha256:04ba77ee68f4a895ae080a4ddcbf6805b502da6a95a4146734acbddff92de307] | USD 46.00 | — | [Passed envelopes and cleanup](evidence/gate9-20260830-e-edge-resource-envelopes.json) publish all four exact acquisition/envelope records and prove complete client cleanup; [cost authorization](evidence/gate9-20260830-e-cost-authorization.json) binds the exact wheel or exact-commit source archive, signed catalog/bootstrap, Qwen/Gemma manifests, owner-authorized parallel platform/model execution, 60-minute model windows, 90-minute client deletion backstops, exact cleanup targets, protected resources, and zero Fly/image/mirror operations. Native provider authentication was refreshed before the USD 18 Windows-client expansion and again before the zero-ceiling-increase Gemma memory retry; the exact plan permits one cache-preserving in-place resize to `e2-standard-8`. Complete cleanup permits the explicit owner reset on 2026-08-31; its USD 46 maximum is historical and delayed billing remains informational. | CLEANED-RELEASED | | route-20260830-j | GCP | Gate 11 signed-catalog product node route [workload gcp-product-node-route] [source e1d715fd47c852fa12ca50c76e8f4c6a0831fd78] [final runtime source 4cef141746705c3ee8bc8e017693855e0bc4871e] [plan sha256:1a0927e9d83a9a409ac2ea0232c4fceb14821d3f2c5eb87def88b8e7cdcb07d8] | USD 26.00 | — | [Passed live lifecycle](evidence/gate11node-20260830-a-lifecycle.json): generic runtime, signed catalog, direct Hugging Face artifacts, shared persistent cache, complete primary/standby routes, primary/fallback/restoration inference, stable workers, no model image, and protected-bootstrap health. [Gate 13 restoration evidence](evidence/gate13-20260830-c-windows-attempt-and-route-restore.json) proves the route was restored, both product services became active, fresh Qwen/Gemma primary/fallback/restoration inference passed, and a corrected 4,800-second DELETE backstop ended no later than the original deadline. [Post-backstop cleanup](evidence/gate11route-20260830-j-backstop-cleanup.json) proves the route instance, named disk, and both exact run-scoped firewall rules absent, all Gate 13 clients/disks absent, zero GPU use, and the protected bootstrap running; acceptance evidence is preserved but no product route is live. Complete cleanup permits the explicit owner reset on 2026-08-31; its USD 26 maximum is historical and delayed billing remains informational. | CLEANED-RELEASED | @@ -198,7 +230,11 @@ After the Gate 9 clients, Gate 11 product route, and Gate 13 clients were all cl the owner explicitly authorized another cleanup-backed reset for the next run on 2026-08-31. Their USD 98 conservative maxima remain historical evidence but no longer consume the new epoch; delayed observed charges remain informational. The next run starts with a new combined -authorization of **USD 100**. Every paid create still requires fresh native authentication, +authorization of **USD 100**. Later on 2026-08-31, the owner raised that current combined +epoch to **USD 500** without releasing the already committed USD 52 maximum. The dated +[authorization record](evidence/owner-budget-authorization-20260831.json) therefore leaves +USD 448 before a new reservation. The cleaned-committed USD 56 run-B maximum plus the fresh USD 56 run-C reservation now leave +USD 336. Every paid create still requires fresh native authentication, an exact source-bound cost authorization, a conservative ledger reservation, and the existing fail-closed preflight and cleanup controls. diff --git a/docs/REVIVAL.md b/docs/REVIVAL.md index a0ccf347d..62be0a65a 100644 --- a/docs/REVIVAL.md +++ b/docs/REVIVAL.md @@ -60,10 +60,12 @@ agent: directly, then attempt roughly 70B if that passes. This is permission to test those sizes, not permission to claim that an exact larger checkpoint works before its own model-specific evidence passes. -- New temporary GCP and Fly test resources share one combined **USD 100 maximum**. - Track conservative estimates and observed cost in - [`RELEASE_READINESS.md`](RELEASE_READINESS.md). Do not start a run that could exceed - the remaining balance. +- New temporary GCP and Fly test resources share one live owner-authorized combined + ceiling. The baseline is USD 100; on 2026-08-31 the owner raised the current accounting + epoch to **USD 500 maximum**. The already committed USD 52 maximum remains charged to + that epoch, leaving USD 448 before a new reservation. Track conservative estimates and + observed cost in [`RELEASE_READINESS.md`](RELEASE_READINESS.md). Do not start a run that + could exceed the remaining balance. - Use the existing `gcloud`, `flyctl`, and `gh` logins. Do not require the owner to copy provider tokens into environment variables when native CLI authentication works. - On Windows, every registry token, remote credential, and Linux script must follow the @@ -200,10 +202,34 @@ On every implementation run: ends, the release is complete, or that narrow definition applies to every permitted task on the current critical path. +### Durable paid-run contract + +A multi-hour paid qualification must not depend on an operator terminal, SSH/IAP session, +or untracked repair script remaining alive. Before its first create, it must have one +source-bound, persisted, idempotent controller with `start`, `status`, `collect`, and +`cleanup` operations. Every operation begins by inventorying the exact authorized +instances, disks, firewalls, ownership metadata, and absolute deadlines. Matching resources +are reattached; foreign or ambiguous exact-name resources fail closed; missing resources are +never recreated merely because local state was lost. + +Long-running work runs as one named host-local durable service or task and writes only a +bounded sanitized status plus a digest-bound terminal record. Repeating `start` observes the +existing job; it does not launch a second lifecycle. Once a packaged product lifecycle or a +diagnostic product launch begins, any non-pass consumes that client for acceptance. Removing +its files or credentials does not make it fresh again, and phase-level lifecycle resumption +is prohibited. + +For Gate 13, accept the complete product route before creating a client. Run the higher-risk +Windows/Qwen lifecycle first; collect its canonical 16-phase record and delete that client +before creating Linux/Gemma. This is an operational cost/risk sequence, not a relaxation of +the two-platform acceptance contract. Any route failure, ambiguous host job, expired runway, +or client failure goes directly to exact cleanup. A gate passes only after both complete +fresh-host records and final provider absence proof exist. + ### Cloud safety rules - Before provisioning, record a conservative maximum estimate in the spend ledger and - confirm it fits under the combined USD 100 ceiling. + confirm it fits under the live combined ceiling recorded in the readiness tracker. - An explicit owner budget reset starts a new USD 100 accounting epoch only after every prior run is cleanup-proved. Preserve those historical rows as `CLEANED-RELEASED` rather than pretending their actual cost was zero; their maxima no longer consume the new epoch, @@ -225,7 +251,7 @@ Do not block on these while another roadmap item can proceed. Ask the owner only input is on the critical path: - a provider login expires and native CLI reauthentication is required; -- the next bounded cloud run does not fit under the remaining USD 100 ceiling; +- the next bounded cloud run does not fit under the remaining live owner-authorized ceiling; - platform code-signing/notarization credentials or a publisher identity are required; - production catalog signing needs independent human key holders; - an independent seed or mirror operator must accept operational responsibility; or diff --git a/docs/evidence/gate13-20260831-a-cost-authorization.json b/docs/evidence/gate13-20260831-a-cost-authorization.json new file mode 100644 index 000000000..242cf5a40 --- /dev/null +++ b/docs/evidence/gate13-20260831-a-cost-authorization.json @@ -0,0 +1,186 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-a", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "4818da304f4eeafc81978873bcdfa8a41f6cad36", + "linux_lifecycle_helper_commit": "c3dc9234af7980bcaffd481c6f8e4e974ed117d4" + }, + "authorization": { + "combined_cloud_ceiling_usd": "100.00", + "ledger_committed_before_run_usd": "0.00", + "maximum_estimate_usd": "52.00", + "route_maximum_estimate_usd": "26.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "48.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "same-day previously accepted 14-hour G2/L4 route ceiling plus the same client ceiling; both new clients are CPU-only e2-standard-8 rather than the prior L4 client class" + }, + "immutable_inputs": { + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "sha256": "2272b3bd9a59ccbe66963558f2cee66b41f68403f611f8eff92362eebf8e22b8", + "bytes": 131385 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "sha256": "f50497fa06f465e8dba8b146e963faa11d102bc7c7fc8e5073e6c0b9e1beaf94", + "bytes": 113260 + } + }, + "provider_plan_digest": "sha256:313f5d34eefd64c71e265bdb7044d8ef5f56550360a7e9a7104265434292fd69", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-a-node", + "zone": "us-central1-a", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260819", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 50400, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-a-dht", + "route-20260831-a-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-a-node", + "gate13-20260831-a-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-a-win", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image_family": "windows-2025", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-windows-qwen-e", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-a-client" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-a-linux", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image_family": "ubuntu-2404-lts-amd64", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-linux-gemma-e", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-a-client" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "focused_software_tests_required_before_create": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-a-node", + "route-20260831-a-node boot disk", + "route-20260831-a-dht", + "route-20260831-a-iap", + "gate13-20260831-a-win", + "gate13-20260831-a-win boot disk", + "gate13-20260831-a-linux", + "gate13-20260831-a-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 2, + "previous_provider_plan_digest": "sha256:d3854d047beba36d1415382b296f90f2aaf0fc135d45c7613852c69b85ccf5a1", + "reason": "target both fresh client hosts through the same exact run-scoped IAP-only firewall without reusing the route DHT tag or creating another resource", + "resource_set_changed": false, + "cost_ceiling_changed": false + } +} diff --git a/docs/evidence/gate13-20260831-a-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-a-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..17d86b07f --- /dev/null +++ b/docs/evidence/gate13-20260831-a-failed-attempt-and-cleanup.json @@ -0,0 +1,88 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-attempt", + "run_id": "gate13-20260831-a", + "gate": 13, + "result": "failed", + "recorded_at": "2026-08-31T18:03:21Z", + "authorization": { + "provider_plan_digest": "sha256:313f5d34eefd64c71e265bdb7044d8ef5f56550360a7e9a7104265434292fd69", + "maximum_estimate_usd": "52.00", + "route_source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_source_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_helper_commit": "4818da304f4eeafc81978873bcdfa8a41f6cad36", + "linux_helper_commit": "c3dc9234af7980bcaffd481c6f8e4e974ed117d4" + }, + "attempt": { + "route_created": true, + "windows_client_created": true, + "linux_client_created": true, + "durable_run_controller_present": false, + "state_aware_reattachment_available": false, + "windows": { + "package_download_verified": true, + "package_sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "package_bytes": 2695065068, + "clean_preflight_passed": true, + "complete_lifecycle_record_present": false + }, + "linux": { + "package_download_verified": true, + "package_sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "package_bytes": 3360717934, + "initial_archive_preflight_passed": false, + "retry_exit_code": 2, + "bounded_diagnostic_failed": true, + "complete_lifecycle_record_present": false + }, + "windows_16_phase_lifecycle_complete": false, + "linux_16_phase_lifecycle_complete": false, + "acceptance_passed": false + }, + "failure": { + "class": "non_durable_orchestration", + "summary": "The paid qualification depended on transient operator sessions and one-off repair scripts, had no authoritative persisted state for crash reattachment, and advanced both clients before a single bootstrap and route path had passed.", + "fresh_host_evidence_reusable": false, + "gate_status": "IN PROGRESS", + "later_gates_unblocked": false + }, + "required_correction": { + "single_idempotent_controller_actions": [ + "reconcile", + "start", + "status", + "collect", + "cleanup" + ], + "reconcile_before_mutation": true, + "host_jobs_survive_operator_disconnect": true, + "route_and_transport_probe_before_full_clients": true, + "windows_lifecycle_collected_and_client_deleted_before_linux_create": true, + "failed_or_diagnostically_modified_client_reused_for_acceptance": false, + "all_16_phases_still_required": true, + "controller_state_contract_commit": "ddfb7c617b428a97b33d2b28e42f4fb75f3509ce", + "controller_contract_tests_passed": 15, + "gate13_regression_tests_passed": 187 + }, + "cleanup": { + "performed_at": "2026-08-31T18:03:21Z", + "route_instance_absent": true, + "route_boot_disk_absent": true, + "route_dht_firewall_absent": true, + "route_iap_firewall_absent": true, + "windows_instance_absent": true, + "windows_boot_disk_absent": true, + "linux_instance_absent": true, + "linux_boot_disk_absent": true, + "all_exact_run_resources_absent": true, + "protected_bootstrap_running": true + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + } +} diff --git a/docs/evidence/gate13-20260831-b-cost-authorization.json b/docs/evidence/gate13-20260831-b-cost-authorization.json new file mode 100644 index 000000000..73b6f64ce --- /dev/null +++ b/docs/evidence/gate13-20260831-b-cost-authorization.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-b", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "52.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "392.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "same-day accepted Gate 13 ceilings scaled conservatively by hard duration: the 16-hour G2/L4 route is rounded up to USD 30 from the prior 14-hour USD 26 ceiling, and the two sequential 6-hour e2-standard-8 CPU clients retain the prior USD 26 ceiling" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "44b26b0f9828121fed9eef65d830bc9c02760fb8f8ddbb31ed0f1491c1b5d9d4", + "bytes": 41294 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "bytes": 132097 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + } + }, + "provider_plan_digest": "sha256:3f3f921ded6eed1729aff175f5c91b4effe1966a31c82bdbe41ed69075442d64", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-b-node", + "zone": "us-central1-a", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260819", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-b-dht", + "route-20260831-b-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-b-node", + "gate13-20260831-b-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-b-win", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-b-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-b-client" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-b-linux", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-b-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-b-client" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "read_only_checked_at": "2026-08-31T20:15:48Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 219, + "independent_review_matrix_before_final_reservation_guard": 218, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-b-node", + "route-20260831-b-node boot disk", + "route-20260831-b-dht", + "route-20260831-b-iap", + "gate13-20260831-b-win", + "gate13-20260831-b-win boot disk", + "gate13-20260831-b-linux", + "gate13-20260831-b-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "previous_provider_plan_digest": null, + "reason": "fresh durable route-first, sequential-client run under the owner-raised USD 500 epoch ceiling; no failed-run resource or authorization is reused", + "resource_set_changed": true, + "cost_ceiling_changed": true + } +} diff --git a/docs/evidence/gate13-20260831-b-durable-host-job-prerequisite.json b/docs/evidence/gate13-20260831-b-durable-host-job-prerequisite.json new file mode 100644 index 000000000..9b8ec3d44 --- /dev/null +++ b/docs/evidence/gate13-20260831-b-durable-host-job-prerequisite.json @@ -0,0 +1,122 @@ +{ + "schema_version": 1, + "scope": "gate13-durable-native-host-job-prerequisite", + "recorded_at": "2026-08-31T20:27:03Z", + "result": "passed_software_prerequisite", + "gate": 13, + "gate_status": "in_progress", + "source": { + "commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "base_commit": "2fe0de9e1591e36be918fcbac82ecf72c96f8959", + "branch": "codex/gate13-20260831-a", + "pushed": true + }, + "artifacts": { + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "sha256": "44b26b0f9828121fed9eef65d830bc9c02760fb8f8ddbb31ed0f1491c1b5d9d4", + "bytes": 41294 + }, + "run_controller": { + "path": "scripts/gate13_run_controller.py", + "sha256": "32b31a3380f4a0e295c0185b9d6c2078b072b601d48a846acc119fbc555f2f8d", + "bytes": 33795 + }, + "linux_lifecycle_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_lifecycle_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "sha256": "9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "bytes": 132097 + } + }, + "contract": { + "action_intent_persisted_before_mutation": true, + "route_acceptance_intent_never_rearmed": true, + "windows_start_intent_never_rearmed": true, + "linux_start_intent_never_rearmed": true, + "client_attempt_ordinal_maximum": 1, + "clients_sequential": true, + "windows_supervisor": "Scheduled Task", + "windows_principal": "exact current ordinary qualification user", + "windows_logon_type": "Interactive", + "windows_run_level": "Limited", + "linux_supervisor": "transient systemd service", + "linux_principal": "gate13", + "linux_no_new_privileges": false, + "linux_sudo_required_for_inner_owned_cgroups": true, + "native_command_and_safety_settings_exactly_inventoried": true, + "lifecycle_config_path_and_digest_bound": true, + "windows_lifecycle_config_exactly_beside_entrypoint": true, + "linux_exec_start_structure_exactly_bound": true, + "stdout_maximum_bytes": 1048576, + "stderr_maximum_bytes": 262144, + "timeout_and_overflow_tree_shutdown": true, + "linux_termination_enters_finally_cleanup": true, + "terminal_record_contains_evidence_digest_only": true, + "successful_collection_revalidates_canonical_evidence": true + }, + "verification": { + "broad_gate13_matrix": { + "passed": 217, + "failed": 0, + "command_scope": [ + "tests/test_gate13_host_job.py", + "tests/test_gate13_run_controller.py", + "tests/test_gate13_packaged_lifecycle.py", + "tests/test_gate13_windows_packaged_lifecycle.py", + "tests/test_gate13_linux_packaged_lifecycle.py", + "tests/test_gate13_linux_localhost_inference.py", + "tests/test_gate13_download_artifact.py", + "desktop/tests/test_build_desktop.py", + "desktop/tests/test_credentials.py" + ] + }, + "focused_host_job_matrix": { + "passed": 23, + "failed": 0 + }, + "independent_broad_gate13_matrix": { + "passed": 217, + "failed": 0, + "warnings": 29 + }, + "windows_powershell_native_parser": "passed", + "python_compile": "passed", + "black_check": "passed", + "isort_check": "passed", + "git_diff_check": "passed" + }, + "provider_preflight": { + "mutation_performed": false, + "native_authentication": "passed", + "protected_bootstrap_running": true, + "next_run_namespace": "gate13-20260831-b", + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "g2_standard_8_available": true, + "e2_standard_8_available": true, + "windows_2025_image_resolved": true, + "ubuntu_2404_image_resolved": true, + "l4_quota_limit": 1, + "l4_quota_usage": 0 + }, + "cost": { + "cloud_resources_created": 0, + "cloud_resources_changed": 0, + "cloud_spend_usd": "0.00", + "current_epoch_committed_before_next_run_usd": "52.00", + "current_epoch_remaining_usd": "48.00" + }, + "claims": { + "paid_run_authorized_by_this_record": false, + "clean_host_lifecycle_completed": false, + "gate_13_passed": false, + "credits_work": false, + "macos_work": false + } +} diff --git a/docs/evidence/gate13-20260831-b-failed-start-and-cleanup.json b/docs/evidence/gate13-20260831-b-failed-start-and-cleanup.json new file mode 100644 index 000000000..5cea9d854 --- /dev/null +++ b/docs/evidence/gate13-20260831-b-failed-start-and-cleanup.json @@ -0,0 +1,62 @@ +{ + "schema_version": 1, + "scope": "gate13-route-start-failure-and-cleanup", + "run_id": "gate13-20260831-b", + "gate": 13, + "result": "failed_cleaned", + "recorded_at": "2026-08-31T21:04:21Z", + "source": { + "controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "authorization_commit": "c1e1f86", + "provider_plan_digest": "sha256:3f3f921ded6eed1729aff175f5c91b4effe1966a31c82bdbe41ed69075442d64" + }, + "stages": { + "native_auth_and_provider_preflight": "passed", + "intent_persisted_before_mutation": true, + "dht_firewall_create": "passed", + "iap_firewall_create": "failed", + "route_instance_create": "not_attempted", + "client_create": "not_attempted", + "cleanup": "passed" + }, + "failure": { + "code": "iap_target_tags_collapsed_by_operator_shell", + "classification": "operator_command_boundary", + "provider_or_environment_failure": false, + "detail": "The two exact IAP target tags reached gcloud as one space-joined value. The command failed before route instance creation." + }, + "cleanup": { + "route_instance_absent": true, + "route_disk_absent": true, + "windows_instance_and_disk_absent": true, + "linux_instance_and_disk_absent": true, + "dht_firewall_absent": true, + "iap_firewall_absent": true, + "protected_bootstrap_running": true + }, + "controller_terminal": { + "phase": "CLEANED_FAILURE", + "failure_code": "resources_disappeared_before_completion", + "cleanup_verified": true, + "revision": 2 + }, + "cost": { + "maximum_estimate_usd": "56.00", + "billable_instance_created": false, + "observed_cost_usd": null, + "maximum_remains_committed": true + }, + "privacy": { + "credentials_retained": false, + "provider_output_retained": false, + "private_paths_retained": false, + "endpoints_retained": false + }, + "claims": { + "route_accepted": false, + "lifecycle_started": false, + "gate_13_passed": false, + "credits_work": false, + "macos_work": false + } +} diff --git a/docs/evidence/gate13-20260831-c-cost-authorization.json b/docs/evidence/gate13-20260831-c-cost-authorization.json new file mode 100644 index 000000000..accbb13ec --- /dev/null +++ b/docs/evidence/gate13-20260831-c-cost-authorization.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-c", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "108.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "336.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "same-day accepted Gate 13 ceilings scaled conservatively by hard duration: the 16-hour G2/L4 route is rounded up to USD 30 from the prior 14-hour USD 26 ceiling, and the two sequential 6-hour e2-standard-8 CPU clients retain the prior USD 26 ceiling" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "44b26b0f9828121fed9eef65d830bc9c02760fb8f8ddbb31ed0f1491c1b5d9d4", + "bytes": 41294 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "bytes": 132097 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + } + }, + "provider_plan_digest": "sha256:07b6cd399ef7a9733602dfc19a741feddec8d15e5f4b5bac7347a192675f6d9c", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-c-node", + "zone": "us-central1-a", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260819", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-c-dht", + "route-20260831-c-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-c-node", + "gate13-20260831-c-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-c-win", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-c-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-c-client" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-c-linux", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-c-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-c-client" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "read_only_checked_at": "2026-08-31T21:06:16Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 219, + "independent_review_matrix_before_final_reservation_guard": 218, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-c-node", + "route-20260831-c-node boot disk", + "route-20260831-c-dht", + "route-20260831-c-iap", + "gate13-20260831-c-win", + "gate13-20260831-c-win boot disk", + "gate13-20260831-c-linux", + "gate13-20260831-c-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "previous_provider_plan_digest": null, + "reason": "fresh replacement after the prior identity failed before VM creation; exact resource names change and the corrected IAP target tags remain two explicit values", + "resource_set_changed": true, + "cost_ceiling_changed": true + } +} diff --git a/docs/evidence/gate13-20260831-c-terminal-state-and-cleanup.json b/docs/evidence/gate13-20260831-c-terminal-state-and-cleanup.json new file mode 100644 index 000000000..12162d0c7 --- /dev/null +++ b/docs/evidence/gate13-20260831-c-terminal-state-and-cleanup.json @@ -0,0 +1,58 @@ +{ + "schema_version": 1, + "scope": "gate13-run-controller-terminal-and-cleanup", + "run_id": "gate13-20260831-c", + "gate": 13, + "result": "failed_cleaned", + "recorded_at": "2026-08-31T21:38:14Z", + "source": { + "authorization_commit": "0dc2345f7c0880bbb63b1d8951187e4e4b744a84", + "authorization_sha256": "sha256:3bb9f79d91d9ef4df2e0082f0358f5edfef199e80f15359560013a10a8b758bd", + "provider_plan_digest": "sha256:07b6cd399ef7a9733602dfc19a741feddec8d15e5f4b5bac7347a192675f6d9c" + }, + "controller_terminal": { + "phase": "CLEANED_FAILURE", + "failure_code": "resources_disappeared_before_completion", + "cleanup_verified": true, + "revision": 2, + "next_action": "none", + "windows_consumed": false, + "linux_consumed": false + }, + "classification": { + "category": "local_orchestration_state", + "durable_provider_run_record_present": false, + "product_failure_evidenced": false, + "reusable_run_id": false, + "detail": "The reserved run was armed locally and reached a terminal absence state without a durable provider execution record. It is retired rather than reset or reused." + }, + "provider_reconciliation": { + "checked_at": "2026-08-31T21:38:14Z", + "route_instance_absent": true, + "route_disk_absent": true, + "windows_instance_and_disk_absent": true, + "linux_instance_and_disk_absent": true, + "dht_firewall_absent": true, + "iap_firewall_absent": true, + "global_gpu_limit": 1, + "global_gpu_usage": 0, + "protected_bootstrap_running": true + }, + "cost": { + "maximum_estimate_usd": "56.00", + "observed_cost_usd": null, + "maximum_remains_committed": true + }, + "privacy": { + "credentials_retained": false, + "provider_output_retained": false, + "private_paths_retained": false, + "endpoints_retained": false + }, + "claims": { + "route_accepted": false, + "windows_lifecycle_passed": false, + "linux_lifecycle_passed": false, + "gate_13_passed": false + } +} diff --git a/docs/evidence/gate13-20260831-d-cost-authorization.json b/docs/evidence/gate13-20260831-d-cost-authorization.json new file mode 100644 index 000000000..5915c2f36 --- /dev/null +++ b/docs/evidence/gate13-20260831-d-cost-authorization.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-d", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "0dc2345f7c0880bbb63b1d8951187e4e4b744a84", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "164.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "280.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "same-day accepted Gate 13 ceilings scaled conservatively by hard duration: the 16-hour G2/L4 route is rounded up to USD 30 from the prior 14-hour USD 26 ceiling, and the two sequential 6-hour e2-standard-8 CPU clients retain the prior USD 26 ceiling" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "44b26b0f9828121fed9eef65d830bc9c02760fb8f8ddbb31ed0f1491c1b5d9d4", + "bytes": 41294 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "bytes": 132097 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + } + }, + "provider_plan_digest": "sha256:d32050a51b8f696aa224fc7e748c9113e174e3c3069c1f8b2bc769b0c5ecea18", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-d-node", + "zone": "us-central1-a", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260819", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-d-dht", + "route-20260831-d-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-d-node", + "gate13-20260831-d-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-d-win", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-d-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-d-client" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-d-linux", + "zone": "us-central1-a", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-d-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-d-client" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "read_only_checked_at": "2026-08-31T21:38:14Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 219, + "independent_review_matrix_before_final_reservation_guard": 218, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-d-node", + "route-20260831-d-node boot disk", + "route-20260831-d-dht", + "route-20260831-d-iap", + "gate13-20260831-d-win", + "gate13-20260831-d-win boot disk", + "gate13-20260831-d-linux", + "gate13-20260831-d-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "previous_provider_plan_digest": null, + "reason": "fresh replacement after the prior identity failed before VM creation; exact resource names change and the corrected IAP target tags remain two explicit values", + "resource_set_changed": true, + "cost_ceiling_changed": true + } +} diff --git a/docs/evidence/gate13-20260831-d-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-d-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..118cca646 --- /dev/null +++ b/docs/evidence/gate13-20260831-d-failed-attempt-and-cleanup.json @@ -0,0 +1,76 @@ +{ + "schema_version": 1, + "scope": "gate13-failed-attempt-and-cleanup", + "run_id": "gate13-20260831-d", + "recorded_at": "2026-08-31T22:29:33Z", + "result": "failed_cleanup_verified", + "gate": 13, + "route_acceptance": { + "completed_before_client_create": true, + "result": "passed", + "evidence_digest": "sha256:1a8f1adc3bdf876466e4c4a6f66e35a1650286476cc0fd499092bd393f333624", + "primary_model": "Qwen3.5 2B", + "standby_model": "Gemma 4 E2B IT", + "fallback_and_restoration_passed": true, + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained": false + }, + "windows_attempt": { + "lifecycle_run_id": "gate13-20260831-d-windows", + "attempt_ordinal": 1, + "result": "failed", + "failure_code": "windows_packaged_lifecycle_failed", + "terminal_failure_code": "lifecycle_failed", + "terminal_exit_code": 2, + "canonical_lifecycle_evidence_present": false, + "failure_record_sha256": "sha256:8d44d0f9529cf358e8941d1114265d654f537ab1cdc75a6944abe6a3b7f9b53e", + "linux_client_created": false + }, + "findings": [ + { + "class": "host_supervisor", + "cause": "The Interactive scheduled task never ran from a headless SSH session, and principal observation compared a scheduler-normalized leaf name to a qualified identity.", + "proof": "The old task remained Ready with Task Scheduler result 0x41303 and no status, terminal, evidence, or stderr record.", + "correction_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "correction": "Use S4U with Limited run level and compare resolved principal SID to the current identity SID.", + "repaired_before_lifecycle_attempt": true + }, + { + "class": "client_prerequisite", + "cause": "The exact package downloader and config were staged, but the 2,695,065,068-byte archive itself was not downloaded before the one-attempt job was armed.", + "proof": "The repaired S4U supervisor ran, wrote its status and terminal records, and the lifecycle failed immediately with its bounded failure record; no canonical phase record was produced.", + "correction": "Download and verify the exact archive on the fresh client before host-job start; verify its fixed size and SHA-256 before consuming the attempt.", + "gcp_related": false + } + ], + "gate11_comparison": { + "repeated_gate11_gcp_issue": false, + "gate11_quota_fix_reused": true, + "route_product_acceptance_passed": true, + "gcp_route_or_quota_blocker": false, + "delay_class": "gate13_operator_and_supervisor_orchestration" + }, + "cleanup": { + "native_windows_task_absent": true, + "exact_instances_absent": true, + "exact_disks_absent": true, + "exact_firewalls_absent": true, + "regional_l4_limit": 1, + "regional_l4_usage": 0, + "verified_at": "2026-08-31T22:29:33Z", + "protected_bootstrap_deleted": false + }, + "cost": { + "maximum_committed_usd": "56.00", + "observed_cost_usd": null, + "additional_supervisor_repair_cost_usd": "0.00" + }, + "claims": { + "gate_13_passed": false, + "windows_lifecycle_passed": false, + "linux_lifecycle_started": false, + "run_id_reusable": false, + "resources_reusable": false + } +} diff --git a/docs/evidence/gate13-20260831-d-windows-supervisor-repair-authorization.json b/docs/evidence/gate13-20260831-d-windows-supervisor-repair-authorization.json new file mode 100644 index 000000000..008ba5454 --- /dev/null +++ b/docs/evidence/gate13-20260831-d-windows-supervisor-repair-authorization.json @@ -0,0 +1,54 @@ +{ + "schema_version": 1, + "scope": "gate13-windows-supervisor-repair-authorization", + "run_id": "gate13-20260831-d", + "recorded_at": "2026-08-31T22:20:51Z", + "objective": "Launch the already-authorized Windows packaged lifecycle exactly once under a headless ordinary-user supervisor.", + "finding": { + "task_name": "communityai-gate13-gate13-20260831-d-windows", + "registered_principal": "M", + "registered_logon_type": "Interactive", + "registered_run_level": "Limited", + "scheduler_state": "Ready", + "scheduler_last_result_decimal": 267011, + "scheduler_last_result_hex": "0x41303", + "scheduler_interpretation": "task_has_not_yet_run", + "status_record_present": false, + "terminal_record_present": false, + "evidence_record_present": false, + "lifecycle_attempt_consumed": false, + "additional_cloud_resources_required": false + }, + "root_cause": [ + "Interactive scheduled tasks do not start from this headless SSH operator session.", + "Task Scheduler canonicalized the registered principal to a leaf account name, while the observer compared it to the qualified current identity string." + ], + "repair_binding": { + "implementation_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "host_adapter_path": "scripts/gate13_host_job.py", + "host_adapter_sha256": "sha256:922a9365269a2c1f0aee09473b15fdcc9b9a08522cfd5e9934145b4966b0ad31", + "host_adapter_bytes": 41592, + "replacement_logon_type": "S4U", + "principal_match": "resolved_principal_sid_equals_current_identity_sid", + "unchanged_entrypoint_sha256": "sha256:9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "unchanged_lifecycle_config_sha256": "sha256:a926514b35b10baab742ff270db606fef2f7ddf58317e1023f12b1cd66affbd0" + }, + "authorized_actions": [ + "Verify the exact old task name, action, ordinary-user principal, Interactive logon type, Limited run level, Ready state, 0x41303 last result, and absence of status, terminal, and evidence records.", + "Unregister only that exact never-run task.", + "Replace only the host adapter and its digest binding in host-job.json.", + "Register and start the same exact task once with S4U, Limited run level, and SID-based principal verification.", + "Fail closed if any lifecycle output appeared before repair or if the old task binding differs." + ], + "invariants": [ + "The paid Windows VM is not recreated.", + "The lifecycle entrypoint, package, model, route acceptance, evidence contract, and one-attempt ceiling are unchanged.", + "No second lifecycle attempt is authorized.", + "Windows evidence must still be collected and the Windows VM deleted before Linux is created." + ], + "added_cost_usd": "0.00", + "tests": { + "command": ".\\.venv-cuda\\Scripts\\python.exe -m pytest -q tests/test_gate13_host_job.py", + "result": "23 passed" + } +} diff --git a/docs/evidence/gate13-20260831-e-cost-authorization.json b/docs/evidence/gate13-20260831-e-cost-authorization.json new file mode 100644 index 000000000..99820a7ee --- /dev/null +++ b/docs/evidence/gate13-20260831-e-cost-authorization.json @@ -0,0 +1,230 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-e", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "220.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "224.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "unchanged same-day Gate 13 ceilings: one bounded 16-hour G2/L4 route rounded to USD 30 and two sequential bounded 6-hour e2-standard-8 CPU clients at USD 26 total; each exact package archive must be downloaded and hash-verified before its one-attempt host job is armed" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "sha256": "922a9365269a2c1f0aee09473b15fdcc9b9a08522cfd5e9934145b4966b0ad31", + "bytes": 41592 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "9b860ff820851b31ac272ee4d28dd3a899f56014b8dfce45e912ba4fc9ab605c", + "bytes": 132097 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "sha256": "1972d54efe5b9ffb73c3d96e005edc238f7f57aaf720b202d091e541c82e044a", + "bytes": 3371 + } + }, + "provider_plan_digest": "sha256:9ca0fa516017c4a3709a467752f779bcb3bbc0a7c790f9bc61de56d385804c62", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-e-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-e-dht", + "route-20260831-e-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-e-node", + "gate13-20260831-e-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-e-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-e-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-e-client" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-e-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-e-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-e-client" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "read_only_checked_at": "2026-08-31T22:40:11Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 23, + "independent_review_matrix_before_final_reservation_guard": 0, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-e-node", + "route-20260831-e-node boot disk", + "route-20260831-e-dht", + "route-20260831-e-iap", + "gate13-20260831-e-win", + "gate13-20260831-e-win boot disk", + "gate13-20260831-e-linux", + "gate13-20260831-e-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 4, + "previous_provider_plan_digest": "sha256:0b8c423090e27ec792c55287676d753db4042a5817d25d550d2ef5f2e4b4119b", + "reason": "the controller requires route and sequential clients in one exact zone; after the us-central1-a L4 stockout created no VM or disk, the complete run moves to provider-recommended us-central1-b at unchanged cost", + "resource_set_changed": true, + "cost_ceiling_changed": false + } +} diff --git a/docs/evidence/gate13-20260831-e-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-e-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..0092ed872 --- /dev/null +++ b/docs/evidence/gate13-20260831-e-failed-attempt-and-cleanup.json @@ -0,0 +1,95 @@ +{ + "schema_version": 1, + "scope": "gate13-failed-attempt-and-cleanup", + "run_id": "gate13-20260831-e", + "recorded_at": "2026-08-31T23:45:44Z", + "result": "failed_cleanup_verified", + "gate": 13, + "route_acceptance": { + "completed_before_client_create": true, + "result": "passed", + "evidence_digest": "sha256:064e1699bff2d0998c37ec6a3be5d37f8b6400a12de9627ea13fddb302508e55", + "primary_model": "Qwen3.5 2B", + "standby_model": "Gemma 4 E2B IT", + "fallback_and_restoration_passed": true, + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained": false + }, + "windows_attempt": { + "lifecycle_run_id": "gate13-20260831-e-windows", + "attempt_ordinal": 1, + "result": "failed", + "failure_code": "windows_packaged_lifecycle_failed", + "terminal_failure_code": "lifecycle_failed", + "terminal_exit_code": 2, + "started_at_unix": 1788219070, + "finished_at_unix": 1788219581, + "elapsed_seconds": 511, + "canonical_lifecycle_evidence_present": false, + "failure_record_sha256": "sha256:8d44d0f9529cf358e8941d1114265d654f537ab1cdc75a6944abe6a3b7f9b53e", + "failure_stderr_bytes": 0, + "linux_client_created": false + }, + "pre_start_proofs": { + "windows_archive_bytes": 2695065068, + "windows_archive_sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "archive_verified_before_attempt": true, + "ordinary_ssh_identity_verified": true, + "ordinary_ssh_identity_admin": false, + "scheduler_task_s4u": true, + "scheduler_task_run_level": "Limited", + "scheduler_running_observed_before_operator_demoted_account": true + }, + "findings": [ + { + "class": "windows_ssh_authorization", + "cause": "The Windows OpenSSH service allowed only Administrators and OpenSSH Users; removing M from Administrators without adding it to OpenSSH Users made the valid public key appear rejected.", + "proof": "The OpenSSH operational log reported that M was not allowed because none of the user's groups were listed in AllowGroups. Adding M to OpenSSH Users restored SSH while an independent token check reported is-admin=false.", + "correction": "Provision M in the existing OpenSSH Users group before removing it from Administrators.", + "gcp_related": false + }, + { + "class": "windows_supervisor_boundary", + "cause": "Windows Server 2025 denies an ordinary account access to the ScheduledTasks CIM provider and denies task registration in the root folder, even when the task principal is the same ordinary user.", + "proof": "Get-ScheduledTask failed with CIM access denied and a direct Task Scheduler COM registration probe failed with E_ACCESSDENIED. A privileged bootstrap could register the exact S4U/Limited task, which then ran durably.", + "correction": "Do not ask the ordinary account to provision its own native supervisor. For the next run, keep the bounded host adapter in the ordinary foreground and keep the IAP SSH transport durable at the operator boundary.", + "gcp_related": false + }, + { + "class": "lifecycle_diagnosability", + "cause": "The Windows lifecycle catch path emitted one generic failure record and failure cleanup removed the phase workspace, so the completed attempt could not identify the failed acceptance phase.", + "proof": "After 511 seconds the only lifecycle output was the 91-byte generic failure record and stderr was empty.", + "correction": "Emit only the hard-coded current phase name with the generic failure code; never emit exception text, paths, endpoints, prompts, outputs, or credentials.", + "gcp_related": false + } + ], + "gate11_comparison": { + "repeated_gate11_gcp_issue": false, + "gate11_quota_fix_reused": true, + "route_product_acceptance_passed": true, + "gcp_route_or_quota_blocker": false, + "delay_class": "gate13_windows_operator_boundary_and_packaged_lifecycle" + }, + "cleanup": { + "native_windows_task_absent_with_instance": true, + "exact_instances_absent": true, + "exact_disks_absent": true, + "exact_firewalls_absent": true, + "regional_l4_limit": 1, + "regional_l4_usage": 0, + "verified_at": "2026-08-31T23:45:44Z", + "protected_bootstrap_deleted": false + }, + "cost": { + "maximum_committed_usd": "56.00", + "observed_cost_usd": null + }, + "claims": { + "gate_13_passed": false, + "windows_lifecycle_passed": false, + "linux_lifecycle_started": false, + "run_id_reusable": false, + "resources_reusable": false + } +} diff --git a/docs/evidence/gate13-20260831-f-cost-authorization.json b/docs/evidence/gate13-20260831-f-cost-authorization.json new file mode 100644 index 000000000..d55afdf6b --- /dev/null +++ b/docs/evidence/gate13-20260831-f-cost-authorization.json @@ -0,0 +1,232 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-f", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "2b895f59cabb8b294c8afad09dbcacfb51a0db6b", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "2b895f59cabb8b294c8afad09dbcacfb51a0db6b", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "276.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "168.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "unchanged same-day Gate 13 ceilings: one bounded 16-hour G2/L4 route rounded to USD 30 and two sequential bounded 6-hour e2-standard-8 CPU clients at USD 26 total; each exact package archive is downloaded and hash-verified before one durable foreground host-adapter execution over IAP SSH as the ordinary OS user" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "3a1d1fcaa8430e3c0bfc47910666f11989eb0a3d", + "sha256": "922a9365269a2c1f0aee09473b15fdcc9b9a08522cfd5e9934145b4966b0ad31", + "bytes": 41592 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "2b895f59cabb8b294c8afad09dbcacfb51a0db6b", + "sha256": "d6363fe00867f2b6ffccc855197ee078d808fa7890ec0234e73050db9d5aa6e1", + "bytes": 132564 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "sha256": "1972d54efe5b9ffb73c3d96e005edc238f7f57aaf720b202d091e541c82e044a", + "bytes": 3371 + } + }, + "provider_plan_digest": "sha256:c9a2aafc84940df901a7db1755af2e684f845b78dcdfac04332cfed36388ba25", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-f-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-f-dht", + "route-20260831-f-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-f-node", + "gate13-20260831-f-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-f-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-f-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-f-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-f-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-f-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-f-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true, + "client_host_adapter_mode": "foreground_execute_over_durable_iap_ssh" + } + }, + "preflight": { + "read_only_checked_at": "2026-08-31T23:49:33Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 39, + "independent_review_matrix_before_final_reservation_guard": 0, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-e-node", + "route-20260831-e-node boot disk", + "route-20260831-e-dht", + "route-20260831-e-iap", + "gate13-20260831-e-win", + "gate13-20260831-e-win boot disk", + "gate13-20260831-e-linux", + "gate13-20260831-e-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "reason": "replace the Windows Scheduled Tasks provisioning boundary with one durable foreground host-adapter execution over IAP SSH as the ordinary user; retain exact archives, route-first ordering, all sixteen phases, costs, privacy, and cleanup", + "resource_set_changed": true, + "cost_ceiling_changed": false + } +} diff --git a/docs/evidence/gate13-20260831-f-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-f-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..bde995797 --- /dev/null +++ b/docs/evidence/gate13-20260831-f-failed-attempt-and-cleanup.json @@ -0,0 +1,96 @@ +{ + "schema_version": 1, + "scope": "gate13-failed-attempt-and-cleanup", + "run_id": "gate13-20260831-f", + "recorded_at": "2026-09-01T00:46:17Z", + "result": "failed_cleanup_verified", + "gate": 13, + "route_acceptance": { + "completed_before_client_create": true, + "result": "passed", + "evidence_digest": "sha256:eb614c1b281a3661560536825edde1ad0e960c0c564e5aabd4900dd41bb2dea4", + "primary_model": "Qwen3.5 2B", + "standby_model": "Gemma 4 E2B IT", + "fallback_and_restoration_passed": true, + "total_duration_ms": 295357, + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained": false + }, + "windows_attempt": { + "lifecycle_run_id": "gate13-20260831-f-windows", + "attempt_ordinal": 1, + "execution_transport": "durable_iap_ssh_foreground_as_ordinary_user", + "ordinary_user_admin": false, + "result": "failed", + "failure_code": "windows_packaged_lifecycle_failed", + "failure_phase": "signed_bootstrap", + "terminal_failure_code": "lifecycle_failed", + "terminal_exit_code": 2, + "started_at_unix": 1788222739, + "finished_at_unix": 1788223261, + "elapsed_seconds": 522, + "failure_record_bytes": 126, + "failure_stderr_bytes": 0, + "canonical_lifecycle_evidence_present": false, + "linux_client_created": false + }, + "pre_start_proofs": { + "windows_archive_bytes": 2695065068, + "windows_archive_sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "archive_verified_before_attempt": true, + "staged_file_hashes_verified": 12, + "host_config_validated": true, + "status_absent_before_start": true, + "terminal_absent_before_start": true, + "evidence_absent_before_start": true + }, + "findings": [ + { + "class": "supervisor_hypothesis_eliminated", + "finding": "The direct ordinary-user foreground execution failed at the same elapsed time and same signed_bootstrap phase as the S4U/Limited execution.", + "conclusion": "Scheduled Tasks, S4U, SSH account authorization, and the privileged provisioning boundary are not the packaged lifecycle failure.", + "gcp_related": false + }, + { + "class": "windows_runtime_environment", + "finding": "The host adapter passed only SYSTEMROOT, WINDIR, TEMP, TMP, and USERPROFILE into the Windows lifecycle. It removed standard non-secret user runtime variables including APPDATA, LOCALAPPDATA, PROGRAMDATA, COMSPEC, PATH, and PATHEXT before the packaged desktop was started.", + "timing_inference": "The roughly 300-second difference between preliminary package work and terminal failure is consistent with Wait-Gate13ProductStatus exhausting its 300-second readiness bound in signed_bootstrap.", + "correction": "Keep a fixed allowlist of standard non-secret Windows runtime variables while continuing to exclude arbitrary variables and credential/token names.", + "gcp_related": false + }, + { + "class": "bounded_failure_localization", + "finding": "The phase-bounded failure record localized the deterministic defect to signed_bootstrap without retaining exception text, paths, endpoints, prompts, outputs, or credentials.", + "correction": "Add a hard-coded operation name inside signed_bootstrap for the next run while preserving the same privacy boundary.", + "gcp_related": false + } + ], + "gate11_comparison": { + "repeated_gate11_gcp_issue": false, + "gate11_quota_fix_reused": true, + "route_product_acceptance_passed": true, + "gcp_route_or_quota_blocker": false, + "delay_class": "gate13_windows_host_environment_contract" + }, + "cleanup": { + "exact_instances_absent": true, + "exact_disks_absent": true, + "exact_firewalls_absent": true, + "regional_l4_limit": 1, + "regional_l4_usage": 0, + "verified_at": "2026-09-01T00:46:17Z", + "protected_bootstrap_deleted": false + }, + "cost": { + "maximum_committed_usd": "56.00", + "observed_cost_usd": null + }, + "claims": { + "gate_13_passed": false, + "windows_lifecycle_passed": false, + "linux_lifecycle_started": false, + "run_id_reusable": false, + "resources_reusable": false + } +} diff --git a/docs/evidence/gate13-20260831-g-cost-authorization.json b/docs/evidence/gate13-20260831-g-cost-authorization.json new file mode 100644 index 000000000..dec7ad6b3 --- /dev/null +++ b/docs/evidence/gate13-20260831-g-cost-authorization.json @@ -0,0 +1,232 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-g", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "332.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "112.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "unchanged same-day Gate 13 ceilings: one bounded 16-hour G2/L4 route rounded to USD 30 and two sequential bounded 6-hour e2-standard-8 CPU clients at USD 26 total; each exact package archive is downloaded and hash-verified before one durable foreground host-adapter execution over IAP SSH as the ordinary OS user" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "6a01626718a8cf7bd5743d2f07e20d808d6708280c9005cd6a03de8ff807567e", + "bytes": 42051 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "a85c51eb0231bcef991a77c57b622fad575049f000c8b95bbeec7f78eaec7a1e", + "bytes": 133351 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "sha256": "1972d54efe5b9ffb73c3d96e005edc238f7f57aaf720b202d091e541c82e044a", + "bytes": 3371 + } + }, + "provider_plan_digest": "sha256:f27f36158f2ad16019578555023cc854cb1e6e3b10ebae8cd3ed24d757b8e032", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-g-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-g-dht", + "route-20260831-g-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-g-node", + "gate13-20260831-g-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-g-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-g-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-g-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-g-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-g-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-g-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true, + "client_host_adapter_mode": "foreground_execute_over_durable_iap_ssh" + } + }, + "preflight": { + "read_only_checked_at": "2026-09-01T00:51:42.840Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 40, + "independent_review_matrix_before_final_reservation_guard": 0, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-g-node", + "route-20260831-g-node boot disk", + "route-20260831-g-dht", + "route-20260831-g-iap", + "gate13-20260831-g-win", + "gate13-20260831-g-win boot disk", + "gate13-20260831-g-linux", + "gate13-20260831-g-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "reason": "preserve a bounded allowlist of standard per-user Windows runtime variables required by the installed desktop package; retain direct ordinary-user execution, exact archives, route-first ordering, all sixteen phases, costs, privacy, and cleanup", + "resource_set_changed": true, + "cost_ceiling_changed": false + } +} diff --git a/docs/evidence/gate13-20260831-g-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-g-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..f3592182b --- /dev/null +++ b/docs/evidence/gate13-20260831-g-failed-attempt-and-cleanup.json @@ -0,0 +1,90 @@ +{ + "schema_version": 1, + "scope": "gate13-failed-attempt-and-cleanup", + "run_id": "gate13-20260831-g", + "recorded_at": "2026-09-01T01:24:46.392Z", + "result": "failed_cleanup_verified", + "provider_plan_digest": "sha256:f27f36158f2ad16019578555023cc854cb1e6e3b10ebae8cd3ed24d757b8e032", + "route_acceptance": { + "result": "passed", + "evidence_sha256": "04271d6ac93410974d26818f3c3f930f0031877ae6c6d11441c7e75569453633", + "total_duration_ms": 298408, + "qwen": { + "covered_blocks": 24, + "total_blocks": 24, + "peer_count": 1 + }, + "gemma": { + "covered_blocks": 35, + "total_blocks": 35, + "peer_count": 1 + }, + "primary_inference": true, + "fallback_inference": true, + "restoration_inference": true, + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained_in_evidence": false + }, + "windows_attempt": { + "lifecycle_run_id": "gate13-20260831-g-windows", + "attempt_ordinal": 1, + "execution_transport": "durable_iap_ssh_foreground_as_ordinary_user", + "ordinary_user": "M", + "ordinary_user_admin": false, + "package_preflight": { + "archive_bytes": 2695065068, + "archive_sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "download_result": "passed", + "url_retained": false + }, + "committed_adapter_sha256": "6a01626718a8cf7bd5743d2f07e20d808d6708280c9005cd6a03de8ff807567e", + "committed_entrypoint_sha256": "a85c51eb0231bcef991a77c57b622fad575049f000c8b95bbeec7f78eaec7a1e", + "started_at_unix": 1788225618, + "finished_at_unix": 1788225620, + "exit_code": 2, + "failure_code": "windows_packaged_lifecycle_failed", + "failure_phase": "package_verification", + "failure_operation": "package_verification", + "bounded_failure_record_bytes": 173, + "stderr_bytes": 0 + }, + "root_cause": { + "class": "operator_staging_omission", + "finding": "the fresh staging upload contained the eight executable and configuration files but omitted the four already-pinned release audit inputs required by Test-Gate13PackageAudit", + "missing_inputs": [ + "audit/desktop-metrics.json", + "audit/provenance.json", + "audit/release-metadata.json", + "audit/SHA256SUMS" + ], + "product_or_route_regression": false, + "next_attempt_change": "stage and hash-verify the four existing audit inputs before archive download and execution; no lifecycle or route redesign" + }, + "linux_attempt": { + "instance_created": false, + "attempt_ordinal": 0 + }, + "cleanup": { + "route_instance_absent": true, + "route_disk_absent": true, + "windows_instance_absent": true, + "windows_disk_absent": true, + "linux_instance_absent": true, + "linux_disk_absent": true, + "route_firewalls_absent": true, + "gpus_all_regions": { + "limit": 1, + "usage": 0 + }, + "protected_bootstrap_running": true + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + } +} diff --git a/docs/evidence/gate13-20260831-h-cost-authorization.json b/docs/evidence/gate13-20260831-h-cost-authorization.json new file mode 100644 index 000000000..ee9fa7d66 --- /dev/null +++ b/docs/evidence/gate13-20260831-h-cost-authorization.json @@ -0,0 +1,268 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-h", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "0c67f0b8912ce8323cae2008642783b4aa23f436", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "388.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "56.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "unchanged same-day Gate 13 ceilings: one bounded 16-hour G2/L4 route rounded to USD 30 and two sequential bounded 6-hour e2-standard-8 CPU clients at USD 26 total; both pinned four-file release-audit bundles and each exact package archive are staged and hash-verified before one durable foreground host-adapter execution as the ordinary OS user" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "6a01626718a8cf7bd5743d2f07e20d808d6708280c9005cd6a03de8ff807567e", + "bytes": 42051 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "a85c51eb0231bcef991a77c57b622fad575049f000c8b95bbeec7f78eaec7a1e", + "bytes": 133351 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "sha256": "1972d54efe5b9ffb73c3d96e005edc238f7f57aaf720b202d091e541c82e044a", + "bytes": 3371 + }, + "windows_audit_bundle": { + "audit/desktop-metrics.json": { + "sha256": "953fc814d3d7d6787cbe7ecc25e8ab9f60c68515b94c80575207e15e78d69549", + "bytes": 3795 + }, + "audit/provenance.json": { + "sha256": "ac04b71d35493ba4967628af1ac05ca290b1af09aab4e8955ac09031c87ce7f8", + "bytes": 1241883 + }, + "audit/release-metadata.json": { + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "bytes": 872 + }, + "audit/SHA256SUMS": { + "sha256": "a458760c1e4636956c9fba5a7da93ed869544f4817d5ab883b5d8b34cc9ad964", + "bytes": 674380 + } + }, + "linux_audit_bundle": { + "audit/desktop-metrics.json": { + "sha256": "5d2b261505e949a15c332c6e5bb817611e340bf897f9b1952ebde8e573e07bd2", + "bytes": 3798 + }, + "audit/provenance.json": { + "sha256": "c9b5e47017b003f6b2d81c9ab8273fcbf3c72f7743f1ebbf99383ae7cd5accda", + "bytes": 1357051 + }, + "audit/release-metadata.json": { + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "bytes": 872 + }, + "audit/SHA256SUMS": { + "sha256": "4766c587a1d8e430f892128b8667c04ae7e83bde869a3098b24012c4f99cb74d", + "bytes": 737970 + } + } + }, + "provider_plan_digest": "sha256:f243254cc5fb65f44d0c9e707be36feb3284fd6e15b15620882843798fb456b1", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-h-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-h-dht", + "route-20260831-h-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-h-node", + "gate13-20260831-h-client" + ] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-h-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-h-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-h-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-h-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-h-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-h-client", + "host_execution": "durable_iap_ssh_foreground_as_ordinary_user" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true, + "client_host_adapter_mode": "foreground_execute_over_durable_iap_ssh" + } + }, + "preflight": { + "read_only_checked_at": "2026-09-01T01:25:51.454Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 40, + "independent_review_matrix_before_final_reservation_guard": 0, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-h-node", + "route-20260831-h-node boot disk", + "route-20260831-h-dht", + "route-20260831-h-iap", + "gate13-20260831-h-win", + "gate13-20260831-h-win boot disk", + "gate13-20260831-h-linux", + "gate13-20260831-h-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "reason": "stage and hash-verify the four already-pinned release audit inputs for each platform before archive download and ordinary-user execution; retain the corrected Windows runtime allowlist, exact archives, route-first ordering, all sixteen phases, costs, privacy, and cleanup", + "resource_set_changed": true, + "cost_ceiling_changed": false + } +} diff --git a/docs/evidence/gate13-20260831-h-failed-attempt-and-cleanup.json b/docs/evidence/gate13-20260831-h-failed-attempt-and-cleanup.json new file mode 100644 index 000000000..2b5f5e1ae --- /dev/null +++ b/docs/evidence/gate13-20260831-h-failed-attempt-and-cleanup.json @@ -0,0 +1,54 @@ +{ + "schema_version": 1, + "scope": "gate13-failed-attempt-and-cleanup", + "run_id": "gate13-20260831-h", + "gate": 13, + "result": "failed-cleaned", + "recorded_at": "2026-09-01T02:28:35.232Z", + "source": { + "authorization_sha256": "sha256:6714a7a33b671c3fc177e182c354b80e3fb54c6657d5008eff74080c088be523", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "host_job_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20" + }, + "acceptance": { + "route_acceptance_passed": true, + "windows_archive_sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "windows_archive_bytes": 2695065068, + "windows_attempts": 1, + "windows_result": "failed", + "failure_phase": "signed_bootstrap", + "failure_operation": "product_readiness", + "linux_created": false, + "linux_attempts": 0, + "gate_passed": false + }, + "conclusion": "The exact route and package prerequisites passed. The opaque non-interactive Windows launch timed out waiting for product readiness and retained no useful product error. This run does not show a Gate 11 or GCP route failure.", + "cleanup": { + "verified_at": "2026-09-01T02:28:35.232Z", + "instances_absent": [ + "route-20260831-h-node", + "gate13-20260831-h-win", + "gate13-20260831-h-linux" + ], + "disks_absent": [ + "route-20260831-h-node", + "gate13-20260831-h-win", + "gate13-20260831-h-linux" + ], + "firewalls_absent": [ + "route-20260831-h-dht", + "route-20260831-h-iap" + ], + "global_l4_quota_limit": 1, + "global_l4_quota_usage": 0, + "protected_bootstrap_status": "RUNNING" + }, + "next_action": "Run the exact Windows package manually in a real interactive console session with visible node and desktop diagnostics; do not invoke the lifecycle wrapper until the manual flow passes.", + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false + } +} diff --git a/docs/evidence/gate13-20260831-i-cost-authorization.json b/docs/evidence/gate13-20260831-i-cost-authorization.json new file mode 100644 index 000000000..8a66f5118 --- /dev/null +++ b/docs/evidence/gate13-20260831-i-cost-authorization.json @@ -0,0 +1,291 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-lifecycle-cost-authorization", + "run_id": "gate13-20260831-i", + "gate": 13, + "result": "authorized", + "recorded_at": "2026-08-31", + "source": { + "reservation_commit": "9ad67da6728430b965add981d088821d4d600027", + "durable_controller_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "host_job_adapter_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_lifecycle_helper_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "linux_lifecycle_helper_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3" + }, + "authorization": { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "444.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "0.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-08-31", + "pricing_basis": "unchanged same-day Gate 13 ceiling: one bounded 16-hour G2/L4 route and two sequential bounded 6-hour CPU clients; Windows is exercised first in a real interactive console session with virtual display and visible product diagnostics, one step at a time, before any lifecycle automation is permitted" + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "ee0c05f83035ad22015fb089f5cb30f1713076e6", + "sha256": "3a33f1c0782222d400296c944651ca80a58fb67b4df79ce6a7d9c7216fd23b84", + "bytes": 34143 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "6a01626718a8cf7bd5743d2f07e20d808d6708280c9005cd6a03de8ff807567e", + "bytes": 42051 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "36e04fbb599dce57e2a7d9d105511e731ad0ceafaaf0b0820de7022711066157", + "bytes": 27268 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "2a4f30bad7ae897fed019bc7da330a09965adb35685d11abaeaebf7a1d40aa60", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068 + }, + "linux_package": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934 + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "windows_helper": { + "path": "scripts/gate13_windows_packaged_lifecycle.ps1", + "source_commit": "8c56feced219c521818ef6ad79ec830fc6b30a20", + "sha256": "a85c51eb0231bcef991a77c57b622fad575049f000c8b95bbeec7f78eaec7a1e", + "bytes": 133351 + }, + "linux_helper": { + "path": "scripts/gate13_linux_packaged_lifecycle.py", + "source_commit": "0e16ac2aa088e0719e9c3c89451176544602bab3", + "sha256": "90f3af65bb4f77317f707a6b52e329e1d5f81cdeddcb9615a210ec9a5a4cf535", + "bytes": 113748 + }, + "windows_download_config": { + "path": "scripts/gate13_download_windows.json", + "sha256": "bf36b66bf22a8f4453ba481c5a4ad37d8e1856ad93ef25e5e105b566b39affc4", + "bytes": 293 + }, + "linux_download_config": { + "path": "scripts/gate13_download_linux.json", + "sha256": "c3e9dcd94ef0a8e61c95e650416a8c6ca8b169f4b6699a0acd06608e60da5550", + "bytes": 294 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "0f481d73e712ea14eb022906e2e0debc0a6e9931", + "sha256": "1972d54efe5b9ffb73c3d96e005edc238f7f57aaf720b202d091e541c82e044a", + "bytes": 3371 + }, + "windows_audit_bundle": { + "audit/desktop-metrics.json": { + "sha256": "953fc814d3d7d6787cbe7ecc25e8ab9f60c68515b94c80575207e15e78d69549", + "bytes": 3795 + }, + "audit/provenance.json": { + "sha256": "ac04b71d35493ba4967628af1ac05ca290b1af09aab4e8955ac09031c87ce7f8", + "bytes": 1241883 + }, + "audit/release-metadata.json": { + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "bytes": 872 + }, + "audit/SHA256SUMS": { + "sha256": "a458760c1e4636956c9fba5a7da93ed869544f4817d5ab883b5d8b34cc9ad964", + "bytes": 674380 + } + }, + "linux_audit_bundle": { + "audit/desktop-metrics.json": { + "sha256": "5d2b261505e949a15c332c6e5bb817611e340bf897f9b1952ebde8e573e07bd2", + "bytes": 3798 + }, + "audit/provenance.json": { + "sha256": "c9b5e47017b003f6b2d81c9ab8273fcbf3c72f7743f1ebbf99383ae7cd5accda", + "bytes": 1357051 + }, + "audit/release-metadata.json": { + "sha256": "6a434cf14100572954452052b8a1e6e8565b2930e3251b1b8327cfdcd7383a25", + "bytes": 872 + }, + "audit/SHA256SUMS": { + "sha256": "4766c587a1d8e430f892128b8667c04ae7e83bde869a3098b24012c4f99cb74d", + "bytes": 737970 + } + } + }, + "provider_plan_digest": "sha256:8525c3099f273c099aba26de57c1f610a0c74cac65ed2640589d51e874bd0c44", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260831-i-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [ + 31337, + 31338 + ], + "firewalls": [ + "route-20260831-i-dht", + "route-20260831-i-iap" + ], + "service_account": false, + "scopes": [], + "operator_access_target_tags": [ + "route-20260831-i-node", + "gate13-20260831-i-client" + ], + "enable_virtual_display": true + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260831-i-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-i-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-i-client", + "host_execution": "manual_interactive_console_as_ordinary_user", + "enable_virtual_display": true + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260831-i-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260831-i-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260831-i-client", + "host_execution": "manual_foreground_shell_as_ordinary_user" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": true, + "exact_cleanup_before_pass": true, + "client_host_adapter_mode": "manual_phase_by_phase_before_adapter", + "manual_windows_desktop_required": true, + "automation_prohibited_until_manual_pass": true + } + }, + "preflight": { + "read_only_checked_at": "2026-09-01T02:28:35.230Z", + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "final_software_tests_passed": 22, + "independent_review_matrix_before_final_reservation_guard": 0, + "independent_reservation_guard_finding_resolved": true, + "fresh_native_revalidation_immediately_before_create_required": true + }, + "cleanup": { + "delete_only_exact_run_resources": true, + "verify_absent": [ + "route-20260831-i-node", + "route-20260831-i-node boot disk", + "route-20260831-i-dht", + "route-20260831-i-iap", + "gate13-20260831-i-win", + "gate13-20260831-i-win boot disk", + "gate13-20260831-i-linux", + "gate13-20260831-i-linux boot disk" + ], + "protected_resources": [ + "communityai-bootstrap-1", + "all resources not named by this plan" + ], + "cleanup_required_on_success_or_failure": true + }, + "prohibited": { + "fly_operations": 0, + "model_specific_images_or_mirrors": 0, + "macos_work": 0, + "credits_or_payments_work": 0 + }, + "privacy": { + "credentials_retained": false, + "prompts_retained": false, + "outputs_retained": false, + "endpoints_retained": false, + "provider_output_retained": false, + "private_paths_retained": false + }, + "plan_revision": { + "revision": 1, + "reason": "replace the opaque host lifecycle launch with a literal clean-host playthrough: verify and extract the exact package, run bootstrap and node visibly, launch the Windows desktop in an interactive session, exercise the required controls and inference, restart and exercise the second control, then repeat on Linux; translate only proven commands back into adapters", + "resource_set_changed": true, + "cost_ceiling_changed": false + }, + "manual_execution": { + "windows": [ + "verify exact archive and four audit records", + "extract into an empty per-user install root", + "run packaged self-tests", + "run signed bootstrap with visible output", + "start packaged node directly and inspect visible readiness", + "launch CommunityAI.exe in a real interactive ordinary-user console", + "exercise sharing control and public inference", + "restart the desktop", + "exercise the second control and re-run inference", + "uninstall/reinstall/cache and cleanup phases" + ], + "linux": [ + "repeat the proven phase sequence in one ordinary-user foreground session" + ], + "wrapper_use_before_manual_pass": false, + "retain_private_prompts_or_outputs": false + } +} diff --git a/docs/evidence/gate13-20260831-i-linux-paused.png b/docs/evidence/gate13-20260831-i-linux-paused.png new file mode 100644 index 000000000..19e2ab7b6 Binary files /dev/null and b/docs/evidence/gate13-20260831-i-linux-paused.png differ diff --git a/docs/evidence/gate13-20260831-i-linux-ready.png b/docs/evidence/gate13-20260831-i-linux-ready.png new file mode 100644 index 000000000..627e036a4 Binary files /dev/null and b/docs/evidence/gate13-20260831-i-linux-ready.png differ diff --git a/docs/evidence/gate13-20260831-i-linux-sharing.png b/docs/evidence/gate13-20260831-i-linux-sharing.png new file mode 100644 index 000000000..93cbe6267 Binary files /dev/null and b/docs/evidence/gate13-20260831-i-linux-sharing.png differ diff --git a/docs/evidence/gate13-20260831-i-manual-qualification-and-cleanup.json b/docs/evidence/gate13-20260831-i-manual-qualification-and-cleanup.json new file mode 100644 index 000000000..83c8a4907 --- /dev/null +++ b/docs/evidence/gate13-20260831-i-manual-qualification-and-cleanup.json @@ -0,0 +1,203 @@ +{ + "schema_version": 1, + "scope": "gate13-packaged-clean-install-manual-qualification", + "gate": 13, + "run_id": "gate13-20260831-i", + "result": "passed", + "recorded_at": "2026-09-01T05:25:34.3103224Z", + "goal": "Prove that a normal user can install, open, use, restart, and control the packaged CommunityAI desktop on clean Windows and Linux hosts against the public route.", + "decision": { + "manual_playthrough_is_acceptance_source": true, + "opaque_lifecycle_wrapper_required_for_gate_decision": false, + "reason": "The manual playthrough exercised the actual desktop and exposed the real product defect hidden by the wrappers. Reinstall, uninstall, retained-data choice, and publisher release work remain Gate 15." + }, + "source": { + "published_package_commit": "1971f106cc5bf90724d938c986a719ce2744f3e7", + "windows_path_fix_commit": "f1dc3a0e38b0b2ee12150fe403fd1de435c49f71", + "windows_path_fix": "Use extended-length Windows paths for long manifest artifact partial, final, and lock paths without changing their on-disk layout." + }, + "route": { + "accepted_before_client_creation": true, + "qwen": { + "model": "Qwen3.5 2B", + "manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "blocks": 24, + "peer_count": 1, + "primary_inference_passed": true, + "restored_inference_passed": true + }, + "gemma": { + "model": "Gemma 4 E2B IT", + "manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "blocks": 35, + "peer_count": 1, + "fallback_inference_passed": true + }, + "primary_fallback_restoration_total_duration_ms": 337038, + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained": false + }, + "windows": { + "result": "passed", + "image": "windows-server-2025-dc-v20260814", + "machine_type": "e2-standard-8", + "ordinary_user": true, + "is_admin_during_product_run": false, + "interactive_console_session": true, + "published_archive": { + "sha256": "45e9cdb439bcb8a6d7ed67914a490f3bc2e12ea1981af944034d62d865f5adc6", + "bytes": 2695065068, + "download_verified_before_install": true + }, + "packaged_self_tests": { + "runtime": "passed", + "application": "passed", + "ui": "passed", + "onboarding_ui": "passed" + }, + "manual_flow": [ + "installed the verified archive into an empty ordinary-user install root", + "opened CommunityAI.exe in the real interactive console", + "observed complete Qwen and Gemma routes", + "ran one-token Qwen inference", + "closed the desktop and verified the desktop and node stopped", + "restarted the desktop in the same ordinary-user console", + "edited sharing limits through the UI", + "selected Qwen and clicked Start sharing", + "clicked Pause sharing after restart" + ], + "initial_defect": { + "classification": "product", + "not_gcp": true, + "not_ssh_or_scheduler": true, + "error": "Windows legacy MAX_PATH rejected a manifest artifact lock path under the default per-user data directory.", + "reproduced_with_published_package": true, + "short_data_root_control_inference": { + "passed": true, + "model": "Qwen3.5 2B", + "completion_tokens": 1, + "duration_ms": 239683 + } + }, + "fixed_default_path_inference": { + "passed": true, + "model": "Qwen3.5 2B", + "completion_tokens": 1, + "duration_ms": 204348, + "data_root": "default per-user data root", + "fixed_node_source": "f1dc3a0e38b0b2ee12150fe403fd1de435c49f71", + "prompt_retained": false, + "output_retained": false, + "credential_retained": false + }, + "screenshots": { + "ready": { + "path": "gate13-20260831-i-windows-ready.png", + "sha256": "19926d36d8d9fa1c2a8b36449e891e31dd7dfe244ec85ad8921e8793ea9e8d24" + }, + "paused_after_restart": { + "path": "gate13-20260831-i-windows-paused.png", + "sha256": "f9a1da9a94a6d9d158f0479b8e3b1647b39a7a8aa19f08328c7f533f820463b8" + } + } + }, + "linux": { + "result": "passed", + "image": "ubuntu-2404-noble-amd64-v20260826", + "machine_type": "e2-standard-8", + "ordinary_user": true, + "sudo_available_during_product_run": false, + "display": "Xvfb interactive X11 display", + "native_credential_store": "GNOME Secret Service", + "published_archive": { + "sha256": "f96d3ca651964380d4684855ab08682e8187b33386327ec3895cda25b43c2a00", + "bytes": 3360717934, + "download_verified_before_install": true + }, + "packaged_self_tests": { + "runtime": "passed", + "application": "passed", + "ui": "passed", + "onboarding_ui": "passed" + }, + "manual_flow": [ + "installed the verified archive into an empty ordinary-user install root", + "opened the real Linux desktop on an X11 display", + "observed complete Qwen and Gemma routes", + "ran one-token Gemma inference", + "edited sharing limits through the UI", + "selected Gemma and clicked Start sharing", + "stopped the complete app and node process tree", + "restarted the desktop with the native credential store", + "observed Gemma sharing resume", + "clicked Pause sharing", + "ran one-token Gemma inference again after restart and pause" + ], + "initial_inference": { + "passed": true, + "model": "Gemma 4 E2B IT", + "completion_tokens": 1, + "duration_ms": 197652, + "prompt_retained": false, + "output_retained": false, + "credential_retained": false + }, + "post_restart_inference": { + "passed": true, + "model": "Gemma 4 E2B IT", + "completion_tokens": 1, + "duration_ms": 54352, + "prompt_retained": false, + "output_retained": false, + "credential_retained": false + }, + "screenshots": { + "ready": { + "path": "gate13-20260831-i-linux-ready.png", + "sha256": "1c51f20269a032de813a309e8230abae0d87759e30288d2347c8de46f48d2e6a" + }, + "sharing": { + "path": "gate13-20260831-i-linux-sharing.png", + "sha256": "a0ef7ec97bc186d3a3ecffcb2baf9241074cf382ff2bb441bcf6853da86857fc" + }, + "paused_after_restart": { + "path": "gate13-20260831-i-linux-paused.png", + "sha256": "343abc26d180578ba97723e2f31e1717dbed2eb247b81ae23aaab69f53207a0b" + } + } + }, + "cleanup": { + "exact_instances_absent": [ + "route-20260831-i-node", + "gate13-20260831-i-win", + "gate13-20260831-i-linux" + ], + "exact_disks_absent": [ + "route-20260831-i-node", + "gate13-20260831-i-win", + "gate13-20260831-i-linux" + ], + "exact_firewalls_absent": [ + "route-20260831-i-dht", + "route-20260831-i-iap" + ], + "global_l4_quota": { + "metric": "GPUS_ALL_REGIONS", + "limit": 1, + "usage": 0 + }, + "protected_bootstrap": { + "name": "communityai-bootstrap-1", + "status": "RUNNING" + }, + "passed": true + }, + "privacy": { + "prompts_retained": false, + "outputs_retained": false, + "credentials_retained": false, + "signed_urls_retained": false, + "provider_endpoints_retained": false + } +} diff --git a/docs/evidence/gate13-20260831-i-windows-paused.png b/docs/evidence/gate13-20260831-i-windows-paused.png new file mode 100644 index 000000000..3e9846db3 Binary files /dev/null and b/docs/evidence/gate13-20260831-i-windows-paused.png differ diff --git a/docs/evidence/gate13-20260831-i-windows-ready.png b/docs/evidence/gate13-20260831-i-windows-ready.png new file mode 100644 index 000000000..3c14a3002 Binary files /dev/null and b/docs/evidence/gate13-20260831-i-windows-ready.png differ diff --git a/docs/evidence/gate13-20260901-a-automated-qualification-and-cleanup.json b/docs/evidence/gate13-20260901-a-automated-qualification-and-cleanup.json new file mode 100644 index 000000000..841d622fd --- /dev/null +++ b/docs/evidence/gate13-20260901-a-automated-qualification-and-cleanup.json @@ -0,0 +1,238 @@ +{ + "schema_version": 1, + "scope": "gate13-automated-paid-cloud-qualification-and-cleanup", + "gate": 13, + "run_id": "gate13-20260901-a", + "result": "passed", + "recorded_at": "2026-09-02T08:54:19Z", + "goal": "Replay the successful manual Gate 13 clean-host desktop procedure automatically on Windows and Linux against the paid public route, without manual UI recovery.", + "authorization": { + "evidence": "gate13-20260901-a-cost-authorization.json", + "owner_reset_recorded": true, + "maximum_lifetime_reservation_usd": "56.00", + "maximum_is_not_a_bill_forecast": true, + "cost_note": "The owner reports the comparable real-world replay cost is approximately USD 10. Final provider billing for this run was not available at cleanup time." + }, + "production_packages": { + "requested_source_commit": "e904d36416a4f186c0bec05ff20210df9ca19848", + "workflow_merge_commit": "f83c19997d6180c784e2a85f8d5d68c4361ad99e", + "workflow_merge_parents": [ + "f64a388a47b098ac7f69d2affc59816376b43bb1", + "e904d36416a4f186c0bec05ff20210df9ca19848" + ], + "workflow_run": 33600715239, + "workflow_result": "success", + "style_run": 33600715224, + "style_result": "success", + "test_run": 33600715198, + "test_result": "success", + "windows": { + "workflow_artifact": "communityai-desktop-install-windows", + "artifact_id": 9835635064, + "wrapper_sha256": "sha256:d2e6a90b881838b6f738924c3dc222cb7056bf121dc9ed91c56ea055760aa329", + "wrapper_bytes": 2695087981, + "archive_sha256": "sha256:965c24c3235dd5e4621961376e0d563bb50e81ed214297e2826c0a2454accfe5", + "archive_bytes": 2695087805, + "audit_artifact_id": 9835635695, + "audit_artifact_sha256": "sha256:2007bca3fd77d7debacf2fec7f2975b6ee3e3a7d02f29e8baa7014915d57b407" + }, + "linux": { + "workflow_artifact": "communityai-desktop-install-linux", + "artifact_id": 9835679452, + "wrapper_sha256": "sha256:2712b6adc33f9b932b359afe97764a4361dc6f286342c52b46d09aa536a7389d", + "wrapper_bytes": 3360754507, + "archive_sha256": "sha256:9f7c8629f3f91f1a1b291e2f3f7e1019d1497440ed399b08c837c887f2b0107a", + "archive_bytes": 3360754329, + "audit_artifact_id": 9835680252, + "audit_artifact_sha256": "sha256:dc47d2406b84fba40811b9d2bec44e2e99c18992e049179712ea6bb76485d543" + } + }, + "manual_findings_translated": [ + { + "source": "FLUJO conversation 264a0383-8cbf-4e0a-9073-1ae6072d19fe narration and tool calls", + "finding": "The first inference can return Model unavailable while the exact selected model is still becoming complete.", + "automation": "Poll /v1/models every five seconds for up to 90 seconds, then replay the exact one-token model:auto request once." + }, + { + "source": "FLUJO manual desktop sequence", + "finding": "Sharing controls are page-scoped and the manual run navigated to Sharing before editing policy or clicking master controls.", + "automation": "Open the Sharing page before every sharing action and retain the accessible-name, toggle, selection, legacy, and focus/Enter control fallbacks." + }, + { + "source": "FLUJO sequence 17517-17532", + "finding": "The manual run saved the policy, toggled Share compute with the selected model, clicked Start sharing, observed it, then clicked Pause sharing.", + "automation": "If policy reconciliation already enabled sharing, click the exact checked per-model control to restore the paused baseline before exercising literal Start and Pause." + } + ], + "automation_source": { + "inference_recovery_commit": "05fbe4fd40e1daa9f33fabfe6ca5fedc9a6798d6", + "sharing_page_commit": "984aef348c2b09e1a8383bb74873525d62065db8", + "automatic_sharing_normalization_commit": "e904d36416a4f186c0bec05ff20210df9ca19848", + "route_service_timeout_commit": "b093b850625235b9fa6a10605d15005511ee15f6", + "route_stale_advertisement_commit": "66f440bc02d7920b0a697b4095c243c0ff17ae78", + "fresh_linux_supervisor_commit": "4c6eaca8b0de8d20885c932e5bdcd1f50fc67947", + "automated_playthrough": { + "path": "scripts/gate13_automated_playthrough.py", + "sha256": "sha256:9ffda923a37ef64631898ad82139457a329393f15b78afd45709611f7f4a087f", + "bytes": 20545 + }, + "lifecycle_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "sha256": "sha256:c899ffb162aef49e7dc54c1e62a86505713652e341bd32f77f123e76bde8d1d4", + "bytes": 33278 + }, + "final_host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "sha256": "sha256:79a3220ff06faa1c359feb771903d3d842c004b122244cfef161945403fe91d9", + "bytes": 45781 + }, + "final_route_fence": { + "path": "scripts/gate13_route_fence.py", + "sha256": "sha256:e90f48bbf2e582ef7bae47e71c8e635e69f94dc86c8a7b5197a0d079af1bd5a5", + "bytes": 9907 + } + }, + "prequalification_diagnostics": { + "manual_ui_assistance_accepted_as_qualification": false, + "client_attempts_started_before_final_windows_run": 0, + "client_attempts_started_before_final_linux_run": 0, + "findings": [ + "A Windows diagnostic run proved that clicking master controls while the Sharing page was hidden could wait indefinitely; navigation was added and tested.", + "A later Windows diagnostic showed policy reconciliation can auto-start the selected worker before the automation reaches Start; the exact FLUJO per-model toggle sequence was restored.", + "The first Linux route fence failed closed after an API-side bootstrap startup failure. A later live retry exposed and fixed the bounded service-action and stale-DHT-advertisement cases.", + "The Linux host adapter rejected the fresh host before creating a unit because Ubuntu omits ExecStart for LoadState=not-found; the exact fresh inventory is now accepted while extra fields remain rejected." + ] + }, + "route": { + "instance": "route-20260901-a-node", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "runtime_source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "windows_fence": { + "result": "passed", + "target": "windows", + "model_id": "Qwen3.5 2B", + "manifest_digest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "covered_blocks": 24, + "total_blocks": 24, + "peer_count_minimum": 1, + "stable_rechecks": 2, + "standby_service_stopped": true + }, + "linux_fence": { + "result": "passed", + "target": "linux", + "model_id": "Gemma 4 E2B IT", + "manifest_digest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "covered_blocks": 35, + "total_blocks": 35, + "peer_count_minimum": 1, + "stable_rechecks": 2, + "standby_service_stopped": true + } + }, + "windows": { + "result": "passed", + "attempt_ordinal": 1, + "instance": "gate13-20260901-a-win", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "ordinary_user": "M", + "interactive_console_session": true, + "manual_ui_actions_after_launch": 0, + "evidence_digest": "sha256:764c63b24b0339ded38862d26787b236699ab7c2fc4c15ddaf2717218c4e5adb", + "real_window_sessions": 2, + "localhost_inference_count": 1, + "policy_dialog_saved": true, + "start_clicked": true, + "start_observation_seconds": 25.0, + "pause_control_observed": true, + "pause_clicked": true, + "sharing_intent_paused": true, + "session_duration_seconds": { + "initial": 260.828, + "restart": 66.328 + }, + "qualification_temporaries_removed": true + }, + "linux": { + "result": "passed", + "attempt_ordinal": 1, + "instance": "gate13-20260901-a-linux", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "ordinary_user": "gate13", + "display": "TCP-disabled Xvfb X11 with private D-Bus and native Secret Service", + "manual_ui_actions_after_launch": 0, + "evidence_digest": "sha256:624bcfd01763ac81b1ebfac50aee196988bbbdd30f94610997464faa6dfe637c", + "real_window_sessions": 2, + "localhost_inference_count": 2, + "policy_dialog_saved": true, + "start_clicked": true, + "start_observation_seconds": 20.0, + "restart_resume_observed": true, + "pause_control_observed": true, + "pause_clicked": true, + "sharing_intent_paused": true, + "session_duration_seconds": { + "initial": 229.269578, + "restart": 44.956091 + }, + "qualification_temporaries_removed": true + }, + "validation": { + "desktop_gate13_tests": { + "passed": 215, + "failed": 0 + }, + "route_fence_tests": { + "passed": 5, + "failed": 0 + }, + "host_job_tests": { + "passed": 28, + "failed": 0 + }, + "latest_source_style_run": 33610030286, + "latest_source_style_result": "success", + "latest_source_test_run": 33610030231, + "latest_source_test_result": "success", + "windows_evidence_validator_result": "passed", + "linux_evidence_validator_result": "passed" + }, + "cleanup": { + "exact_instances_absent": [ + "route-20260901-a-node", + "gate13-20260901-a-win", + "gate13-20260901-a-linux" + ], + "exact_disks_absent": [ + "route-20260901-a-node", + "gate13-20260901-a-win", + "gate13-20260901-a-linux" + ], + "exact_firewalls_absent": [ + "route-20260901-a-dht", + "route-20260901-a-iap", + "route-20260901-a-relay" + ], + "regional_l4_quota": { + "metric": "NVIDIA_L4_GPUS", + "limit": 1, + "usage": 0 + }, + "protected_bootstrap": { + "name": "communityai-bootstrap-1", + "status": "RUNNING" + }, + "passed": true + }, + "privacy": { + "prompts_retained": false, + "outputs_retained": false, + "token_identifiers_retained": false, + "credentials_retained": false, + "signed_urls_retained": false, + "provider_endpoints_retained": false + } +} diff --git a/docs/evidence/gate13-20260901-a-cost-authorization.json b/docs/evidence/gate13-20260901-a-cost-authorization.json new file mode 100644 index 000000000..90318ac33 --- /dev/null +++ b/docs/evidence/gate13-20260901-a-cost-authorization.json @@ -0,0 +1,220 @@ +{ + "schema_version": 1, + "scope": "gate13-automated-paid-cloud-authorization", + "gate": 13, + "run_id": "gate13-20260901-a", + "result": "authorized", + "recorded_at": "2026-09-01T20:58:50Z", + "source": { + "durable_controller_commit": "905cabd13c481adff4a5a7de850ceb6ee839a25b", + "host_job_adapter_commit": "d5dc3537eb4a1e405ed9a6bfde0236bda0a58d7f", + "client_session_bootstrap_commit": "cd3e347488c2a79399ba3dd6ce8c31a26ac78ef7", + "route_runtime_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "route_setup_commit": "905cabd13c481adff4a5a7de850ceb6ee839a25b", + "package_commit": "cd3e347488c2a79399ba3dd6ce8c31a26ac78ef7", + "production_workflow_run": 33582031380 + }, + "authorization": { + "owner_reset_recorded": true, + "combined_cloud_ceiling_usd": "100.00", + "ledger_committed_before_run_usd": "0.00", + "maximum_estimate_usd": "56.00", + "route_maximum_estimate_usd": "30.00", + "clients_maximum_estimate_usd": "26.00", + "remaining_after_run_maximum_usd": "44.00", + "reservation_recorded": true, + "provisioning_authorized_after_fail_closed_preflight": true, + "provider_calls_authorized_without_preflight": false, + "pricing_as_of": "2026-09-01", + "pricing_basis": "The owner reports the prior real-world replay cost was approximately USD 10. USD 56 is retained only as a fail-safe maximum-lifetime reservation, not as a bill forecast." + }, + "immutable_inputs": { + "durable_controller": { + "path": "scripts/gate13_run_controller.py", + "source_commit": "905cabd13c481adff4a5a7de850ceb6ee839a25b", + "sha256": "5ab9bf2c49267e3b2b7cefcb2d2c9b1eb331a652970e3345ab443f09db4cf8aa", + "bytes": 34281 + }, + "host_job_adapter": { + "path": "scripts/gate13_host_job.py", + "source_commit": "d5dc3537eb4a1e405ed9a6bfde0236bda0a58d7f", + "sha256": "3fd232fa291c849fb45b0432b556bf01a6cb22650d7911389eb533e2f95ce3ff", + "bytes": 45573 + }, + "windows_client_startup": { + "path": "scripts/gate13_windows_client_startup.ps1", + "source_commit": "e60c3577c7205ff434cad6e9396f89555626aceb", + "sha256": "3f8600c42a3c0765e100963c2e28cdef7c6b248992924ff3406941aefce7cf47", + "bytes": 8779 + }, + "linux_client_startup": { + "path": "scripts/gate13_linux_client_startup.sh", + "source_commit": "d5dc3537eb4a1e405ed9a6bfde0236bda0a58d7f", + "sha256": "72ac32fb78946ac09b60bbef571a944a018d790871fafbb818ec7006bee292c6", + "bytes": 2138 + }, + "lifecycle_evidence_validator": { + "path": "scripts/gate13_packaged_lifecycle.py", + "source_commit": "cd3e347488c2a79399ba3dd6ce8c31a26ac78ef7", + "sha256": "c899ffb162aef49e7dc54c1e62a86505713652e341bd32f77f123e76bde8d1d4", + "bytes": 33278 + }, + "automated_playthrough": { + "path": "scripts/gate13_automated_playthrough.py", + "source_commit": "cd3e347488c2a79399ba3dd6ce8c31a26ac78ef7", + "sha256": "9ffda923a37ef64631898ad82139457a329393f15b78afd45709611f7f4a087f", + "bytes": 20545 + }, + "route_client_fence": { + "path": "scripts/gate13_route_fence.py", + "source_commit": "238122692655c083d534f2a8359635f6588931e7", + "sha256": "6d42e80a30aaacd3f7b80c89be15af435ab6427b1c62a338cadf11ac32237772", + "bytes": 9367 + }, + "route_setup": { + "path": "scripts/gate13_route_setup.sh", + "source_commit": "905cabd13c481adff4a5a7de850ceb6ee839a25b", + "sha256": "045372ea0be9c4a8f31756a502b2a9ec799087eeaac294ebad2b34eccfe0affc", + "bytes": 3371 + }, + "route_runtime_wheel": { + "filename": "drift-2.3.0.dev2-py3-none-any.whl", + "source_commit": "f64a388a47b098ac7f69d2affc59816376b43bb1", + "sha256": "7a42803811289e14f69835331e0fbab69dd353c70c835131c10bdfa96ca5f111", + "bytes": 389107, + "model_artifacts_embedded": false + }, + "windows_package": { + "sha256": "127ea96d5eafa908aa6221e11e86af1c05e4183e5cef05696a0a58ea381ebbc0", + "bytes": 2695083895, + "workflow_artifact": "communityai-desktop-install-windows" + }, + "linux_package": { + "sha256": "9791ffa6d3cfa86ef8aabdec518918cef65a961eb69439880306880b741cfe20", + "bytes": 3360741913, + "workflow_artifact": "communityai-desktop-install-linux" + }, + "qwen_manifest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "gemma_manifest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd" + }, + "provider_plan_digest": "sha256:3fd1e3907d431222a876fa192e8dbc08611dfcfad46cd5c865110eccf01fe547", + "provider_plan": { + "project": "community-ai-506321", + "route": { + "instance": "route-20260901-a-node", + "zone": "us-central1-b", + "machine_type": "g2-standard-8", + "accelerator": "1 x NVIDIA L4", + "image": "deeplearning-platform-release/common-cu129-ubuntu-2404-nvidia-580-v20260831", + "boot_disk_gib": 200, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 57600, + "termination_action": "DELETE", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "public_ports": [31337, 31338], + "firewalls": ["route-20260901-a-dht", "route-20260901-a-iap"], + "service_account": false, + "scopes": [] + }, + "clients": [ + { + "platform": "windows", + "model": "Qwen3.5 2B", + "instance": "gate13-20260901-a-win", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "windows-server-2025-dc-v20260814", + "image_project": "windows-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260901-a-windows", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260901-a-client", + "host_execution": "automated_real_window_foreground_as_ordinary_user" + }, + { + "platform": "linux", + "model": "Gemma 4 E2B IT", + "instance": "gate13-20260901-a-linux", + "zone": "us-central1-b", + "machine_type": "e2-standard-8", + "image": "ubuntu-2404-noble-amd64-v20260826", + "image_project": "ubuntu-os-cloud", + "boot_disk_gib": 120, + "boot_disk_type": "pd-balanced", + "boot_disk_auto_delete": true, + "max_run_seconds": 21600, + "termination_action": "DELETE", + "lifecycle_run_id": "gate13-20260901-a-linux", + "device_profile": "cpu", + "network": "communityai-discovery", + "subnet": "communityai-us-central1", + "network_tag": "gate13-20260901-a-client", + "host_execution": "automated_real_window_dbus_session_as_ordinary_user" + } + ], + "sequencing": { + "route_live_for_both_lifecycles": true, + "clients_may_run_concurrently": false, + "route_accepted_before_any_client_create": true, + "route_fenced_and_revalidated_for_each_client": true, + "windows_collected_and_deleted_before_linux_create": true, + "all_16_phases_required_per_platform": false, + "automated_gate13_replay_required": true, + "exact_cleanup_before_pass": true + } + }, + "preflight": { + "native_gcloud_token_refresh": true, + "compute_api_access": true, + "exact_target_instances_absent": true, + "exact_target_disks_absent": true, + "exact_target_firewalls_absent": true, + "machine_types_available": true, + "images_available": true, + "one_l4_quota_free": true, + "protected_bootstrap_running": true, + "production_packages_passed": true + }, + "prohibited": { + "fly_resources": 0, + "macos_hosts": 0, + "service_accounts": 0, + "client_gpu_instances": 0, + "concurrent_clients": 0, + "credit_operations": 0 + }, + "privacy": { + "retain_prompts": false, + "retain_outputs": false, + "retain_credentials": false, + "retain_signed_urls": false, + "retain_provider_endpoints": false + }, + "completion": { + "result": "passed", + "recorded_at": "2026-09-02T08:54:19Z", + "qualification_evidence": "gate13-20260901-a-automated-qualification-and-cleanup.json", + "package_source_commit": "e904d36416a4f186c0bec05ff20210df9ca19848", + "final_route_fence_commit": "66f440bc02d7920b0a697b4095c243c0ff17ae78", + "final_host_job_adapter_commit": "4c6eaca8b0de8d20885c932e5bdcd1f50fc67947", + "windows_attempt_ordinal": 1, + "linux_attempt_ordinal": 1, + "manual_ui_recovery_accepted": false, + "reservation_state": "cleaned-committed-pending-provider-billing", + "provider_billing_available": false, + "cost_note": "The USD 56 value was a maximum-lifetime fail-safe reservation, not an estimate of the provider bill. The owner reports comparable real-world use at approximately USD 10.", + "exact_run_instances_absent": true, + "exact_run_disks_absent": true, + "exact_run_firewalls_absent": true, + "regional_l4_usage": 0, + "protected_bootstrap_running": true + } +} diff --git a/docs/evidence/owner-budget-authorization-20260831.json b/docs/evidence/owner-budget-authorization-20260831.json new file mode 100644 index 000000000..5bccd5661 --- /dev/null +++ b/docs/evidence/owner-budget-authorization-20260831.json @@ -0,0 +1,39 @@ +{ + "schema_version": 1, + "scope": "combined-cloud-budget-epoch-authorization", + "recorded_at": "2026-08-31", + "timezone": "America/Bogota", + "result": "authorized", + "owner_decision": { + "combined_cloud_ceiling_usd": "500.00", + "authorized_on": "2026-08-31", + "providers": [ + "GCP", + "Fly" + ], + "purpose": "CommunityAI public inference alpha critical-path infrastructure and qualification" + }, + "epoch_state_at_authorization": { + "prior_ceiling_usd": "100.00", + "committed_maximum_usd": "52.00", + "unreserved_maximum_usd": "448.00", + "committed_run": "gate13-20260831-a" + }, + "controls_unchanged": { + "exact_source_bound_authorization_per_paid_run": true, + "fresh_native_auth_and_fail_closed_preflight_before_create": true, + "exact_run_scoped_resources_only": true, + "cleanup_and_absence_proof_required": true, + "protected_bootstrap_must_remain": "communityai-bootstrap-1", + "observed_cost_tracking_required": true + }, + "prohibited": { + "credits_or_payments_work": true, + "macos_work": true + }, + "claims": { + "specific_provider_run_authorized_by_this_record": false, + "budget_is_observed_spend": false, + "prior_committed_maximum_released": false + } +} diff --git a/scripts/gate13_automated_playthrough.py b/scripts/gate13_automated_playthrough.py new file mode 100644 index 000000000..6cc100b82 --- /dev/null +++ b/scripts/gate13_automated_playthrough.py @@ -0,0 +1,552 @@ +"""Run the proven Gate 13 desktop playthrough without operator UI actions. + +Invoke this only after a production archive has been verified and unpacked on a +clean host. The frozen desktop opens its real window twice and replays the exact +platform-specific chronology accepted in the manual Gate 13 run. Windows performs +default-root inference before a full restart, then saves policy, starts, observes for +25 seconds, and pauses. Linux performs inference/policy/start before the restart, +then proves persisted intent, pauses, and performs post-restart inference. + +The script prints one bounded aggregate record. Private per-session files live +only in an exact run-scoped temporary root and are removed before success. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import shutil +import stat +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +SCHEMA_VERSION = 2 +SCOPE = "gate13-automated-desktop-replay" +POLICY_PROFILE = "gate13-manual-cpu-v1" +SEQUENCE_PROFILES = { + "windows": "gate13-manual-windows-v1", + "linux": "gate13-manual-linux-v1", +} +MAX_CONFIG_BYTES = 65_536 +MAX_EVIDENCE_BYTES = 65_536 + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_MODEL_RE = re.compile(r"[ -~]{1,128}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_CONFIG_FIELDS = { + "schema_version", + "run_id", + "platform", + "source_commit", + "package_archive", + "package_sha256", + "package_bytes", + "desktop_executable", + "work_root", + "model_id", + "manifest_digest", + "total_blocks", + "policy", + "session_timeout_seconds", + "inference_timeout_seconds", +} +_POLICY_FIELDS = { + "sharing_enabled", + "allowed_models", + "preferred_models", + "denied_models", + "max_disk_space", + "max_vram", + "max_bandwidth_mbps", + "max_power_watts", + "pause_timeout", + "schedule", +} +_SESSION_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "stage", + "result", + "model_id", + "manifest_digest", + "duration_seconds", + "route", + "inference", + "ui", + "limits", + "timing", + "privacy", +} + + +def _manual_schedule() -> dict[str, Any]: + return { + "timezone": "UTC", + "windows": [ + { + "days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"], + "start": "00:00", + "end": "23:59", + } + ], + } + + +class ReplayError(ValueError): + """A replay input, session, or cleanup boundary failed closed.""" + + +def _reject_constant(_value: str) -> None: + raise ReplayError("JSON contains a non-finite value") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ReplayError("JSON contains a duplicate field") + value[key] = item + return value + + +def _regular_bytes(path: Path, maximum: int) -> bytes: + _regular_metadata(path, maximum) + path = Path(path) + try: + return path.read_bytes() + except OSError as exc: + raise ReplayError("required file is unreadable") from exc + + +def _regular_metadata(path: Path, maximum: int) -> os.stat_result: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise ReplayError("required file is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= maximum: + raise ReplayError("required file is not a bounded regular file") + return metadata + + +def _json_file(path: Path, maximum: int) -> Mapping[str, Any]: + try: + value = json.loads( + _regular_bytes(path, maximum).decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ReplayError("JSON is invalid") from exc + if not isinstance(value, dict): + raise ReplayError("JSON root is invalid") + return value + + +def _number(value: Any, label: str, minimum: float, maximum: float) -> float: + if type(value) not in (int, float): + raise ReplayError(f"{label} is invalid") + rendered = float(value) + if not math.isfinite(rendered) or not minimum <= rendered <= maximum: + raise ReplayError(f"{label} is invalid") + return rendered + + +def _absolute_path(value: Any, label: str) -> Path: + if not isinstance(value, str) or not value or "\x00" in value: + raise ReplayError(f"{label} is invalid") + path = Path(value) + if not path.is_absolute(): + raise ReplayError(f"{label} must be absolute") + return path + + +@dataclass(frozen=True) +class ReplayConfig: + run_id: str + platform: str + source_commit: str + package_archive: Path + package_sha256: str + package_bytes: int + desktop_executable: Path + work_root: Path + model_id: str + manifest_digest: str + total_blocks: int + policy: Mapping[str, Any] + session_timeout_seconds: float + inference_timeout_seconds: float + + +def load_config(path: Path) -> ReplayConfig: + raw = _json_file(path, MAX_CONFIG_BYTES) + if set(raw) != _CONFIG_FIELDS or raw.get("schema_version") != SCHEMA_VERSION: + raise ReplayError("configuration schema is invalid") + run_id = raw["run_id"] + platform = raw["platform"] + source_commit = raw["source_commit"] + package_sha256 = raw["package_sha256"] + package_bytes = raw["package_bytes"] + model_id = raw["model_id"] + digest = raw["manifest_digest"] + blocks = raw["total_blocks"] + policy = raw["policy"] + if not isinstance(run_id, str) or _RUN_RE.fullmatch(run_id) is None: + raise ReplayError("run id is invalid") + if platform not in ("windows", "linux"): + raise ReplayError("platform is invalid") + if not isinstance(source_commit, str) or _COMMIT_RE.fullmatch(source_commit) is None: + raise ReplayError("source commit is invalid") + if not isinstance(package_sha256, str) or _DIGEST_RE.fullmatch(package_sha256) is None: + raise ReplayError("package digest is invalid") + if type(package_bytes) is not int or not 1 <= package_bytes <= 8 * 1024**3: + raise ReplayError("package size is invalid") + if not isinstance(model_id, str) or _MODEL_RE.fullmatch(model_id) is None or model_id != model_id.strip(): + raise ReplayError("model id is invalid") + if not isinstance(digest, str) or _DIGEST_RE.fullmatch(digest) is None: + raise ReplayError("manifest digest is invalid") + if type(blocks) is not int or not 1 <= blocks <= 512: + raise ReplayError("block count is invalid") + if not isinstance(policy, dict) or set(policy) != _POLICY_FIELDS: + raise ReplayError("policy schema is invalid") + if ( + policy["sharing_enabled"] is not True + or policy["allowed_models"] != [model_id] + or policy["preferred_models"] != [model_id] + or policy["denied_models"] != [] + or policy["schedule"] != _manual_schedule() + ): + raise ReplayError("policy does not match the proven manual replay") + if policy["max_disk_space"] != "32GB": + raise ReplayError("storage ceiling does not match the proven manual replay") + if policy["max_vram"] != "20GB": + raise ReplayError("memory ceiling does not match the proven manual replay") + if _number(policy["max_bandwidth_mbps"], "bandwidth ceiling", 0.001, 1_000_000) != 100.0: + raise ReplayError("bandwidth ceiling does not match the proven manual replay") + if policy["max_power_watts"] is not None: + raise ReplayError("the manual CPU-host replay requires an unset power ceiling") + if _number(policy["pause_timeout"], "pause timeout", 1, 300) != 120.0: + raise ReplayError("pause timeout does not match the proven manual replay") + executable = _absolute_path(raw["desktop_executable"], "desktop executable") + _regular_metadata(executable, 2 * 1024**3) + package_archive = _absolute_path(raw["package_archive"], "package archive") + if _regular_metadata(package_archive, 8 * 1024**3).st_size != package_bytes: + raise ReplayError("package size changed") + work_root = _absolute_path(raw["work_root"], "work root") + if work_root.name != f".gate13-playthrough-{run_id}" or work_root.exists() or not work_root.parent.is_dir(): + raise ReplayError("work root is not a fresh exact run root") + return ReplayConfig( + run_id=run_id, + platform=platform, + source_commit=source_commit, + package_archive=package_archive, + package_sha256=package_sha256, + package_bytes=package_bytes, + desktop_executable=executable, + work_root=work_root, + model_id=model_id, + manifest_digest=digest, + total_blocks=blocks, + policy=policy, + session_timeout_seconds=_number(raw["session_timeout_seconds"], "session timeout", 30, 3_600), + inference_timeout_seconds=_number(raw["inference_timeout_seconds"], "inference timeout", 10, 600), + ) + + +def _session_plan(config: ReplayConfig, stage: str) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "run_id": config.run_id, + "platform": config.platform, + "stage": stage, + "model_id": config.model_id, + "manifest_digest": config.manifest_digest, + "total_blocks": config.total_blocks, + "policy": dict(config.policy), + "timeout_seconds": config.session_timeout_seconds, + "inference_timeout_seconds": config.inference_timeout_seconds, + } + + +def _write_private_json(path: Path, value: Mapping[str, Any]) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_BINARY"): + flags |= os.O_BINARY + descriptor = os.open(path, flags, 0o600) + try: + payload = (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + with os.fdopen(descriptor, "wb", closefd=False) as output: + output.write(payload) + output.flush() + os.fsync(output.fileno()) + finally: + os.close(descriptor) + + +def _validate_session(path: Path, config: ReplayConfig, stage: str) -> Mapping[str, Any]: + value = _json_file(path, MAX_EVIDENCE_BYTES) + if set(value) != _SESSION_FIELDS: + raise ReplayError("session evidence schema is invalid") + if ( + value["schema_version"] != SCHEMA_VERSION + or value["scope"] != "gate13-packaged-desktop-playthrough" + or value["run_id"] != config.run_id + or value["platform"] != config.platform + or value["stage"] != stage + or value["result"] != "passed" + or value["model_id"] != config.model_id + or value["manifest_digest"] != config.manifest_digest + ): + raise ReplayError("session evidence identity is invalid") + _number(value["duration_seconds"], "session duration", 0, config.session_timeout_seconds + 30) + route = value["route"] + inference = value["inference"] + ui = value["ui"] + limits = value["limits"] + privacy = value["privacy"] + if route != { + "rendered_in_real_window": True, + "complete": True, + "covered_blocks": config.total_blocks, + "total_blocks": config.total_blocks, + }: + raise ReplayError("session route evidence is invalid") + inference_required = (config.platform, stage) in { + ("windows", "initial"), + ("linux", "initial"), + ("linux", "restart"), + } + if inference_required: + if ( + not isinstance(inference, dict) + or inference.get("passed") is not True + or inference.get("model_id") != config.model_id + or inference.get("manifest_digest") != config.manifest_digest + or inference.get("completion_count") != 1 + or inference.get("generated_token_count") != 1 + or inference.get("response_content_retained") is not False + or inference.get("token_identifiers_retained") is not False + or inference.get("temporary_key_removed") is not True + ): + raise ReplayError("session inference evidence is invalid") + elif inference is not None: + raise ReplayError("unexpected session inference evidence") + policy_session = (config.platform, stage) in { + ("windows", "restart"), + ("linux", "initial"), + } + start_session = policy_session + pause_session = stage == "restart" + resumed_session = config.platform == "linux" and stage == "restart" + expected_ui = { + "real_window_opened": True, + "policy_dialog_saved": policy_session, + "start_clicked": start_session, + "pause_control_observed": start_session or resumed_session, + "pause_clicked": pause_session, + "restart_resume_observed": resumed_session, + "sharing_intent_enabled_observed": start_session or resumed_session, + "sharing_intent_disabled_observed": pause_session, + } + expected_limits = { + "storage": policy_session, + "memory_or_vram": policy_session, + "bandwidth": policy_session, + "power": False, + "pause_timeout": policy_session, + "schedule": policy_session, + } + expected_privacy = { + "prompt_retained": False, + "response_content_retained": False, + "token_identifiers_retained": False, + "credentials_retained": False, + "paths_retained": False, + "endpoints_retained": False, + } + expected_timing = { + "start_observation_seconds": 25.0 + if config.platform == "windows" and stage == "restart" + else (20.0 if config.platform == "linux" and stage == "initial" else 0.0), + "restart_observation_seconds": 15.0 if resumed_session else 0.0, + } + if ui != expected_ui or limits != expected_limits or value["timing"] != expected_timing: + raise ReplayError("session UI or limit evidence is invalid") + if privacy != expected_privacy: + raise ReplayError("session privacy evidence is invalid") + forbidden = ("prompt", "response", "secret", "credential", "endpoint", "path", "address") + rendered = json.dumps(value, sort_keys=True).lower() + for field in forbidden: + if f'"{field}"' in rendered: + raise ReplayError("session evidence retained a forbidden field") + return value + + +def _run_session( + config: ReplayConfig, + stage: str, + runner: Callable[..., subprocess.CompletedProcess] = subprocess.run, +) -> Mapping[str, Any]: + plan_path = config.work_root / f"{stage}-plan.json" + evidence_path = config.work_root / f"{stage}-evidence.json" + _write_private_json(plan_path, _session_plan(config, stage)) + try: + result = runner( + [ + os.fspath(config.desktop_executable), + "--gate13-ui-playthrough", + os.fspath(plan_path), + "--gate13-ui-evidence", + os.fspath(evidence_path), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=config.session_timeout_seconds + 60, + close_fds=True, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ReplayError("packaged desktop session failed") from exc + if result.returncode != 0: + raise ReplayError("packaged desktop session failed") + return _validate_session(evidence_path, config, stage) + + +def _digest_file(path: Path) -> str: + _regular_metadata(path, 8 * 1024**3) + digest = hashlib.sha256() + try: + with path.open("rb") as source: + while chunk := source.read(1024 * 1024): + digest.update(chunk) + except OSError as exc: + raise ReplayError("package archive could not be hashed") from exc + return "sha256:" + digest.hexdigest() + + +def _run_package_self_tests( + config: ReplayConfig, + runner: Callable[..., subprocess.CompletedProcess], +) -> None: + for action in ("--check-runtime", "--self-test", "--ui-self-test", "--onboarding-ui-self-test"): + try: + result = runner( + [os.fspath(config.desktop_executable), action], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=120, + close_fds=True, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ReplayError("packaged desktop self-test failed") from exc + if result.returncode != 0: + raise ReplayError("packaged desktop self-test failed") + + +def run_replay( + config: ReplayConfig, + *, + runner: Callable[..., subprocess.CompletedProcess] = subprocess.run, +) -> Mapping[str, Any]: + package_digest = _digest_file(config.package_archive) + executable_digest = _digest_file(config.desktop_executable) + if package_digest != config.package_sha256: + raise ReplayError("package digest changed") + _run_package_self_tests(config, runner) + config.work_root.mkdir(mode=0o700) + cleanup_passed = False + start: Mapping[str, Any] | None = None + resumed: Mapping[str, Any] | None = None + try: + start = _run_session(config, "initial", runner) + resumed = _run_session(config, "restart", runner) + finally: + try: + resolved = config.work_root.resolve(strict=True) + parent = config.work_root.parent.resolve(strict=True) + if resolved.parent != parent or resolved.name != f".gate13-playthrough-{config.run_id}": + raise ReplayError("work-root cleanup target changed") + shutil.rmtree(resolved) + cleanup_passed = not config.work_root.exists() + except OSError as exc: + raise ReplayError("qualification temporary cleanup failed") from exc + if start is None or resumed is None or not cleanup_passed: + raise ReplayError("automated replay did not complete") + if ( + _digest_file(config.package_archive) != package_digest + or _digest_file(config.desktop_executable) != executable_digest + ): + raise ReplayError("package inputs changed during the replay") + return { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "run_id": config.run_id, + "platform": config.platform, + "result": "passed", + "source_commit": config.source_commit, + "package": { + "sha256": config.package_sha256, + "bytes": config.package_bytes, + "verified_before_run": True, + "self_test_count": 4, + }, + "model_id": config.model_id, + "manifest_digest": config.manifest_digest, + "real_window_sessions": 2, + "localhost_inference_count": 1 if config.platform == "windows" else 2, + "policy_dialog_saved": True, + "start_clicked": True, + "pause_control_observed": True, + "restart_resume_observed": config.platform == "linux", + "pause_clicked": True, + "sharing_intent_paused": True, + "policy_profile": POLICY_PROFILE, + "sequence_profile": SEQUENCE_PROFILES[config.platform], + "start_observation_seconds": 25.0 if config.platform == "windows" else 20.0, + "session_duration_seconds": { + "initial": start["duration_seconds"], + "restart": resumed["duration_seconds"], + }, + "privacy_safe": True, + "qualification_temporaries_removed": True, + } + + +def _failure() -> Mapping[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "result": "failed", + "failure_code": "automated_replay_failed", + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Run the automated Gate 13 packaged desktop replay") + parser.add_argument("--config", type=Path, required=True) + args = parser.parse_args(argv) + try: + value = run_replay(load_config(args.config)) + except BaseException: + value = _failure() + print(json.dumps(value, sort_keys=True, separators=(",", ":"))) + return 0 if value.get("result") == "passed" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gate13_host_job.py b/scripts/gate13_host_job.py new file mode 100644 index 000000000..3368d7f0b --- /dev/null +++ b/scripts/gate13_host_job.py @@ -0,0 +1,1276 @@ +"""Durable native host-job adapter for Gate 13 packaged lifecycle runs. + +A paid client attempt is launched exactly once under a native supervisor. The adapter +persists bounded status before starting the lifecycle, validates the canonical evidence, +and writes a digest-only terminal record. Re-entry never relaunches an attempt. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import signal +import stat +import subprocess +import sys +import tempfile +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +import gate13_packaged_lifecycle as lifecycle + +SCHEMA_VERSION = 1 +MAX_CONFIG_BYTES = 65_536 +MAX_STATE_BYTES = 262_144 +MAX_EVIDENCE_BYTES = lifecycle.MAX_INPUT_BYTES +MAX_STDERR_BYTES = 262_144 +MAX_SCRIPT_BYTES = 8 * 1024 * 1024 +MIN_RUN_SECONDS = 300 +MAX_RUN_SECONDS = 21_600 +SUPERVISOR_GRACE_SECONDS = 60 +READ_CHUNK_BYTES = 65_536 +POSIX_SIGTERM = getattr(signal, "SIGTERM", 15) +POSIX_SIGKILL = getattr(signal, "SIGKILL", 9) + +WINDOWS_RUNTIME_ENVIRONMENT = ( + "ALLUSERSPROFILE", + "APPDATA", + "COMMONPROGRAMFILES", + "COMMONPROGRAMFILES(X86)", + "COMMONPROGRAMW6432", + "COMSPEC", + "HOMEDRIVE", + "HOMEPATH", + "LOCALAPPDATA", + "OS", + "PATH", + "PATHEXT", + "PROGRAMDATA", + "PROGRAMFILES", + "PROGRAMFILES(X86)", + "PROGRAMW6432", + "PUBLIC", + "SYSTEMDRIVE", + "SYSTEMROOT", + "TEMP", + "TMP", + "USERDOMAIN", + "USERNAME", + "USERPROFILE", + "WINDIR", +) +LINUX_RUNTIME_ENVIRONMENT = ( + "DBUS_SESSION_BUS_ADDRESS", + "DISPLAY", + "GNOME_KEYRING_CONTROL", + "HOME", + "LANG", + "LC_ALL", + "QT_QPA_PLATFORM", + "TMPDIR", + "XAUTHORITY", + "XDG_RUNTIME_DIR", +) + +HOST_ROOTS = { + "windows": Path(r"C:\Gate13Run"), + "linux": Path("/qualification"), +} +HOST_PYTHON = { + "windows": Path(r"C:\Gate13Python\python.exe"), + "linux": Path("/usr/bin/python3"), +} +ADAPTER_PATH = Path(__file__).resolve() + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_USER_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"(?:sha256:)?[0-9a-f]{64}") +_JOB_RE = re.compile(r"communityai-gate13-[a-z0-9-]{1,63}-(?:windows|linux)") +_CONFIG_FIELDS = { + "schema_version", + "run_id", + "lifecycle_run_id", + "platform", + "attempt_ordinal", + "source_commit", + "job_name", + "host_user", + "adapter_path", + "adapter_sha256", + "config_path", + "entrypoint_path", + "entrypoint_sha256", + "lifecycle_config_path", + "lifecycle_config_sha256", + "evidence_path", + "stderr_path", + "status_path", + "terminal_path", + "working_directory", + "python_executable", + "max_run_seconds", +} +_STATUS_FIELDS = { + "schema_version", + "run_id", + "platform", + "attempt_ordinal", + "state", + "started_at_unix", +} +_TERMINAL_FIELDS = { + "schema_version", + "run_id", + "platform", + "attempt_ordinal", + "result", + "failure_code", + "evidence_digest", + "exit_code", + "finished_at_unix", +} +_NATIVE_FIELDS = {"native_state", "binding_ok"} + + +class HostJobError(ValueError): + """The host job config, state, or native supervisor failed closed.""" + + +@dataclass(frozen=True) +class HostJobConfig: + run_id: str + lifecycle_run_id: str + platform: str + attempt_ordinal: int + source_commit: str + job_name: str + host_user: str + adapter_path: Path + adapter_sha256: str + config_path: Path + entrypoint_path: Path + entrypoint_sha256: str + lifecycle_config_path: Path + lifecycle_config_sha256: str + evidence_path: Path + stderr_path: Path + status_path: Path + terminal_path: Path + working_directory: Path + python_executable: Path + max_run_seconds: int + + +Runner = Callable[..., subprocess.CompletedProcess[str]] + + +def _reject_constant(_value: str) -> None: + raise HostJobError("invalid JSON") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise HostJobError("duplicate JSON field") + result[key] = value + return result + + +def _regular_bytes(path: Path, maximum: int, *, allow_empty: bool = False) -> bytes: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise HostJobError("required file is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + minimum = 0 if allow_empty else 1 + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not minimum <= metadata.st_size <= maximum: + raise HostJobError("required file is unsafe") + try: + return path.read_bytes() + except OSError as exc: + raise HostJobError("required file is unreadable") from exc + + +def _strict_json(payload: bytes, maximum: int) -> Mapping[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= maximum: + raise HostJobError("JSON size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise HostJobError("invalid JSON") from exc + if not isinstance(value, dict): + raise HostJobError("JSON root is invalid") + return value + + +def _exact_mapping(value: Mapping[str, Any], fields: set[str], label: str) -> Mapping[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise HostJobError(f"{label} schema is invalid") + return value + + +def _string(value: Any, pattern: re.Pattern[str], label: str) -> str: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise HostJobError(f"{label} is invalid") + return value + + +def _integer(value: Any, label: str, minimum: int, maximum: int) -> int: + if type(value) is not int or not minimum <= value <= maximum: + raise HostJobError(f"{label} is invalid") + return value + + +def _digest_bytes(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _digest_file(path: Path) -> str: + return _digest_bytes(_regular_bytes(path, MAX_SCRIPT_BYTES)) + + +def _normalized_path(value: Any, label: str) -> Path: + if not isinstance(value, str) or not value or "\x00" in value: + raise HostJobError(f"{label} is invalid") + path = Path(value) + if not path.is_absolute(): + raise HostJobError(f"{label} is not absolute") + return Path(os.path.abspath(os.fspath(path))) + + +def _same_path(left: Path, right: Path) -> bool: + return os.path.normcase(os.path.abspath(os.fspath(left))) == os.path.normcase(os.path.abspath(os.fspath(right))) + + +def _inside(path: Path, root: Path) -> bool: + try: + return os.path.commonpath( + [os.path.normcase(os.path.abspath(os.fspath(path))), os.path.normcase(os.path.abspath(os.fspath(root)))] + ) == os.path.normcase(os.path.abspath(os.fspath(root))) + except ValueError: + return False + + +def _safe_existing_output(path: Path) -> None: + if not path.exists(): + return + metadata = path.lstat() + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise HostJobError("output path is unsafe") + + +def load_config(path: Path) -> HostJobConfig: + config_path = Path(os.path.abspath(os.fspath(path))) + raw = _exact_mapping( + _strict_json(_regular_bytes(config_path, MAX_CONFIG_BYTES), MAX_CONFIG_BYTES), + _CONFIG_FIELDS, + "config", + ) + if raw["schema_version"] != SCHEMA_VERSION: + raise HostJobError("config version is invalid") + platform = raw["platform"] + if platform not in HOST_ROOTS: + raise HostJobError("platform is invalid") + run_id = _string(raw["run_id"], _RUN_RE, "run id") + lifecycle_run_id = raw["lifecycle_run_id"] + if lifecycle_run_id != f"{run_id}-{platform}": + raise HostJobError("lifecycle run id is invalid") + attempt = _integer(raw["attempt_ordinal"], "attempt ordinal", 1, 1) + source_commit = _string(raw["source_commit"], _COMMIT_RE, "source commit") + job_name = _string(raw["job_name"], _JOB_RE, "job name") + if job_name != f"communityai-gate13-{run_id}-{platform}": + raise HostJobError("job name is not source-bound") + host_user = _string(raw["host_user"], _USER_RE, "host user") + if (platform == "linux" and host_user != "gate13") or host_user.casefold() in { + "system", + "local service", + "network service", + "administrator", + "root", + }: + raise HostJobError("host user is not an ordinary qualification user") + + values = { + field: _normalized_path(raw[field], field) + for field in ( + "adapter_path", + "config_path", + "entrypoint_path", + "lifecycle_config_path", + "evidence_path", + "stderr_path", + "status_path", + "terminal_path", + "working_directory", + "python_executable", + ) + } + root = Path(os.path.abspath(os.fspath(HOST_ROOTS[platform]))) + if not _same_path(values["working_directory"], root): + raise HostJobError("working directory changed") + for field in ( + "adapter_path", + "config_path", + "entrypoint_path", + "lifecycle_config_path", + "evidence_path", + "stderr_path", + "status_path", + "terminal_path", + ): + if not _inside(values[field], root): + raise HostJobError(f"{field} escapes the host root") + if not _same_path(config_path, values["config_path"]): + raise HostJobError("config path binding changed") + expected_lifecycle_name = "gate13-windows-run.json" if platform == "windows" else "gate13-linux-run.json" + if values["lifecycle_config_path"].name != expected_lifecycle_name: + raise HostJobError("lifecycle config path changed") + if platform == "windows" and not _same_path( + values["lifecycle_config_path"], + values["entrypoint_path"].parent / expected_lifecycle_name, + ): + raise HostJobError("Windows lifecycle config is not beside its entrypoint") + entrypoint_suffix = values["entrypoint_path"].suffix.casefold() + if (platform == "windows" and entrypoint_suffix not in {".ps1", ".py"}) or ( + platform == "linux" and entrypoint_suffix != ".py" + ): + raise HostJobError("entrypoint type is invalid") + if not _same_path(values["python_executable"], HOST_PYTHON[platform]): + raise HostJobError("Python executable changed") + if not _same_path(values["adapter_path"], ADAPTER_PATH): + raise HostJobError("adapter invocation changed") + root_metadata = root.lstat() + root_reparse = bool( + getattr(root_metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + ) + if not root.is_dir() or root.is_symlink() or root_reparse: + raise HostJobError("host root is unsafe") + + outputs = [ + values["evidence_path"], + values["stderr_path"], + values["status_path"], + values["terminal_path"], + ] + bound_paths = [ + values["adapter_path"], + values["config_path"], + values["entrypoint_path"], + values["lifecycle_config_path"], + *outputs, + ] + if len({os.path.normcase(os.fspath(item)) for item in bound_paths}) != len(bound_paths): + raise HostJobError("bound paths overlap") + for output in outputs: + _safe_existing_output(output) + + adapter_sha = _string(raw["adapter_sha256"], _DIGEST_RE, "adapter digest") + entrypoint_sha = _string(raw["entrypoint_sha256"], _DIGEST_RE, "entrypoint digest") + lifecycle_config_sha = _string( + raw["lifecycle_config_sha256"], + _DIGEST_RE, + "lifecycle config digest", + ) + if _digest_file(values["adapter_path"]) != "sha256:" + adapter_sha.removeprefix("sha256:"): + raise HostJobError("adapter digest changed") + if _digest_file(values["entrypoint_path"]) != "sha256:" + entrypoint_sha.removeprefix("sha256:"): + raise HostJobError("entrypoint digest changed") + if _digest_file(values["lifecycle_config_path"]) != "sha256:" + lifecycle_config_sha.removeprefix("sha256:"): + raise HostJobError("lifecycle config digest changed") + + return HostJobConfig( + run_id=run_id, + lifecycle_run_id=lifecycle_run_id, + platform=platform, + attempt_ordinal=attempt, + source_commit=source_commit, + job_name=job_name, + host_user=host_user, + adapter_path=values["adapter_path"], + adapter_sha256="sha256:" + adapter_sha.removeprefix("sha256:"), + config_path=values["config_path"], + entrypoint_path=values["entrypoint_path"], + entrypoint_sha256="sha256:" + entrypoint_sha.removeprefix("sha256:"), + lifecycle_config_path=values["lifecycle_config_path"], + lifecycle_config_sha256="sha256:" + lifecycle_config_sha.removeprefix("sha256:"), + evidence_path=values["evidence_path"], + stderr_path=values["stderr_path"], + status_path=values["status_path"], + terminal_path=values["terminal_path"], + working_directory=values["working_directory"], + python_executable=values["python_executable"], + max_run_seconds=_integer( + raw["max_run_seconds"], + "maximum run seconds", + MIN_RUN_SECONDS, + MAX_RUN_SECONDS, + ), + ) + + +def _atomic_json(path: Path, value: Mapping[str, Any], *, exclusive: bool = False) -> None: + payload = (json.dumps(value, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n").encode("utf-8") + if not 1 <= len(payload) <= MAX_STATE_BYTES: + raise HostJobError("state is too large") + if exclusive: + try: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError as exc: + raise HostJobError("state already exists") from exc + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + return + + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + try: + os.chmod(temporary, 0o600) + except OSError: + pass + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _load_status(config: HostJobConfig) -> Mapping[str, Any] | None: + if not config.status_path.exists(): + return None + raw = _exact_mapping( + _strict_json( + _regular_bytes(config.status_path, MAX_STATE_BYTES), + MAX_STATE_BYTES, + ), + _STATUS_FIELDS, + "status", + ) + if ( + raw["schema_version"] != SCHEMA_VERSION + or raw["run_id"] != config.run_id + or raw["platform"] != config.platform + or raw["attempt_ordinal"] != config.attempt_ordinal + or raw["state"] != "running" + or type(raw["started_at_unix"]) is not int + ): + raise HostJobError("status binding changed") + return raw + + +def _load_terminal(config: HostJobConfig) -> Mapping[str, Any] | None: + if not config.terminal_path.exists(): + return None + raw = _exact_mapping( + _strict_json( + _regular_bytes(config.terminal_path, MAX_STATE_BYTES), + MAX_STATE_BYTES, + ), + _TERMINAL_FIELDS, + "terminal", + ) + if ( + raw["schema_version"] != SCHEMA_VERSION + or raw["run_id"] != config.run_id + or raw["platform"] != config.platform + or raw["attempt_ordinal"] != config.attempt_ordinal + or raw["result"] not in {"passed", "failed"} + or (raw["failure_code"] is not None and not re.fullmatch(r"[a-z0-9_]{1,64}", str(raw["failure_code"]))) + or type(raw["exit_code"]) is not int + or type(raw["finished_at_unix"]) is not int + ): + raise HostJobError("terminal binding changed") + digest = raw["evidence_digest"] + if raw["result"] == "passed": + _string(digest, _DIGEST_RE, "terminal evidence digest") + if raw["failure_code"] is not None or raw["exit_code"] != 0: + raise HostJobError("terminal success is inconsistent") + elif digest is not None or raw["failure_code"] is None: + raise HostJobError("terminal failure is inconsistent") + return raw + + +def _terminal( + config: HostJobConfig, + *, + result: str, + failure_code: str | None, + evidence_digest: str | None, + exit_code: int, + finished_at_unix: int, +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "run_id": config.run_id, + "platform": config.platform, + "attempt_ordinal": config.attempt_ordinal, + "result": result, + "failure_code": failure_code, + "evidence_digest": evidence_digest, + "exit_code": exit_code, + "finished_at_unix": finished_at_unix, + } + + +def _entrypoint_argv(config: HostJobConfig) -> list[str]: + if config.platform == "windows" and config.entrypoint_path.suffix.casefold() == ".ps1": + return [ + r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe", + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + os.fspath(config.entrypoint_path), + ] + return [ + os.fspath(config.python_executable), + os.fspath(config.entrypoint_path), + "--config", + os.fspath(config.lifecycle_config_path), + ] + + +def _bounded_environment(config: HostJobConfig) -> dict[str, str]: + allowed = WINDOWS_RUNTIME_ENVIRONMENT if config.platform == "windows" else LINUX_RUNTIME_ENVIRONMENT + return {key: os.environ[key] for key in allowed if key in os.environ} + + +def _bounded_copy( + stream: Any, + destination: Path, + maximum: int, + overflow: threading.Event, + errors: list[BaseException], +) -> None: + total = 0 + try: + with destination.open("xb") as output: + while True: + chunk = stream.read(READ_CHUNK_BYTES) + if not chunk: + break + if not isinstance(chunk, bytes): + raise HostJobError("child output type is invalid") + remaining = max(0, maximum - total) + if remaining: + output.write(chunk[:remaining]) + total += len(chunk) + if total > maximum: + overflow.set() + output.flush() + os.fsync(output.fileno()) + except BaseException as exc: + errors.append(exc) + overflow.set() + finally: + try: + stream.close() + except BaseException: + pass + + +def _wait_for_exit(process: Any, timeout: float) -> bool: + try: + process.wait(timeout=timeout) + return True + except subprocess.TimeoutExpired: + return False + + +def _stop_process_tree(config: HostJobConfig, process: Any) -> None: + if config.platform == "windows": + try: + process.send_signal(signal.CTRL_BREAK_EVENT) + except (OSError, ValueError, AttributeError): + pass + if _wait_for_exit(process, SUPERVISOR_GRACE_SECONDS): + return + try: + subprocess.run( + [ + r"C:\Windows\System32\taskkill.exe", + "/PID", + str(process.pid), + "/T", + "/F", + ], + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + shell=False, + ) + except (OSError, subprocess.TimeoutExpired): + pass + else: + try: + os.killpg(process.pid, POSIX_SIGTERM) + except (OSError, AttributeError): + try: + process.terminate() + except OSError: + pass + if _wait_for_exit(process, SUPERVISOR_GRACE_SECONDS): + return + try: + os.killpg(process.pid, POSIX_SIGKILL) + except (OSError, AttributeError): + try: + process.kill() + except OSError: + pass + if not _wait_for_exit(process, 30): + raise HostJobError("entrypoint process tree did not stop") + + +def _run_entrypoint(config: HostJobConfig) -> int: + if config.evidence_path.exists() or config.stderr_path.exists(): + raise HostJobError("attempt output already exists") + creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if config.platform == "windows" else 0 + process = subprocess.Popen( + _entrypoint_argv(config), + cwd=config.working_directory, + env=_bounded_environment(config), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + bufsize=0, + start_new_session=config.platform == "linux", + creationflags=creationflags, + ) + if process.stdout is None or process.stderr is None: + _stop_process_tree(config, process) + raise HostJobError("entrypoint pipes are unavailable") + + overflow = threading.Event() + copy_errors: list[BaseException] = [] + threads = [ + threading.Thread( + target=_bounded_copy, + args=(process.stdout, config.evidence_path, MAX_EVIDENCE_BYTES, overflow, copy_errors), + daemon=True, + ), + threading.Thread( + target=_bounded_copy, + args=(process.stderr, config.stderr_path, MAX_STDERR_BYTES, overflow, copy_errors), + daemon=True, + ), + ] + for thread in threads: + thread.start() + + deadline = time.monotonic() + config.max_run_seconds + stop_code: int | None = None + while process.poll() is None: + if overflow.is_set(): + stop_code = 126 + break + if time.monotonic() >= deadline: + stop_code = 124 + break + time.sleep(0.05) + if stop_code is not None: + _stop_process_tree(config, process) + + for thread in threads: + thread.join(SUPERVISOR_GRACE_SECONDS) + if any(thread.is_alive() for thread in threads): + _stop_process_tree(config, process) + raise HostJobError("entrypoint output streams did not close") + if copy_errors: + raise HostJobError("entrypoint output could not be bounded") + if stop_code is not None: + return stop_code + if overflow.is_set(): + return 126 + return int(process.returncode) + + +def _validate_evidence(config: HostJobConfig) -> tuple[bytes, str]: + payload = _regular_bytes(config.evidence_path, MAX_EVIDENCE_BYTES) + try: + document = lifecycle.load_lifecycle_json(payload.decode("utf-8")) + summary = lifecycle.validate_lifecycle_document(document) + except Exception as exc: + raise HostJobError("lifecycle evidence is invalid") from exc + if ( + summary.get("run_id") != config.lifecycle_run_id + or summary.get("platform") != config.platform + or summary.get("source_commit") != config.source_commit + ): + raise HostJobError("lifecycle evidence binding changed") + return payload, _digest_bytes(payload) + + +def execute( + config_path: Path, + *, + clock: Callable[[], float] = time.time, + entrypoint_runner: Callable[[HostJobConfig], int] = _run_entrypoint, +) -> Mapping[str, Any]: + config = load_config(config_path) + existing_terminal = _load_terminal(config) + if existing_terminal is not None: + return existing_terminal + if _load_status(config) is not None: + raise HostJobError("attempt was already started") + + _atomic_json( + config.status_path, + { + "schema_version": SCHEMA_VERSION, + "run_id": config.run_id, + "platform": config.platform, + "attempt_ordinal": config.attempt_ordinal, + "state": "running", + "started_at_unix": int(clock()), + }, + exclusive=True, + ) + + exit_code = 125 + failure_code: str | None = "host_job_failed" + evidence_digest: str | None = None + result = "failed" + try: + exit_code = int(entrypoint_runner(config)) + if exit_code == 0: + _payload, evidence_digest = _validate_evidence(config) + result = "passed" + failure_code = None + elif exit_code == 124: + failure_code = "host_job_timed_out" + elif exit_code == 126: + failure_code = "host_job_output_exceeded" + else: + failure_code = "lifecycle_failed" + except Exception: + failure_code = "invalid_lifecycle_evidence" if exit_code == 0 else "host_job_failed" + result = "failed" + evidence_digest = None + + terminal = _terminal( + config, + result=result, + failure_code=failure_code, + evidence_digest=evidence_digest, + exit_code=exit_code, + finished_at_unix=int(clock()), + ) + _atomic_json(config.terminal_path, terminal, exclusive=True) + return terminal + + +def _native_snapshot(value: Mapping[str, Any]) -> Mapping[str, Any]: + raw = _exact_mapping(value, _NATIVE_FIELDS, "native snapshot") + if raw["native_state"] not in {"absent", "starting", "running", "inactive"}: + raise HostJobError("native state is invalid") + if type(raw["binding_ok"]) is not bool: + raise HostJobError("native binding is invalid") + if raw["native_state"] == "absent" and raw["binding_ok"]: + raise HostJobError("absent native job has a binding") + return raw + + +def observe_job(config: HostJobConfig, native: Mapping[str, Any]) -> dict[str, Any]: + snapshot = _native_snapshot(native) + terminal = _load_terminal(config) + status = _load_status(config) + if not snapshot["binding_ok"] and snapshot["native_state"] != "absent": + return {"job_state": "ambiguous", "attempt_ordinal": 1, "evidence_digest": None} + if terminal is not None: + return { + "job_state": "passed" if terminal["result"] == "passed" else "failed", + "attempt_ordinal": config.attempt_ordinal, + "evidence_digest": terminal["evidence_digest"], + } + if status is not None: + state = snapshot["native_state"] + return { + "job_state": "running" if state in {"starting", "running"} else "ambiguous", + "attempt_ordinal": config.attempt_ordinal, + "evidence_digest": None, + } + if snapshot["native_state"] == "absent": + return {"job_state": "absent", "attempt_ordinal": 0, "evidence_digest": None} + if snapshot["binding_ok"] and snapshot["native_state"] in {"starting", "running"}: + return {"job_state": "starting", "attempt_ordinal": 1, "evidence_digest": None} + return {"job_state": "ambiguous", "attempt_ordinal": 1, "evidence_digest": None} + + +def collect(config_path: Path) -> bytes: + config = load_config(config_path) + terminal = _load_terminal(config) + if terminal is None or terminal["result"] != "passed": + raise HostJobError("successful terminal record is absent") + payload, digest = _validate_evidence(config) + if digest != terminal["evidence_digest"]: + raise HostJobError("evidence digest changed") + return payload + + +def _default_runner( + argv: Sequence[str], + *, + timeout: int = 60, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + list(argv), + check=False, + capture_output=True, + text=True, + timeout=timeout, + shell=False, + ) + + +def _powershell_argv(script: str) -> list[str]: + import base64 + + encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii") + return [ + r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe", + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-EncodedCommand", + encoded, + ] + + +def _ps_quote(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _windows_action_arguments(config: HostJobConfig) -> str: + return f'"{config.adapter_path}" execute --config ' f'"{config.config_path}"' + + +def _windows_register_script(config: HostJobConfig) -> str: + task_path = "\\" + return "\n".join( + [ + "$ErrorActionPreference = 'Stop'", + f"$taskPath = {_ps_quote(task_path)}", + f"$taskName = {_ps_quote(config.job_name)}", + "$identity = [Security.Principal.WindowsIdentity]::GetCurrent()", + "$operator = [Security.Principal.WindowsPrincipal]::new($identity)", + "if (-not $operator.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { throw 'privileged task registration required' }", + f"$targetUser = Get-LocalUser -Name {_ps_quote(config.host_user)} -ErrorAction Stop", + "$targetAccount = [Security.Principal.NTAccount]::new($env:COMPUTERNAME, [string]$targetUser.Name)", + "$targetSid = $targetAccount.Translate([Security.Principal.SecurityIdentifier]).Value", + "$existing = Get-ScheduledTask -TaskPath $taskPath -TaskName $taskName -ErrorAction SilentlyContinue", + "if ($null -ne $existing) { throw 'exact task already exists' }", + ( + "$action = New-ScheduledTaskAction " + f"-Execute {_ps_quote(os.fspath(config.python_executable))} " + f"-Argument {_ps_quote(_windows_action_arguments(config))}" + ), + ( + "$principal = New-ScheduledTaskPrincipal -UserId $targetAccount.Value " + "-LogonType Interactive -RunLevel Limited" + ), + ( + "$settings = New-ScheduledTaskSettingsSet -MultipleInstances IgnoreNew " + f"-ExecutionTimeLimit (New-TimeSpan -Seconds {config.max_run_seconds + 2 * SUPERVISOR_GRACE_SECONDS})" + ), + ( + "Register-ScheduledTask -TaskPath $taskPath -TaskName $taskName " + "-Action $action -Principal $principal -Settings $settings | Out-Null" + ), + "Start-ScheduledTask -TaskPath $taskPath -TaskName $taskName", + ] + ) + + +def _windows_snapshot_script(config: HostJobConfig) -> str: + task_path = "\\" + arguments = _windows_action_arguments(config) + return "\n".join( + [ + "$ErrorActionPreference = 'Stop'", + f"$taskPath = {_ps_quote(task_path)}", + f"$taskName = {_ps_quote(config.job_name)}", + "$task = Get-ScheduledTask -TaskPath $taskPath -TaskName $taskName -ErrorAction SilentlyContinue", + "if ($null -eq $task) {", + " [pscustomobject]@{ native_state = 'absent'; binding_ok = $false } | ConvertTo-Json -Compress", + " exit 0", + "}", + "$identity = [Security.Principal.WindowsIdentity]::GetCurrent()", + "$operator = [Security.Principal.WindowsPrincipal]::new($identity)", + f"$targetUser = Get-LocalUser -Name {_ps_quote(config.host_user)} -ErrorAction Stop", + "$targetAccount = [Security.Principal.NTAccount]::new($env:COMPUTERNAME, [string]$targetUser.Name)", + "$targetSid = $targetAccount.Translate([Security.Principal.SecurityIdentifier]).Value", + "$taskSid = ''", + "try {", + " $taskAccount = [Security.Principal.NTAccount]::new([string]$task.Principal.UserId)", + " $taskSid = $taskAccount.Translate([Security.Principal.SecurityIdentifier]).Value", + "} catch {", + " $taskSid = ''", + "}", + "$action = @($task.Actions)[0]", + f"$expectedLimit = [Xml.XmlConvert]::ToString([TimeSpan]::FromSeconds({config.max_run_seconds + 2 * SUPERVISOR_GRACE_SECONDS}))", + ( + "$binding = (@($task.Actions).Count -eq 1) -and " + f"($action.Execute -eq {_ps_quote(os.fspath(config.python_executable))}) -and " + f"($action.Arguments -eq {_ps_quote(arguments)}) -and " + "($operator.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) -and " + "($taskSid -eq $targetSid) -and " + "($task.Principal.LogonType -eq 'Interactive') -and " + "($task.Principal.RunLevel -eq 'Limited') -and " + "($task.Settings.MultipleInstances -eq 'IgnoreNew') -and " + "($task.Settings.ExecutionTimeLimit -eq $expectedLimit)" + ), + "$state = if ($task.State -eq 'Running') { 'running' } elseif ($task.State -eq 'Queued') { 'starting' } else { 'inactive' }", + "[pscustomobject]@{ native_state = $state; binding_ok = [bool]$binding } | ConvertTo-Json -Compress", + ] + ) + + +def _parse_json_stdout(result: subprocess.CompletedProcess[str]) -> Mapping[str, Any]: + if result.returncode != 0 or len(result.stdout.encode("utf-8")) > 32_768: + raise HostJobError("native supervisor inventory failed") + return _strict_json(result.stdout.encode("utf-8"), 32_768) + + +def _windows_snapshot(config: HostJobConfig, runner: Runner) -> Mapping[str, Any]: + result = runner(_powershell_argv(_windows_snapshot_script(config)), timeout=60) + return _native_snapshot(_parse_json_stdout(result)) + + +def _linux_service(config: HostJobConfig) -> str: + return config.job_name + ".service" + + +def _linux_start_argv(config: HostJobConfig) -> list[str]: + return [ + "sudo", + "-n", + "/usr/bin/systemd-run", + "--quiet", + "--collect", + "--service-type=exec", + "--unit", + config.job_name, + f"--property=User={config.host_user}", + f"--property=Group={config.host_user}", + f"--property=WorkingDirectory={config.working_directory}", + "--property=Restart=no", + "--property=KillMode=control-group", + "--property=UMask=0077", + "--property=NoNewPrivileges=no", + "--property=PrivateTmp=no", + "--property=TimeoutStartSec=120", + f"--property=RuntimeMaxSec={config.max_run_seconds + 2 * SUPERVISOR_GRACE_SECONDS}", + "--setenv=DISPLAY=:99", + "--setenv=HOME=/home/gate13", + "--setenv=XDG_RUNTIME_DIR=/qualification/runtime", + "/usr/bin/dbus-run-session", + os.fspath(config.python_executable), + os.fspath(config.adapter_path), + "execute-linux-desktop-session", + "--config", + os.fspath(config.config_path), + ] + + +def _parse_systemd_seconds(value: str) -> float: + if value == "0": + return 0.0 + factors = { + "us": 0.000001, + "ms": 0.001, + "s": 1.0, + "min": 60.0, + "h": 3600.0, + "d": 86_400.0, + } + parts = re.findall(r"(\d+(?:\.\d+)?)(us|ms|s|min|h|d)", value) + compact = re.sub(r"\s+", "", value) + if not parts or "".join(number + unit for number, unit in parts) != compact: + raise HostJobError("native supervisor duration is invalid") + return sum(float(number) * factors[unit] for number, unit in parts) + + +def _systemd_exec_start_matches(config: HostJobConfig, value: str) -> bool: + if not value or any(character in value for character in ("\r", "\n", "\x00")): + return False + normalized = " ".join(value.split()) + matched = re.fullmatch( + ( + r"\{ path=(?P\S+) ; argv\[\]=(?P[^;]+) ; " + r"ignore_errors=(?Pyes|no) ; " + r"start_time=\[[^\]]+\] ; stop_time=\[[^\]]+\] ; " + r"pid=(?P\d+) ; code=(?P\(null\)|[a-z-]+) ; " + r"status=(?P[A-Za-z0-9()/+.-]+) \}" + ), + normalized, + ) + if matched is None: + return False + expected_argv = ( + f"/usr/bin/dbus-run-session {config.python_executable} {config.adapter_path} " + f"execute-linux-desktop-session --config {config.config_path}" + ) + return ( + matched["path"] == "/usr/bin/dbus-run-session" + and matched["argv"] == expected_argv + and matched["ignore"] == "no" + ) + + +def _systemd_environment_matches(value: str) -> bool: + assignments = re.findall(r'(?:^|\s)(?:"([^"\\]*(?:\\.[^"\\]*)*)"|(\S+))', value) + rendered = {quoted or plain for quoted, plain in assignments} + return rendered == { + "DISPLAY=:99", + "HOME=/home/gate13", + "XDG_RUNTIME_DIR=/qualification/runtime", + } + + +def _linux_snapshot(config: HostJobConfig, runner: Runner) -> Mapping[str, Any]: + argv = [ + "sudo", + "-n", + "/usr/bin/systemctl", + "show", + _linux_service(config), + "--no-pager", + "--property=LoadState", + "--property=ActiveState", + "--property=SubState", + "--property=User", + "--property=Group", + "--property=ExecStart", + "--property=WorkingDirectory", + "--property=Restart", + "--property=KillMode", + "--property=UMask", + "--property=NoNewPrivileges", + "--property=PrivateTmp", + "--property=Environment", + "--property=TimeoutStartUSec", + "--property=RuntimeMaxUSec", + ] + result = runner(argv, timeout=60) + if result.returncode != 0: + raise HostJobError("native supervisor inventory failed") + if len(result.stdout.encode("utf-8")) > 32_768: + raise HostJobError("native supervisor inventory is too large") + fields: dict[str, str] = {} + for line in result.stdout.splitlines(): + key, separator, value = line.partition("=") + if not separator or key in fields: + raise HostJobError("native supervisor inventory is invalid") + fields[key] = value + expected_fields = { + "LoadState", + "ActiveState", + "SubState", + "User", + "Group", + "ExecStart", + "WorkingDirectory", + "Restart", + "KillMode", + "UMask", + "NoNewPrivileges", + "PrivateTmp", + "Environment", + "TimeoutStartUSec", + "RuntimeMaxUSec", + } + if fields.get("LoadState") == "not-found": + absent_field_sets = (expected_fields, expected_fields - {"ExecStart"}) + if set(fields) not in absent_field_sets: + raise HostJobError("native supervisor inventory is incomplete") + return {"native_state": "absent", "binding_ok": False} + if set(fields) != expected_fields: + raise HostJobError("native supervisor inventory is incomplete") + binding = ( + fields["LoadState"] == "loaded" + and fields["User"] == config.host_user + and fields["Group"] == config.host_user + and _systemd_exec_start_matches(config, fields["ExecStart"]) + and fields["WorkingDirectory"] == os.fspath(config.working_directory) + and fields["Restart"] == "no" + and fields["KillMode"] == "control-group" + and fields["UMask"] == "0077" + and fields["NoNewPrivileges"] == "no" + and fields["PrivateTmp"] == "no" + and _systemd_environment_matches(fields["Environment"]) + and _parse_systemd_seconds(fields["TimeoutStartUSec"]) == 120.0 + and _parse_systemd_seconds(fields["RuntimeMaxUSec"]) == config.max_run_seconds + 2 * SUPERVISOR_GRACE_SECONDS + ) + if fields["ActiveState"] in {"activating", "reloading"}: + native_state = "starting" + elif fields["ActiveState"] == "active": + native_state = "running" + else: + native_state = "inactive" + return {"native_state": native_state, "binding_ok": binding} + + +def native_snapshot(config: HostJobConfig, runner: Runner = _default_runner) -> Mapping[str, Any]: + return _windows_snapshot(config, runner) if config.platform == "windows" else _linux_snapshot(config, runner) + + +def start(config_path: Path, runner: Runner = _default_runner) -> Mapping[str, Any]: + config = load_config(config_path) + current = observe_job(config, native_snapshot(config, runner)) + if current["job_state"] != "absent" or current["attempt_ordinal"] != 0: + return current + + if config.platform == "windows": + result = runner(_powershell_argv(_windows_register_script(config)), timeout=60) + else: + result = runner(_linux_start_argv(config), timeout=60) + if result.returncode != 0: + raise HostJobError("native supervisor start failed") + observed = observe_job(config, native_snapshot(config, runner)) + if observed["job_state"] == "absent": + raise HostJobError("native supervisor start was not durable") + return observed + + +def cleanup(config_path: Path, runner: Runner = _default_runner) -> Mapping[str, Any]: + config = load_config(config_path) + snapshot = native_snapshot(config, runner) + if snapshot["native_state"] == "absent": + return snapshot + if not snapshot["binding_ok"]: + raise HostJobError("refusing to remove foreign exact-name job") + if config.platform == "windows": + task_path = "\\" + script = "\n".join( + [ + "$ErrorActionPreference = 'Stop'", + f"$taskPath = {_ps_quote(task_path)}", + f"$taskName = {_ps_quote(config.job_name)}", + "Stop-ScheduledTask -TaskPath $taskPath -TaskName $taskName -ErrorAction SilentlyContinue", + "Unregister-ScheduledTask -TaskPath $taskPath -TaskName $taskName -Confirm:$false", + ] + ) + result = runner(_powershell_argv(script), timeout=60) + else: + result = runner( + [ + "sudo", + "-n", + "/usr/bin/systemctl", + "stop", + _linux_service(config), + ], + timeout=60, + ) + if result.returncode != 0: + raise HostJobError("native supervisor cleanup failed") + final = native_snapshot(config, runner) + if final["native_state"] != "absent": + raise HostJobError("native supervisor cleanup is incomplete") + return final + + +def _render(value: Mapping[str, Any]) -> str: + return json.dumps(value, allow_nan=False, separators=(",", ":"), sort_keys=True) + + +def _execute_linux_desktop_session(config_path: Path) -> Mapping[str, Any]: + if not sys.platform.startswith("linux"): + raise HostJobError("Linux desktop session used on another platform") + expected = { + "DISPLAY": ":99", + "HOME": "/home/gate13", + "XDG_RUNTIME_DIR": "/qualification/runtime", + } + if any(os.environ.get(key) != value for key, value in expected.items()): + raise HostJobError("Linux desktop environment is not bound") + if not os.environ.get("DBUS_SESSION_BUS_ADDRESS"): + raise HostJobError("Linux D-Bus session is absent") + try: + keyring = subprocess.run( + ["/usr/bin/gnome-keyring-daemon", "--unlock", "--components=secrets"], + input="\n", + check=False, + capture_output=True, + text=True, + timeout=30, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise HostJobError("Linux Secret Service could not start") from exc + if keyring.returncode != 0 or len(keyring.stdout.encode("utf-8")) > 32_768: + raise HostJobError("Linux Secret Service could not start") + for line in keyring.stdout.splitlines(): + name, separator, value = line.partition("=") + if not separator or name not in {"GNOME_KEYRING_CONTROL", "SSH_AUTH_SOCK"} or not value: + raise HostJobError("Linux Secret Service environment is invalid") + os.environ[name] = value + return execute(config_path) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument( + "action", + choices=("start", "status", "execute", "execute-linux-desktop-session", "collect", "cleanup"), + ) + parser.add_argument("--config", required=True) + try: + arguments = parser.parse_args(sys.argv[1:] if argv is None else argv) + config_path = Path(arguments.config) + if arguments.action == "start": + print(_render(start(config_path))) + elif arguments.action == "status": + config = load_config(config_path) + print(_render(observe_job(config, native_snapshot(config)))) + elif arguments.action == "execute": + terminal = execute(config_path) + print(_render(terminal)) + return 0 if terminal["result"] == "passed" else 2 + elif arguments.action == "execute-linux-desktop-session": + terminal = _execute_linux_desktop_session(config_path) + print(_render(terminal)) + return 0 if terminal["result"] == "passed" else 2 + elif arguments.action == "collect": + sys.stdout.buffer.write(collect(config_path)) + else: + print(_render(cleanup(config_path))) + return 0 + except (Exception, SystemExit): + print( + _render( + { + "failure_code": "host_job_rejected", + "result": "failed", + "schema_version": SCHEMA_VERSION, + } + ) + ) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate13_linux_client_startup.sh b/scripts/gate13_linux_client_startup.sh new file mode 100644 index 000000000..1ad596295 --- /dev/null +++ b/scripts/gate13_linux_client_startup.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +metadata_root=http://metadata.google.internal/computeMetadata/v1/instance/attributes +metadata() { + curl -fsS -H 'Metadata-Flavor: Google' "$metadata_root/$1" +} + +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y -qq \ + python3 xvfb xauth x11-utils xdotool imagemagick dbus-x11 \ + gnome-keyring libsecret-tools libsecret-1-0 libdbus-1-3 \ + libxcb-cursor0 libxcb-icccm4 libxcb-keysyms1 libxcb-shape0 \ + libxkbcommon0 libxkbcommon-x11-0 libegl1 libgl1 libpulse0 libfontconfig1 \ + unzip curl + +if ! id gate13 >/dev/null 2>&1; then + useradd --create-home --home-dir /home/gate13 --shell /bin/bash gate13 +fi + +run_root=/qualification +download_root=/var/tmp/gate13-download +install -d -m 0700 "$download_root" +install -d -o gate13 -g gate13 -m 0700 \ + "$run_root" "$run_root/package" "$run_root/install" "$run_root/runtime" + +package_url="$(metadata package-url)" +package_sha256="$(metadata package-sha256)" +package_bytes="$(metadata package-bytes)" +wrapper="$download_root/artifact.zip" +curl -fL --retry 4 --retry-delay 5 "$package_url" -o "$wrapper" +unzip -q "$wrapper" -d "$download_root/artifact" +archive="$download_root/artifact/communityai-desktop-linux.tar.gz" +test "$(stat -c %s "$archive")" = "$package_bytes" +test "$(sha256sum "$archive" | cut -d' ' -f1)" = "$package_sha256" +mv "$archive" "$run_root/package/communityai-desktop-linux.tar.gz" +tar -xzf "$run_root/package/communityai-desktop-linux.tar.gz" -C "$run_root/install" +rm -rf "$wrapper" "$download_root/artifact" +chown -R gate13:gate13 "$run_root" + +systemd-run --quiet --collect --service-type=exec \ + --unit=communityai-gate13-display \ + --property=User=gate13 \ + --property=Group=gate13 \ + --property=Restart=no \ + --property=KillMode=control-group \ + --property=UMask=0077 \ + --property=RuntimeMaxSec=21600 \ + /usr/bin/Xvfb :99 -screen 0 1280x900x24 -nolisten tcp +for _ in $(seq 1 30); do + if sudo -u gate13 env DISPLAY=:99 xdpyinfo >/dev/null 2>&1; then + touch /var/lib/gate13-bootstrap-ready + exit 0 + fi + sleep 1 +done +echo "Gate 13 X display did not become ready" >&2 +exit 1 diff --git a/scripts/gate13_linux_packaged_lifecycle.py b/scripts/gate13_linux_packaged_lifecycle.py index 1a6e7a18b..ad2d3695d 100644 --- a/scripts/gate13_linux_packaged_lifecycle.py +++ b/scripts/gate13_linux_packaged_lifecycle.py @@ -2712,27 +2712,37 @@ def run_from_config(path: str) -> Mapping[str, Any]: raise LifecycleRunError("lifecycle cleanup was not proved") +def _termination_requested(_signum: int, _frame: Any) -> None: + raise LifecycleRunError("lifecycle termination was requested") + + def main(argv: Sequence[str] | None = None) -> int: arguments = list(sys.argv[1:] if argv is None else argv) os.umask(0o077) + previous_sigterm = signal.signal(signal.SIGTERM, _termination_requested) + previous_sigint = signal.signal(signal.SIGINT, _termination_requested) try: - _disable_core_dumps() - if len(arguments) != 2 or arguments[0] != "--config": - raise LifecycleRunError("exactly one config path is required") - document = run_from_config(arguments[1]) - except BaseException: - print( - _canonical_json( - { - "failure_code": "linux_lifecycle_failed", - "result": "failed", - "schema_version": SCHEMA_VERSION, - } + try: + _disable_core_dumps() + if len(arguments) != 2 or arguments[0] != "--config": + raise LifecycleRunError("exactly one config path is required") + document = run_from_config(arguments[1]) + except BaseException: + print( + _canonical_json( + { + "failure_code": "linux_lifecycle_failed", + "result": "failed", + "schema_version": SCHEMA_VERSION, + } + ) ) - ) - return 2 - print(_canonical_json(document)) - return 0 + return 2 + print(_canonical_json(document)) + return 0 + finally: + signal.signal(signal.SIGTERM, previous_sigterm) + signal.signal(signal.SIGINT, previous_sigint) if __name__ == "__main__": diff --git a/scripts/gate13_packaged_lifecycle.py b/scripts/gate13_packaged_lifecycle.py index 2bf2c09b7..1e27d0716 100644 --- a/scripts/gate13_packaged_lifecycle.py +++ b/scripts/gate13_packaged_lifecycle.py @@ -18,6 +18,13 @@ SCHEMA_VERSION = 1 SCOPE = "gate13-packaged-lifecycle" +AUTOMATED_REPLAY_SCOPE = "gate13-automated-desktop-replay" +AUTOMATED_REPLAY_SCHEMA_VERSION = 2 +AUTOMATED_REPLAY_POLICY_PROFILE = "gate13-manual-cpu-v1" +AUTOMATED_REPLAY_SEQUENCE_PROFILES = { + "windows": "gate13-manual-windows-v1", + "linux": "gate13-manual-linux-v1", +} MAX_INPUT_BYTES = 1_048_576 MAX_COUNT = 1_000_000 MAX_BYTES = 1 << 50 @@ -74,6 +81,31 @@ _DIGEST_RE = re.compile(r"(?:sha256:)?[0-9a-f]{64}") _LABEL_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,63}") _DISPLAY_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._+()-]{0,127}") +_AUTOMATED_REPLAY_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "result", + "source_commit", + "package", + "model_id", + "manifest_digest", + "real_window_sessions", + "localhost_inference_count", + "policy_dialog_saved", + "start_clicked", + "pause_control_observed", + "restart_resume_observed", + "pause_clicked", + "sharing_intent_paused", + "policy_profile", + "sequence_profile", + "start_observation_seconds", + "session_duration_seconds", + "privacy_safe", + "qualification_temporaries_removed", +} class LifecycleEvidenceError(ValueError): @@ -655,7 +687,107 @@ def finalize(self) -> dict[str, Any]: } +def _validate_automated_replay(raw_document: Mapping[str, Any]) -> dict[str, Any]: + document = dict(_mapping(raw_document)) + _exact_fields(document, _AUTOMATED_REPLAY_FIELDS) + if ( + document["schema_version"] != AUTOMATED_REPLAY_SCHEMA_VERSION + or document["scope"] != AUTOMATED_REPLAY_SCOPE + or document["result"] != "passed" + or not isinstance(document["run_id"], str) + or _LABEL_RE.fullmatch(document["run_id"]) is None + or document["platform"] not in ("windows", "linux") + or not isinstance(document["source_commit"], str) + or _HEX40_RE.fullmatch(document["source_commit"]) is None + or document["model_id"] not in MODEL_PROFILES + or document["policy_profile"] != AUTOMATED_REPLAY_POLICY_PROFILE + or document["sequence_profile"] != AUTOMATED_REPLAY_SEQUENCE_PROFILES.get(document["platform"]) + ): + _fail() + profile = MODEL_PROFILES[document["model_id"]] + expected_manifest = "sha256:" + profile["manifest_digest"] + if document["manifest_digest"] != expected_manifest: + _fail() + package = _mapping(document["package"]) + _exact_fields(package, {"sha256", "bytes", "verified_before_run", "self_test_count"}) + if ( + not isinstance(package["sha256"], str) + or _DIGEST_RE.fullmatch(package["sha256"]) is None + or type(package["bytes"]) is not int + or not 1 <= package["bytes"] <= 8 * 1024**3 + or package["verified_before_run"] is not True + or package["self_test_count"] != 4 + ): + _fail() + expected_inferences = 1 if document["platform"] == "windows" else 2 + expected_resume = document["platform"] == "linux" + expected_start_observation = 25.0 if document["platform"] == "windows" else 20.0 + if ( + document["real_window_sessions"] != 2 + or document["localhost_inference_count"] != expected_inferences + or type(document["restart_resume_observed"]) is not bool + or document["restart_resume_observed"] is not expected_resume + or type(document["start_observation_seconds"]) not in (int, float) + or float(document["start_observation_seconds"]) != expected_start_observation + ): + _fail() + for field in ( + "policy_dialog_saved", + "start_clicked", + "pause_control_observed", + "pause_clicked", + "sharing_intent_paused", + "privacy_safe", + "qualification_temporaries_removed", + ): + if document[field] is not True: + _fail() + durations = _mapping(document["session_duration_seconds"]) + _exact_fields(durations, {"initial", "restart"}) + for value in durations.values(): + if type(value) not in (int, float) or not math.isfinite(float(value)) or not 0 <= float(value) <= 3_630: + _fail() + return { + "schema_version": AUTOMATED_REPLAY_SCHEMA_VERSION, + "scope": AUTOMATED_REPLAY_SCOPE, + "result": "passed", + "run_id": document["run_id"], + "platform": document["platform"], + "source_commit": document["source_commit"], + "package_sha256": package["sha256"].removeprefix("sha256:"), + "package_bytes": package["bytes"], + "model_id": document["model_id"], + "manifest_digest": expected_manifest.removeprefix("sha256:"), + "package": dict(package), + "model": { + "id": document["model_id"], + "manifest_digest": expected_manifest, + }, + "lifecycle": { + "real_window_sessions": 2, + "localhost_inference_count": expected_inferences, + "policy_dialog_saved": True, + "start_clicked": True, + "pause_control_observed": True, + "restart_resume_observed": expected_resume, + "pause_clicked": True, + "sharing_intent_paused": True, + "policy_profile": AUTOMATED_REPLAY_POLICY_PROFILE, + "sequence_profile": AUTOMATED_REPLAY_SEQUENCE_PROFILES[document["platform"]], + "start_observation_seconds": expected_start_observation, + "response_content_retained": False, + "token_identifier_count": 0, + }, + "cleanup": { + "qualification_temporaries_removed": True, + "complete": True, + }, + } + + def validate_lifecycle_document(raw_document: Mapping[str, Any]) -> dict[str, Any]: + if isinstance(raw_document, dict) and raw_document.get("scope") == AUTOMATED_REPLAY_SCOPE: + return _validate_automated_replay(raw_document) document = dict(_mapping(raw_document)) _exact_fields(document, _DOCUMENT_FIELDS) phases = document["phases"] diff --git a/scripts/gate13_route_fence.py b/scripts/gate13_route_fence.py new file mode 100644 index 000000000..4630dee7a --- /dev/null +++ b/scripts/gate13_route_fence.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""Fence the Gate 13 product route to one exact client model. + +Run this as root on the already-qualified route VM immediately before each client. +It stops the other product service, restarts the requested service so its DHT +advertisement is fresh, and verifies the exact local product view twice. Only +bounded route facts are emitted; credentials, endpoints, paths, and model outputs +never leave the process. +""" + +from __future__ import annotations + +import argparse +import json +import os +import stat +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence +from urllib.error import HTTPError, URLError +from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener + +SCHEMA_VERSION = 1 +SCOPE = "gate13-route-client-fence" +MAX_RESPONSE_BYTES = 1_048_576 +MAX_SECRET_BYTES = 512 +SERVICE_ACTION_TIMEOUT_SECONDS = 180 + + +@dataclass(frozen=True) +class Profile: + target: str + service: str + other_service: str + origin: str + local_key: Path + control_key: Path + model_id: str + manifest_digest: str + total_blocks: int + + +PROFILES = { + "windows": Profile( + target="windows", + service="communityai-qwen.service", + other_service="communityai-gemma.service", + origin="http://127.0.0.1:8081", + local_key=Path("/srv/communityai/qwen/local-api.key"), + control_key=Path("/srv/communityai/qwen/control-api.key"), + model_id="Qwen3.5 2B", + manifest_digest="sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + total_blocks=24, + ), + "linux": Profile( + target="linux", + service="communityai-gemma.service", + other_service="communityai-qwen.service", + origin="http://127.0.0.1:8082", + local_key=Path("/srv/communityai/gemma/local-api.key"), + control_key=Path("/srv/communityai/gemma/control-api.key"), + model_id="Gemma 4 E2B IT", + manifest_digest="sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + total_blocks=35, + ), +} + + +class FenceError(RuntimeError): + """The exact route service could not be made stable for one client.""" + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ARG002 + return None + + +def _secret(path: Path) -> str: + try: + metadata = path.lstat() + except OSError as exc: + raise FenceError("route credential is unavailable") from exc + if path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= MAX_SECRET_BYTES: + raise FenceError("route credential is unsafe") + try: + value = path.read_text(encoding="ascii").strip() + except (OSError, UnicodeError) as exc: + raise FenceError("route credential is unreadable") from exc + if not value or any(character.isspace() for character in value): + raise FenceError("route credential is invalid") + return value + + +def _request_json(opener: Any, url: str, secret: str) -> Mapping[str, Any]: + request = Request(url, headers={"Authorization": f"Bearer {secret}", "Accept": "application/json"}) + try: + with opener.open(request, timeout=10) as response: + if response.status != 200 or response.headers.get_content_type() != "application/json": + raise FenceError("route API rejected the readiness probe") + payload = response.read(MAX_RESPONSE_BYTES + 1) + except (HTTPError, URLError, OSError, TimeoutError) as exc: + raise FenceError("route API is unavailable") from exc + if not 1 <= len(payload) <= MAX_RESPONSE_BYTES: + raise FenceError("route API response is invalid") + try: + document = json.loads(payload.decode("utf-8")) + except (UnicodeError, json.JSONDecodeError) as exc: + raise FenceError("route API response is invalid") from exc + if not isinstance(document, dict): + raise FenceError("route API response is invalid") + return document + + +def _systemctl( + arguments: Sequence[str], + runner: Callable[..., subprocess.CompletedProcess] = subprocess.run, + *, + timeout_seconds: float = 60, +) -> None: + try: + result = runner( + ["/usr/bin/systemctl", *arguments], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=timeout_seconds, + close_fds=True, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise FenceError("route service action failed") from exc + if result.returncode != 0: + raise FenceError("route service action failed") + + +def _snapshot(profile: Profile, opener: Any) -> bool: + local_secret = _secret(profile.local_key) + control_secret = _secret(profile.control_key) + models = _request_json(opener, f"{profile.origin}/v1/models", local_secret) + status = _request_json(opener, f"{profile.origin}/control/v1/status", control_secret) + local_secret = control_secret = "" + data = models.get("data") + if not isinstance(data, list): + return False + model = next((item for item in data if isinstance(item, dict) and item.get("id") == profile.model_id), None) + selection = status.get("auto_selection") + if not isinstance(model, dict) or not isinstance(selection, dict): + return False + return bool( + model.get("availability") == "complete" + and model.get("manifest_digest") == profile.manifest_digest + and selection.get("status") == "selected" + and selection.get("model") == profile.model_id + and selection.get("manifest_digest") == profile.manifest_digest + and selection.get("covered_blocks") == profile.total_blocks + and selection.get("total_blocks") == profile.total_blocks + and isinstance(selection.get("peer_count"), int) + and selection["peer_count"] > 0 + ) + + +def fence_route( + profile: Profile, + *, + timeout_seconds: float, + settle_seconds: float, + runner: Callable[..., subprocess.CompletedProcess] = subprocess.run, + opener: Any = None, + clock: Callable[[], float] = time.monotonic, + sleeper: Callable[[float], None] = time.sleep, +) -> Mapping[str, Any]: + """Run the fence and require systemd to report the standby as inactive.""" + + if opener is None: + opener = build_opener(ProxyHandler({}), _RejectRedirects()) + _systemctl( + ("stop", profile.other_service), + runner, + timeout_seconds=SERVICE_ACTION_TIMEOUT_SECONDS, + ) + _systemctl( + ("restart", profile.service), + runner, + timeout_seconds=SERVICE_ACTION_TIMEOUT_SECONDS, + ) + deadline = clock() + timeout_seconds + while clock() < deadline: + candidate_ready = False + try: + _systemctl(("is-active", "--quiet", profile.service), runner) + candidate_ready = _snapshot(profile, opener) + except FenceError: + pass + if candidate_ready: + sleeper(settle_seconds) + try: + _systemctl(("is-active", "--quiet", profile.service), runner) + result = runner( + ["/usr/bin/systemctl", "is-active", "--quiet", profile.other_service], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=60, + close_fds=True, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise FenceError("standby route state is unavailable") from exc + if result.returncode == 0: + raise FenceError("route fence did not remain stable") + try: + if _snapshot(profile, opener): + break + except FenceError: + pass + sleeper(5.0) + else: + raise FenceError("route did not become ready before the deadline") + return { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "result": "passed", + "target": profile.target, + "model_id": profile.model_id, + "manifest_digest": profile.manifest_digest, + "covered_blocks": profile.total_blocks, + "total_blocks": profile.total_blocks, + "peer_count_minimum": 1, + "target_service_restarted": True, + "standby_service_stopped": True, + "stable_rechecks": 2, + "settle_seconds": settle_seconds, + "privacy_safe": True, + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Fence the Gate 13 route for one exact client") + parser.add_argument("--target", choices=tuple(PROFILES), required=True) + parser.add_argument("--timeout-seconds", type=float, default=900.0) + parser.add_argument("--settle-seconds", type=float, default=30.0) + args = parser.parse_args(argv) + try: + if hasattr(os, "geteuid") and os.geteuid() != 0: + raise FenceError("route fence requires root") + if not 30 <= args.timeout_seconds <= 1_800 or not 5 <= args.settle_seconds <= 120: + raise FenceError("route fence bounds are invalid") + result = fence_route( + PROFILES[args.target], + timeout_seconds=args.timeout_seconds, + settle_seconds=args.settle_seconds, + ) + except BaseException: + result = { + "schema_version": SCHEMA_VERSION, + "scope": SCOPE, + "result": "failed", + "failure_code": "route_fence_failed", + } + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return 0 if result.get("result") == "passed" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gate13_route_setup.sh b/scripts/gate13_route_setup.sh new file mode 100644 index 000000000..cf4bb8b2a --- /dev/null +++ b/scripts/gate13_route_setup.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 +root=/tmp/gate13-route +wheel="$root/drift-2.3.0.dev2-py3-none-any.whl" +test "$(stat -c %s "$wheel")" = "389107" +test "$(sha256sum "$wheel" | cut -d' ' -f1)" = "7a42803811289e14f69835331e0fbab69dd353c70c835131c10bdfa96ca5f111" +test "$(sha256sum "$root/configure_product_route_node.py" | cut -d' ' -f1)" = "fc385f74e02ca955203b1fc5e8ae493c7f4ccd31bd7383c2ae0a1c461c91363e" +test "$(sha256sum "$root/gate11_product_node_acceptance.py" | cut -d' ' -f1)" = "bdcc9f499a7cd6b727c0e33a0c4c2b0e71e76e28f3f21cb99804a8f39edfa0d2" +apt-get update -qq +DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-venv python3-pip curl +if ! id communityai >/dev/null 2>&1; then + useradd --system --create-home --home-dir /srv/communityai --shell /usr/sbin/nologin communityai +fi +install -d -m 0755 /opt/communityai +if [ ! -x /opt/communityai/venv/bin/drift ]; then + python3 -m venv /opt/communityai/venv + /opt/communityai/venv/bin/pip install --no-cache-dir "$wheel[api]" +fi +chmod -R a+rX /opt/communityai/venv +install -d -o communityai -g communityai -m 0700 /srv/communityai/qwen /srv/communityai/gemma /srv/communityai/cache +install -d -o root -g root -m 0755 /opt/communityai/bootstrap +cp -a "$root/catalog-v1/." /opt/communityai/bootstrap/ +public_ip="$(curl -fsS -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)" +test -n "$public_ip" +for role in qwen gemma; do + data="/srv/communityai/$role" + sudo -u communityai /opt/communityai/venv/bin/drift bootstrap /opt/communityai/bootstrap/catalog-bootstrap.json --data_dir "$data" --node_config "$data/node-config.json" >/dev/null +done +/opt/communityai/venv/bin/python "$root/configure_product_route_node.py" --config /srv/communityai/qwen/node-config.json --role primary --public-ip "$public_ip" --cache-root /srv/communityai/cache >/dev/null +/opt/communityai/venv/bin/python "$root/configure_product_route_node.py" --config /srv/communityai/gemma/node-config.json --role standby --public-ip "$public_ip" --cache-root /srv/communityai/cache >/dev/null +chown -R communityai:communityai /srv/communityai +cat >/etc/systemd/system/communityai-qwen.service <<'UNIT' +[Unit] +Description=CommunityAI Qwen public route +After=network-online.target +Wants=network-online.target +[Service] +Type=simple +User=communityai +Group=communityai +WorkingDirectory=/srv/communityai/qwen +ExecStart=/opt/communityai/venv/bin/drift node --config /srv/communityai/qwen/node-config.json --data_dir /srv/communityai/qwen --host 127.0.0.1 --port 8081 +Restart=on-failure +RestartSec=5 +TimeoutStopSec=30 +LimitCORE=0 +[Install] +WantedBy=multi-user.target +UNIT +cat >/etc/systemd/system/communityai-gemma.service <<'UNIT' +[Unit] +Description=CommunityAI Gemma public route +After=network-online.target +Wants=network-online.target +[Service] +Type=simple +User=communityai +Group=communityai +WorkingDirectory=/srv/communityai/gemma +ExecStart=/opt/communityai/venv/bin/drift node --config /srv/communityai/gemma/node-config.json --data_dir /srv/communityai/gemma --host 127.0.0.1 --port 8082 +Restart=on-failure +RestartSec=5 +TimeoutStopSec=30 +LimitCORE=0 +[Install] +WantedBy=multi-user.target +UNIT +systemctl daemon-reload +systemctl enable --now communityai-qwen.service communityai-gemma.service +rm -rf "$root" +printf 'route-setup=started\n' diff --git a/scripts/gate13_run_controller.py b/scripts/gate13_run_controller.py new file mode 100644 index 000000000..87573273c --- /dev/null +++ b/scripts/gate13_run_controller.py @@ -0,0 +1,807 @@ +"""Durable state contract for one bounded Gate 13 GCP lifecycle. + +This module is deliberately provider-command agnostic. The paid-run adapter supplies a +fresh, exact provider/host observation before every transition and executes only the +returned allowlisted action. Persisting the transition before returning makes a local +operator crash recoverable: the next invocation inventories first and either reattaches +to the same durable host job or proceeds to cleanup. + +A lifecycle is never resumed after a product attempt fails. Such a client is consumed +for acceptance even when its product-owned files were removed successfully. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +import gate13_packaged_lifecycle as lifecycle +import qualification_cost_guard as cost_guard + +SCHEMA_VERSION = 1 +STATE_SCHEMA_VERSION = 1 +MAX_JSON_BYTES = 1_048_576 +MAX_STATE_BYTES = 262_144 +MIN_ROUTE_RUNWAY_SECONDS = 3_600 +ALLOWED_COMBINED_CLOUD_CEILINGS = frozenset({100.0, 500.0}) +PROTECTED_INSTANCE = "communityai-bootstrap-1" + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_NAME_RE = re.compile(r"[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?") +_DIGEST_RE = re.compile(r"(?:sha256:)?[0-9a-f]{64}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") + +PHASES = { + "ABSENT", + "ROUTE_STARTING", + "ROUTE_ACCEPTING", + "ROUTE_ACCEPTED", + "WINDOWS_RUNNING", + "WINDOWS_COLLECTING", + "WINDOWS_COLLECTED", + "WINDOWS_DELETING", + "LINUX_RUNNING", + "LINUX_COLLECTING", + "LINUX_COLLECTED", + "LINUX_DELETING", + "ROUTE_DELETING", + "CLEANING_FAILED", + "CLEANED_PASS", + "CLEANED_FAILURE", +} +TERMINAL_PHASES = {"CLEANED_PASS", "CLEANED_FAILURE"} +JOB_STATES = {"absent", "starting", "running", "passed", "failed", "ambiguous"} +ACTION_STATES = { + "start_route", + "accept_route", + "start_windows", + "collect_windows", + "delete_windows", + "start_linux", + "collect_linux", + "delete_linux", + "delete_route", + "cleanup_failure", + "none", +} +CLEANUP_ACTIONS = frozenset({"delete_windows", "delete_linux", "delete_route", "cleanup_failure"}) + +_STATE_FIELDS = { + "schema_version", + "run_id", + "authorization_sha256", + "provider_plan_digest", + "revision", + "phase", + "failure_code", + "route_acceptance_digest", + "windows_evidence_digest", + "linux_evidence_digest", + "windows_consumed", + "linux_consumed", + "cleanup_verified", + "next_action", +} +_OBSERVATION_FIELDS = { + "schema_version", + "run_id", + "observed_at_unix", + "instances", + "disks", + "firewalls", + "protected_bootstrap_running", + "route_acceptance", + "clients", +} +_INSTANCE_FIELDS = { + "present", + "run_id", + "source_commit", + "termination_unix", +} +_CLIENT_FIELDS = {"job_state", "attempt_ordinal", "evidence_digest"} +_ROUTE_ACCEPTANCE_FIELDS = {"job_state", "evidence_digest"} + + +class RunControllerError(ValueError): + """The run state, authorization, or provider observation failed closed.""" + + +@dataclass(frozen=True) +class RunPlan: + run_id: str + authorization_sha256: str + provider_plan_digest: str + ledger_state: str + project: str + zone: str + route_instance: str + route_disk: str + route_firewalls: tuple[str, str] + route_source_commit: str + windows_instance: str + windows_disk: str + windows_source_commit: str + linux_instance: str + linux_disk: str + linux_source_commit: str + windows_package_sha256: str + windows_package_bytes: int + linux_package_sha256: str + linux_package_bytes: int + qwen_manifest: str + gemma_manifest: str + clients_may_run_concurrently: bool + + @property + def instance_names(self) -> tuple[str, str, str]: + return (self.route_instance, self.windows_instance, self.linux_instance) + + @property + def disk_names(self) -> tuple[str, str, str]: + return (self.route_disk, self.windows_disk, self.linux_disk) + + +def _reject_constant(_value: str) -> None: + raise RunControllerError("invalid JSON") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise RunControllerError("duplicate JSON field") + value[key] = item + return value + + +def _strict_json_bytes(payload: bytes, maximum: int = MAX_JSON_BYTES) -> Mapping[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= maximum: + raise RunControllerError("JSON size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RunControllerError("invalid JSON") from exc + if not isinstance(value, dict): + raise RunControllerError("JSON root is invalid") + return value + + +def _regular_bytes(path: Path, maximum: int) -> bytes: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise RunControllerError("required file is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= maximum: + raise RunControllerError("required file is unsafe") + try: + return path.read_bytes() + except OSError as exc: + raise RunControllerError("required file is unreadable") from exc + + +def _mapping(value: Any, fields: set[str], label: str) -> Mapping[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise RunControllerError(f"{label} schema is invalid") + return value + + +def _string(value: Any, pattern: re.Pattern[str], label: str) -> str: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise RunControllerError(f"{label} is invalid") + return value + + +def _boolean(value: Any, label: str) -> bool: + if type(value) is not bool: + raise RunControllerError(f"{label} is invalid") + return value + + +def _integer(value: Any, label: str, *, minimum: int = 0, maximum: int = 2**63 - 1) -> int: + if type(value) is not int or not minimum <= value <= maximum: + raise RunControllerError(f"{label} is invalid") + return value + + +def _provider_digest(provider_plan: Mapping[str, Any]) -> str: + return cost_guard._provider_plan_digest(provider_plan) + + +def load_plan(authorization_path: Path, ledger_path: Path) -> RunPlan: + authorization_payload = _regular_bytes(authorization_path, MAX_JSON_BYTES) + authorization = _strict_json_bytes(authorization_payload) + if authorization.get("schema_version") != 1 or authorization.get("gate") != 13: + raise RunControllerError("authorization scope is invalid") + if authorization.get("result") != "authorized": + raise RunControllerError("authorization is not active") + + run_id = _string(authorization.get("run_id"), _RUN_RE, "run id") + provider_plan = authorization.get("provider_plan") + if not isinstance(provider_plan, dict): + raise RunControllerError("provider plan is invalid") + provider_plan_digest = _provider_digest(provider_plan) + if authorization.get("provider_plan_digest") != provider_plan_digest: + raise RunControllerError("provider plan digest changed") + + authorization_section = authorization.get("authorization") + if not isinstance(authorization_section, dict): + raise RunControllerError("cost authorization is invalid") + try: + ceiling = float(authorization_section["combined_cloud_ceiling_usd"]) + before = float(authorization_section["ledger_committed_before_run_usd"]) + maximum = float(authorization_section["maximum_estimate_usd"]) + remaining = float(authorization_section["remaining_after_run_maximum_usd"]) + except (KeyError, TypeError, ValueError) as exc: + raise RunControllerError("cost authorization is invalid") from exc + if ( + not all(math.isfinite(value) for value in (ceiling, before, maximum, remaining)) + or ceiling not in ALLOWED_COMBINED_CLOUD_CEILINGS + or before < 0 + or maximum <= 0 + or before + maximum > ceiling + or abs((ceiling - before - maximum) - remaining) > 0.001 + or authorization_section.get("reservation_recorded") is not True + or authorization_section.get("provisioning_authorized_after_fail_closed_preflight") is not True + ): + raise RunControllerError("cost authorization is inconsistent") + + prohibited = authorization.get("prohibited") + if not isinstance(prohibited, dict) or any(value != 0 or type(value) is not int for value in prohibited.values()): + raise RunControllerError("prohibited work is present") + + source = authorization.get("source") + immutable = authorization.get("immutable_inputs") + route = provider_plan.get("route") + clients = provider_plan.get("clients") + sequencing = provider_plan.get("sequencing") + if not all(isinstance(value, dict) for value in (source, immutable, route, sequencing)): + raise RunControllerError("authorization bindings are invalid") + legacy_lifecycle = sequencing.get("all_16_phases_required_per_platform") is True + automated_replay = sequencing.get("automated_gate13_replay_required") is True + if ( + not isinstance(clients, list) + or len(clients) != 2 + or sequencing.get("route_live_for_both_lifecycles") is not True + or legacy_lifecycle == automated_replay + or sequencing.get("exact_cleanup_before_pass") is not True + ): + raise RunControllerError("execution sequencing is invalid") + + by_platform = { + client.get("platform"): client + for client in clients + if isinstance(client, dict) and isinstance(client.get("platform"), str) + } + if set(by_platform) != {"windows", "linux"}: + raise RunControllerError("client plan is invalid") + windows = by_platform["windows"] + linux = by_platform["linux"] + + project = _string(provider_plan.get("project"), _NAME_RE, "project") + route_instance = _string(route.get("instance"), _NAME_RE, "route instance") + zone = route.get("zone") + if not isinstance(zone, str) or not zone or route_instance == PROTECTED_INSTANCE: + raise RunControllerError("route target is invalid") + if windows.get("zone") != zone or linux.get("zone") != zone: + raise RunControllerError("client zones are inconsistent") + firewalls = route.get("firewalls") + if not isinstance(firewalls, list) or len(firewalls) != 2: + raise RunControllerError("firewall plan is invalid") + firewall_names = tuple(_string(value, _NAME_RE, "firewall") for value in firewalls) + instance_names = ( + route_instance, + _string(windows.get("instance"), _NAME_RE, "Windows instance"), + _string(linux.get("instance"), _NAME_RE, "Linux instance"), + ) + if len(set(instance_names)) != 3 or PROTECTED_INSTANCE in instance_names: + raise RunControllerError("instance targets are unsafe") + + ledger = _regular_bytes(ledger_path, MAX_JSON_BYTES * 4).decode("utf-8") + ledger_rows = [line for line in ledger.splitlines() if line.startswith(f"| {run_id} |")] + if len(ledger_rows) != 1 or provider_plan_digest not in ledger_rows[0]: + raise RunControllerError("ledger reservation is absent") + ledger_cells = [cell.strip() for cell in ledger_rows[0].strip().strip("|").split("|")] + if len(ledger_cells) != 7 or ledger_cells[0] != run_id: + raise RunControllerError("ledger reservation is invalid") + ledger_state = ledger_cells[-1] + if ledger_state not in {"RESERVED", "CLEANED-COMMITTED", "CLEANED-RELEASED"}: + raise RunControllerError("ledger state is invalid") + + windows_package = immutable.get("windows_package") + linux_package = immutable.get("linux_package") + if not isinstance(windows_package, dict) or not isinstance(linux_package, dict): + raise RunControllerError("package bindings are invalid") + + route_source = _string(source.get("route_runtime_commit"), _COMMIT_RE, "route source") + package_source = _string(source.get("package_commit"), _COMMIT_RE, "package source") + qwen_manifest = _string(immutable.get("qwen_manifest"), _DIGEST_RE, "Qwen manifest") + gemma_manifest = _string(immutable.get("gemma_manifest"), _DIGEST_RE, "Gemma manifest") + windows_sha = _string(windows_package.get("sha256"), _DIGEST_RE, "Windows package digest") + linux_sha = _string(linux_package.get("sha256"), _DIGEST_RE, "Linux package digest") + + authorization_sha256 = "sha256:" + hashlib.sha256(authorization_payload).hexdigest() + return RunPlan( + run_id=run_id, + authorization_sha256=authorization_sha256, + provider_plan_digest=provider_plan_digest, + ledger_state=ledger_state, + project=project, + zone=zone, + route_instance=route_instance, + route_disk=route_instance, + route_firewalls=(firewall_names[0], firewall_names[1]), + route_source_commit=route_source, + windows_instance=instance_names[1], + windows_disk=instance_names[1], + windows_source_commit=package_source, + linux_instance=instance_names[2], + linux_disk=instance_names[2], + linux_source_commit=package_source, + windows_package_sha256=windows_sha, + windows_package_bytes=_integer(windows_package.get("bytes"), "Windows package bytes", minimum=1), + linux_package_sha256=linux_sha, + linux_package_bytes=_integer(linux_package.get("bytes"), "Linux package bytes", minimum=1), + qwen_manifest=qwen_manifest, + gemma_manifest=gemma_manifest, + clients_may_run_concurrently=_boolean( + sequencing.get("clients_may_run_concurrently"), + "client concurrency policy", + ), + ) + + +def initial_state(plan: RunPlan) -> dict[str, Any]: + if plan.ledger_state != "RESERVED": + raise RunControllerError("authorization is not reserved for provisioning") + if plan.clients_may_run_concurrently: + raise RunControllerError("concurrent clients are forbidden") + return { + "schema_version": STATE_SCHEMA_VERSION, + "run_id": plan.run_id, + "authorization_sha256": plan.authorization_sha256, + "provider_plan_digest": plan.provider_plan_digest, + "revision": 0, + "phase": "ABSENT", + "failure_code": None, + "route_acceptance_digest": None, + "windows_evidence_digest": None, + "linux_evidence_digest": None, + "windows_consumed": False, + "linux_consumed": False, + "cleanup_verified": False, + "next_action": "start_route", + } + + +def validate_state(raw: Mapping[str, Any], plan: RunPlan) -> dict[str, Any]: + state = dict(_mapping(raw, _STATE_FIELDS, "state")) + if ( + state["schema_version"] != STATE_SCHEMA_VERSION + or state["run_id"] != plan.run_id + or state["authorization_sha256"] != plan.authorization_sha256 + or state["provider_plan_digest"] != plan.provider_plan_digest + ): + raise RunControllerError("state authorization binding changed") + _integer(state["revision"], "state revision") + if state["phase"] not in PHASES or state["next_action"] not in ACTION_STATES: + raise RunControllerError("state transition is invalid") + for field in ("windows_consumed", "linux_consumed", "cleanup_verified"): + _boolean(state[field], field) + for field in ("route_acceptance_digest", "windows_evidence_digest", "linux_evidence_digest"): + if state[field] is not None: + _string(state[field], _DIGEST_RE, field) + failure = state["failure_code"] + if failure is not None and (not isinstance(failure, str) or not re.fullmatch(r"[a-z0-9_]{1,64}", failure)): + raise RunControllerError("failure code is invalid") + if state["phase"] in TERMINAL_PHASES and state["cleanup_verified"] is not True: + raise RunControllerError("terminal state lacks cleanup proof") + return state + + +def load_state(path: Path, plan: RunPlan) -> dict[str, Any]: + path = Path(path) + if not path.exists(): + return initial_state(plan) + return validate_state(_strict_json_bytes(_regular_bytes(path, MAX_STATE_BYTES), MAX_STATE_BYTES), plan) + + +def _atomic_write(path: Path, state: Mapping[str, Any]) -> None: + payload = (json.dumps(state, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n").encode("utf-8") + if len(payload) > MAX_STATE_BYTES: + raise RunControllerError("state exceeds its bound") + path = Path(os.path.abspath(os.fspath(path))) + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and (path.is_symlink() or not path.is_file()): + raise RunControllerError("state path is unsafe") + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + try: + os.chmod(temporary, 0o600) + except OSError: + pass + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _present(mapping: Mapping[str, Any]) -> bool: + return any(value is True for value in mapping.values()) + + +def _instance_present(observation: Mapping[str, Any], name: str) -> bool: + return bool(observation["instances"][name]["present"]) + + +def validate_observation(raw: Mapping[str, Any], plan: RunPlan, now_unix: int) -> dict[str, Any]: + observation = dict(_mapping(raw, _OBSERVATION_FIELDS, "observation")) + if observation["schema_version"] != SCHEMA_VERSION or observation["run_id"] != plan.run_id: + raise RunControllerError("observation binding is invalid") + observed_at = _integer(observation["observed_at_unix"], "observation time", minimum=1) + if abs(observed_at - now_unix) > 300: + raise RunControllerError("observation is stale") + _boolean(observation["protected_bootstrap_running"], "protected bootstrap state") + if observation["protected_bootstrap_running"] is not True: + raise RunControllerError("protected bootstrap is unavailable") + + instances = observation["instances"] + disks = observation["disks"] + firewalls = observation["firewalls"] + if ( + not isinstance(instances, dict) + or set(instances) != set(plan.instance_names) + or not isinstance(disks, dict) + or set(disks) != set(plan.disk_names) + or not isinstance(firewalls, dict) + or set(firewalls) != set(plan.route_firewalls) + ): + raise RunControllerError("resource inventory is not exact") + expected_sources = { + plan.route_instance: plan.route_source_commit, + plan.windows_instance: plan.windows_source_commit, + plan.linux_instance: plan.linux_source_commit, + } + for name, value in instances.items(): + item = _mapping(value, _INSTANCE_FIELDS, "instance") + present = _boolean(item["present"], "instance presence") + if present: + if item["run_id"] != plan.run_id or item["source_commit"] != expected_sources[name]: + raise RunControllerError("foreign exact-name instance is present") + termination = _integer(item["termination_unix"], "termination deadline", minimum=1) + if termination <= observed_at: + raise RunControllerError("instance deadline expired") + elif any(item[field] is not None for field in ("run_id", "source_commit", "termination_unix")): + raise RunControllerError("absent instance carries identity") + for value in (*disks.values(), *firewalls.values()): + _boolean(value, "resource presence") + + route_acceptance = _mapping(observation["route_acceptance"], _ROUTE_ACCEPTANCE_FIELDS, "route acceptance") + if route_acceptance["job_state"] not in JOB_STATES: + raise RunControllerError("route acceptance state is invalid") + if route_acceptance["evidence_digest"] is not None: + _string(route_acceptance["evidence_digest"], _DIGEST_RE, "route acceptance digest") + clients = observation["clients"] + if not isinstance(clients, dict) or set(clients) != {"windows", "linux"}: + raise RunControllerError("client job inventory is invalid") + for value in clients.values(): + client = _mapping(value, _CLIENT_FIELDS, "client job") + if client["job_state"] not in JOB_STATES: + raise RunControllerError("client job state is invalid") + _integer(client["attempt_ordinal"], "attempt ordinal", maximum=1) + if client["evidence_digest"] is not None: + _string(client["evidence_digest"], _DIGEST_RE, "client evidence digest") + return observation + + +def _all_resources_absent(observation: Mapping[str, Any]) -> bool: + return ( + not any(value["present"] for value in observation["instances"].values()) + and not _present(observation["disks"]) + and not _present(observation["firewalls"]) + ) + + +def _fail(state: dict[str, Any], code: str, observation: Mapping[str, Any]) -> dict[str, Any]: + state["failure_code"] = code + state["phase"] = "CLEANING_FAILED" + state["next_action"] = "cleanup_failure" + for platform in ("windows", "linux"): + if observation["clients"][platform]["job_state"] != "absent": + state[f"{platform}_consumed"] = True + return state + + +def begin_action(state: Mapping[str, Any], plan: RunPlan, *, action: str) -> dict[str, Any]: + """Persist an action intent before its first provider or host mutation. + + A missing resource after one of these transitions is a consumed failed attempt, not + permission to recreate it. Repeating ``start`` must inventory and reconcile the + durable job instead of calling this function again. + """ + + current = validate_state(state, plan) + if plan.ledger_state != "RESERVED" and action not in CLEANUP_ACTIONS: + raise RunControllerError("authorization is not reserved for forward action") + if action != current["next_action"] or action not in ACTION_STATES - {"none"}: + raise RunControllerError("action intent is out of order") + phases = { + "start_route": "ROUTE_STARTING", + "accept_route": "ROUTE_ACCEPTING", + "start_windows": "WINDOWS_RUNNING", + "collect_windows": "WINDOWS_COLLECTING", + "delete_windows": "WINDOWS_DELETING", + "start_linux": "LINUX_RUNNING", + "collect_linux": "LINUX_COLLECTING", + "delete_linux": "LINUX_DELETING", + "delete_route": "ROUTE_DELETING", + "cleanup_failure": "CLEANING_FAILED", + } + result = dict(current) + result["phase"] = phases[action] + result["next_action"] = "none" + result["revision"] += 1 + return validate_state(result, plan) + + +def reconcile( + state: Mapping[str, Any], + observation: Mapping[str, Any], + plan: RunPlan, + *, + now_unix: int, +) -> dict[str, Any]: + current = validate_state(state, plan) + observed = validate_observation(observation, plan, now_unix) + result = dict(current) + + if current["phase"] in TERMINAL_PHASES: + if not _all_resources_absent(observed): + raise RunControllerError("resource reappeared after terminal cleanup") + return result + + if _all_resources_absent(observed): + if current["phase"] == "ABSENT": + result["next_action"] = "start_route" + elif current["phase"] in {"CLEANING_FAILED", "ROUTE_DELETING", "LINUX_COLLECTED", "LINUX_DELETING"}: + passed = ( + current["failure_code"] is None + and current["route_acceptance_digest"] is not None + and current["windows_evidence_digest"] is not None + and current["linux_evidence_digest"] is not None + ) + result["phase"] = "CLEANED_PASS" if passed else "CLEANED_FAILURE" + result["cleanup_verified"] = True + result["next_action"] = "none" + else: + result["phase"] = "CLEANED_FAILURE" + result["failure_code"] = "resources_disappeared_before_completion" + result["cleanup_verified"] = True + result["next_action"] = "none" + result["revision"] += 1 + return validate_state(result, plan) + + route_present = _instance_present(observed, plan.route_instance) + windows_present = _instance_present(observed, plan.windows_instance) + linux_present = _instance_present(observed, plan.linux_instance) + route_job = observed["route_acceptance"]["job_state"] + windows_job = observed["clients"]["windows"]["job_state"] + linux_job = observed["clients"]["linux"]["job_state"] + windows_attempt = observed["clients"]["windows"]["attempt_ordinal"] + linux_attempt = observed["clients"]["linux"]["attempt_ordinal"] + + if not route_present or route_job in {"failed", "ambiguous"}: + return _fail(result, "route_failed_or_ambiguous", observed) + route_deadline = observed["instances"][plan.route_instance]["termination_unix"] + if route_deadline - now_unix < MIN_ROUTE_RUNWAY_SECONDS: + return _fail(result, "route_runway_exhausted", observed) + + if route_job != "passed": + if windows_present or linux_present: + return _fail(result, "client_started_before_route_acceptance", observed) + if current["phase"] == "ROUTE_ACCEPTING" and current["next_action"] == "none": + if route_job == "absent": + return _fail(result, "route_acceptance_disappeared", observed) + result["phase"] = "ROUTE_ACCEPTING" + result["next_action"] = "none" + else: + result["phase"] = "ROUTE_ACCEPTING" + result["next_action"] = "accept_route" + else: + route_digest = observed["route_acceptance"]["evidence_digest"] + if route_digest is None: + return _fail(result, "route_acceptance_digest_absent", observed) + result["route_acceptance_digest"] = route_digest + if current["windows_evidence_digest"] is None and windows_attempt == 1 and windows_job == "absent": + return _fail(result, "windows_attempt_disappeared", observed) + if current["linux_evidence_digest"] is None and linux_attempt == 1 and linux_job == "absent": + return _fail(result, "linux_attempt_disappeared", observed) + if linux_present and current["windows_evidence_digest"] is None: + return _fail(result, "linux_started_before_windows_evidence", observed) + if current["phase"] == "WINDOWS_DELETING": + if windows_present or observed["disks"][plan.windows_disk]: + result["next_action"] = "delete_windows" + else: + result["phase"] = "WINDOWS_COLLECTED" + result["next_action"] = "start_linux" + elif current["phase"] == "LINUX_DELETING": + if linux_present or observed["disks"][plan.linux_disk]: + result["next_action"] = "delete_linux" + else: + result["phase"] = "LINUX_COLLECTED" + result["next_action"] = "delete_route" + elif windows_present: + result["windows_consumed"] = windows_job != "absent" + if windows_job in {"failed", "ambiguous"}: + return _fail(result, "windows_failed_or_ambiguous", observed) + if windows_job == "passed": + result["phase"] = "WINDOWS_COLLECTING" + result["next_action"] = "collect_windows" + elif windows_job == "absent": + if current["phase"] == "WINDOWS_RUNNING" and current["next_action"] == "none": + result["phase"] = "WINDOWS_RUNNING" + result["next_action"] = "none" + else: + result["phase"] = "ROUTE_ACCEPTED" + result["next_action"] = "start_windows" + else: + result["phase"] = "WINDOWS_RUNNING" + result["next_action"] = "none" + elif current["windows_evidence_digest"] is None: + if current["windows_consumed"]: + return _fail(result, "windows_consumed_without_evidence", observed) + if current["phase"] == "WINDOWS_RUNNING" and current["next_action"] == "none": + return _fail(result, "windows_disappeared_after_start_intent", observed) + result["phase"] = "ROUTE_ACCEPTED" + result["next_action"] = "start_windows" + elif linux_present: + result["linux_consumed"] = linux_job != "absent" + if linux_job in {"failed", "ambiguous"}: + return _fail(result, "linux_failed_or_ambiguous", observed) + if linux_job == "passed": + result["phase"] = "LINUX_COLLECTING" + result["next_action"] = "collect_linux" + elif linux_job == "absent": + if current["phase"] == "LINUX_RUNNING" and current["next_action"] == "none": + result["phase"] = "LINUX_RUNNING" + result["next_action"] = "none" + else: + result["phase"] = "WINDOWS_COLLECTED" + result["next_action"] = "start_linux" + else: + result["phase"] = "LINUX_RUNNING" + result["next_action"] = "none" + elif current["linux_evidence_digest"] is None: + if current["linux_consumed"]: + return _fail(result, "linux_consumed_without_evidence", observed) + if current["phase"] == "LINUX_RUNNING" and current["next_action"] == "none": + return _fail(result, "linux_disappeared_after_start_intent", observed) + result["phase"] = "WINDOWS_COLLECTED" + result["next_action"] = "start_linux" + else: + result["phase"] = "LINUX_COLLECTED" + result["next_action"] = "delete_route" + + result["revision"] += 1 + return validate_state(result, plan) + + +def collect_platform( + state: Mapping[str, Any], + plan: RunPlan, + *, + platform: str, + evidence_payload: bytes, + observed_digest: str, +) -> dict[str, Any]: + current = validate_state(state, plan) + if platform not in {"windows", "linux"}: + raise RunControllerError("platform is invalid") + expected_phase = "WINDOWS_COLLECTING" if platform == "windows" else "LINUX_COLLECTING" + if current["phase"] != expected_phase: + raise RunControllerError("evidence collection is out of order") + digest = "sha256:" + hashlib.sha256(evidence_payload).hexdigest() + if digest != observed_digest: + raise RunControllerError("host evidence digest changed") + try: + raw = lifecycle.load_lifecycle_json(evidence_payload.decode("utf-8")) + validated = lifecycle.validate_lifecycle_document(raw) + except Exception as exc: + raise RunControllerError("lifecycle evidence is invalid") from exc + expected = { + "windows": { + "source_commit": plan.windows_source_commit, + "package_sha256": plan.windows_package_sha256, + "package_bytes": plan.windows_package_bytes, + "model_id": "Qwen3.5 2B", + "manifest_digest": plan.qwen_manifest.removeprefix("sha256:"), + }, + "linux": { + "source_commit": plan.linux_source_commit, + "package_sha256": plan.linux_package_sha256, + "package_bytes": plan.linux_package_bytes, + "model_id": "Gemma 4 E2B IT", + "manifest_digest": plan.gemma_manifest.removeprefix("sha256:"), + }, + }[platform] + for field, value in expected.items(): + if validated.get(field) != value: + raise RunControllerError("lifecycle evidence binding changed") + + result = dict(current) + result[f"{platform}_evidence_digest"] = digest + result[f"{platform}_consumed"] = True + if platform == "windows": + result["phase"] = "WINDOWS_DELETING" + result["next_action"] = "delete_windows" + else: + result["phase"] = "LINUX_DELETING" + result["next_action"] = "delete_linux" + result["revision"] += 1 + return validate_state(result, plan) + + +def mark_client_absent( + state: Mapping[str, Any], + plan: RunPlan, + *, + platform: str, + observation: Mapping[str, Any], + now_unix: int, +) -> dict[str, Any]: + current = validate_state(state, plan) + expected = "WINDOWS_DELETING" if platform == "windows" else "LINUX_DELETING" + if current["phase"] != expected or current[f"{platform}_evidence_digest"] is None: + raise RunControllerError("client deletion is out of order") + observed = validate_observation(observation, plan, now_unix) + instance = plan.windows_instance if platform == "windows" else plan.linux_instance + disk = plan.windows_disk if platform == "windows" else plan.linux_disk + if _instance_present(observed, instance) or observed["disks"][disk]: + raise RunControllerError("client absence is not proved") + if not _instance_present(observed, plan.route_instance): + raise RunControllerError("route disappeared during client deletion") + result = dict(current) + if platform == "windows": + result["phase"] = "WINDOWS_COLLECTED" + result["next_action"] = "start_linux" + else: + result["phase"] = "LINUX_COLLECTED" + result["next_action"] = "delete_route" + result["revision"] += 1 + return validate_state(result, plan) + + +def persist(path: Path, state: Mapping[str, Any], plan: RunPlan) -> None: + _atomic_write(path, validate_state(state, plan)) + + +def public_status(state: Mapping[str, Any], plan: RunPlan) -> dict[str, Any]: + current = validate_state(state, plan) + return { + "schema_version": 1, + "run_id": current["run_id"], + "phase": current["phase"], + "next_action": current["next_action"], + "failure_code": current["failure_code"], + "windows_consumed": current["windows_consumed"], + "linux_consumed": current["linux_consumed"], + "cleanup_verified": current["cleanup_verified"], + } diff --git a/scripts/gate13_windows_client_startup.ps1 b/scripts/gate13_windows_client_startup.ps1 new file mode 100644 index 000000000..a3be52f1f --- /dev/null +++ b/scripts/gate13_windows_client_startup.ps1 @@ -0,0 +1,142 @@ +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$metadataHeaders = @{ "Metadata-Flavor" = "Google" } +$metadataRoot = "http://metadata.google.internal/computeMetadata/v1/instance/attributes" +$bootstrapRoot = "C:\Gate13Bootstrap" +$runRoot = "C:\Gate13Run" +$downloadRoot = "C:\Gate13Download" +New-Item -ItemType Directory -Force -Path $bootstrapRoot, $runRoot, $downloadRoot | Out-Null +$readyMarker = Join-Path $bootstrapRoot "ready.txt" +if (Test-Path -LiteralPath $readyMarker -PathType Leaf) { + Set-Service -Name sshd -StartupType Automatic + if ((Get-Service -Name sshd).Status -ne "Running") { Start-Service -Name sshd } + $sshFirewall = Get-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -ErrorAction SilentlyContinue + if ($null -eq $sshFirewall) { + New-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -DisplayName "OpenSSH Server (sshd)" -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 | Out-Null + } else { + Set-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -Enabled True -Profile Any + } + return +} + +$capability = Get-WindowsCapability -Online -Name "OpenSSH.Server~~~~0.0.1.0" +if ($capability.State -ne "Installed") { + Add-WindowsCapability -Online -Name "OpenSSH.Server~~~~0.0.1.0" | Out-Null +} +Set-Service -Name sshd -StartupType Automatic +Start-Service -Name sshd +if (-not (Get-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -ErrorAction SilentlyContinue)) { + New-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -DisplayName "OpenSSH Server (sshd)" -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 | Out-Null +} +Set-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -Enabled True -Profile Any + +$randomBytes = New-Object byte[] 32 +$randomGenerator = [Security.Cryptography.RandomNumberGenerator]::Create() +try { + $randomGenerator.GetBytes($randomBytes) +} finally { + $randomGenerator.Dispose() +} +$plainPassword = [Convert]::ToBase64String($randomBytes) + "aA1!" +$securePassword = ConvertTo-SecureString $plainPassword -AsPlainText -Force +if (-not (Get-LocalUser -Name "M" -ErrorAction SilentlyContinue)) { + New-LocalUser -Name "M" -Password $securePassword -PasswordNeverExpires -UserMayNotChangePassword | Out-Null +} else { + Set-LocalUser -Name "M" -Password $securePassword +} +if (-not (Get-LocalUser -Name "Gate13Admin" -ErrorAction SilentlyContinue)) { + New-LocalUser -Name "Gate13Admin" -NoPassword -AccountNeverExpires -UserMayNotChangePassword | Out-Null +} +$administratorMemberNames = @(Get-LocalGroupMember -Group "Administrators" | ForEach-Object Name) +if ($administratorMemberNames -notcontains "$env:COMPUTERNAME\Gate13Admin") { + Add-LocalGroupMember -Group "Administrators" -Member "Gate13Admin" +} +$openSshGroup = Get-LocalGroup -Name "OpenSSH Users" +$openSshMemberNames = @(Get-LocalGroupMember -Group $openSshGroup | ForEach-Object Name) +if ($openSshMemberNames -notcontains "$env:COMPUTERNAME\M") { + Add-LocalGroupMember -Group $openSshGroup -Member "M" +} +Remove-LocalGroupMember -Group "Administrators" -Member "M" -ErrorAction SilentlyContinue + +$publicKey = (Invoke-RestMethod -Headers $metadataHeaders -Uri "$metadataRoot/gate13-ssh-public-key").Trim() +$profileRoot = "C:\Users\M" +$sshRoot = Join-Path $profileRoot ".ssh" +New-Item -ItemType Directory -Force -Path $profileRoot, $sshRoot | Out-Null +$authorizedKeys = Join-Path $sshRoot "authorized_keys" +[IO.File]::WriteAllText($authorizedKeys, $publicKey + "`n", [Text.UTF8Encoding]::new($false)) +& icacls.exe $sshRoot /inheritance:r /grant:r "M:(OI)(CI)F" "SYSTEM:(OI)(CI)F" | Out-Null +& icacls.exe $authorizedKeys /inheritance:r /grant:r "M:F" "SYSTEM:F" | Out-Null + +$programDataSsh = Join-Path $env:ProgramData "ssh" +New-Item -ItemType Directory -Force -Path $programDataSsh | Out-Null +$administratorKeys = Join-Path $programDataSsh "administrators_authorized_keys" +$ordinaryKeys = Join-Path $programDataSsh "communityai_gate13_m_authorized_keys" +[IO.File]::WriteAllText($administratorKeys, $publicKey + "`n", [Text.UTF8Encoding]::new($false)) +[IO.File]::WriteAllText($ordinaryKeys, $publicKey + "`n", [Text.UTF8Encoding]::new($false)) +& icacls.exe $administratorKeys /inheritance:r /grant:r "Administrators:F" "SYSTEM:F" | Out-Null +& icacls.exe $ordinaryKeys /inheritance:r /grant:r "Administrators:F" "SYSTEM:F" | Out-Null +$sshdConfig = Join-Path $programDataSsh "sshd_config" +if (-not (Test-Path -LiteralPath $sshdConfig -PathType Leaf)) { + Copy-Item -LiteralPath "$env:WINDIR\System32\OpenSSH\sshd_config_default" -Destination $sshdConfig +} +$marker = "# CommunityAI Gate13 ordinary user" +if (-not (Select-String -LiteralPath $sshdConfig -SimpleMatch $marker -Quiet)) { + [IO.File]::AppendAllText( + $sshdConfig, + "`n$marker`nMatch User M`n AuthorizedKeysFile __PROGRAMDATA__/ssh/communityai_gate13_m_authorized_keys`n", + [Text.UTF8Encoding]::new($false) + ) +} +& "$env:WINDIR\System32\OpenSSH\sshd.exe" -t +if ($LASTEXITCODE -ne 0) { throw "OpenSSH configuration invalid" } +Restart-Service -Name sshd + +$pythonRoot = "C:\Gate13Python" +if (-not (Test-Path -LiteralPath "$pythonRoot\python.exe" -PathType Leaf)) { + $installer = Join-Path $downloadRoot "python-3.12.9-amd64.exe" + & curl.exe -fL --retry 4 --retry-delay 5 "https://www.python.org/ftp/python/3.12.9/python-3.12.9-amd64.exe" -o $installer + if ($LASTEXITCODE -ne 0) { throw "Python download failed" } + if ((Get-Item -LiteralPath $installer).Length -ne 26923696) { throw "Python installer size changed" } + if ((Get-FileHash -LiteralPath $installer -Algorithm SHA256).Hash.ToLowerInvariant() -cne "2a52993092a19cfdffe126e2eeac46a4265e25705614546604ad44988e040c0f") { throw "Python installer digest changed" } + $process = Start-Process -FilePath $installer -ArgumentList "/quiet InstallAllUsers=1 TargetDir=$pythonRoot Include_pip=0 Include_test=0 PrependPath=0" -Wait -PassThru + if ($process.ExitCode -ne 0) { throw "Python installation failed" } + Remove-Item -LiteralPath $installer -Force +} + +$packageUrl = (Invoke-RestMethod -Headers $metadataHeaders -Uri "$metadataRoot/package-url").Trim() +$packageSha256 = (Invoke-RestMethod -Headers $metadataHeaders -Uri "$metadataRoot/package-sha256").Trim().ToLowerInvariant() +$packageBytes = [int64](Invoke-RestMethod -Headers $metadataHeaders -Uri "$metadataRoot/package-bytes") +$wrapper = Join-Path $downloadRoot "artifact.zip" +& curl.exe -fL --retry 4 --retry-delay 5 $packageUrl -o $wrapper +if ($LASTEXITCODE -ne 0) { throw "Package wrapper download failed" } +$staging = Join-Path $downloadRoot "artifact" +New-Item -ItemType Directory -Force -Path $staging | Out-Null +& tar.exe -xf $wrapper -C $staging +if ($LASTEXITCODE -ne 0) { throw "Package wrapper extraction failed" } +$archive = Join-Path $staging "communityai-desktop-windows.zip" +if ((Get-Item -LiteralPath $archive).Length -ne $packageBytes) { throw "Package byte size changed" } +if ((Get-FileHash -LiteralPath $archive -Algorithm SHA256).Hash.ToLowerInvariant() -ne $packageSha256) { throw "Package digest changed" } +$packageRoot = Join-Path $runRoot "package" +$installRoot = Join-Path $runRoot "install" +New-Item -ItemType Directory -Force -Path $packageRoot, $installRoot | Out-Null +Move-Item -LiteralPath $archive -Destination (Join-Path $packageRoot "communityai-desktop-windows.zip") +& tar.exe -xf (Join-Path $packageRoot "communityai-desktop-windows.zip") -C $installRoot +if ($LASTEXITCODE -ne 0) { throw "Product extraction failed" } +Remove-Item -LiteralPath $wrapper, $staging -Recurse -Force +& icacls.exe $runRoot /inheritance:r /grant:r "M:(OI)(CI)F" "Administrators:(OI)(CI)F" "SYSTEM:(OI)(CI)F" | Out-Null + +$winlogon = "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" +Set-ItemProperty -Path $winlogon -Name AutoAdminLogon -Value "1" -Type String +Set-ItemProperty -Path $winlogon -Name DefaultUserName -Value "M" -Type String +Set-ItemProperty -Path $winlogon -Name DefaultDomainName -Value $env:COMPUTERNAME -Type String +Set-ItemProperty -Path $winlogon -Name DefaultPassword -Value $plainPassword -Type String +Set-ItemProperty -Path $winlogon -Name AutoLogonCount -Value 1 -Type DWord +$clearArgument = '-NoProfile -WindowStyle Hidden -Command "Start-Sleep -Seconds 60; $p = ''HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon''; Remove-ItemProperty -Path $p -Name DefaultPassword -ErrorAction SilentlyContinue; Set-ItemProperty -Path $p -Name AutoAdminLogon -Value ''0'' -Type String"' +$clearAction = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $clearArgument +$clearTrigger = New-ScheduledTaskTrigger -AtLogOn -User "M" +Register-ScheduledTask -TaskName "Gate13ClearAutoLogon" -Action $clearAction -Trigger $clearTrigger -User "SYSTEM" -RunLevel Highest -Force | Out-Null +$plainPassword = $null +$securePassword = $null +[IO.File]::WriteAllText($readyMarker, "ready`n", [Text.UTF8Encoding]::new($false)) +Restart-Computer -Force diff --git a/scripts/gate13_windows_packaged_lifecycle.ps1 b/scripts/gate13_windows_packaged_lifecycle.ps1 index 59646ff1b..47f4da0ff 100644 --- a/scripts/gate13_windows_packaged_lifecycle.ps1 +++ b/scripts/gate13_windows_packaged_lifecycle.ps1 @@ -36,6 +36,8 @@ $script:LifecycleProcess = $null $script:LifecycleAcquisitionInvoked = $false $script:LifecycleOwnWorkRoot = $false $script:LifecycleOwnPersistentRoot = $false +$script:LifecycleFailurePhase = "initialization" +$script:LifecycleFailureOperation = "initialization" function Initialize-Gate13NativeHost { if ($null -ne ("Gate13.NativeHost" -as [type])) { @@ -1147,6 +1149,8 @@ function Measure-Gate13Phase { [Parameter(Mandatory = $true)] [string] $Name, [Parameter(Mandatory = $true)] [scriptblock] $Action ) + $script:LifecycleFailurePhase = $Name + $script:LifecycleFailureOperation = $Name $timer = [System.Diagnostics.Stopwatch]::StartNew() $facts = & $Action $timer.Stop() @@ -1199,7 +1203,33 @@ function Get-Gate13Sha256 { if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "required file missing" } - return (Get-FileHash -LiteralPath $Path -Algorithm SHA256 -ErrorAction Stop).Hash.ToLowerInvariant() + $stream = $null + $hasher = $null + $digest = $null + try { + $stream = [System.IO.FileStream]::new( + $Path, + [System.IO.FileMode]::Open, + [System.IO.FileAccess]::Read, + [System.IO.FileShare]::Read, + 1048576, + [System.IO.FileOptions]::SequentialScan + ) + $hasher = [System.Security.Cryptography.SHA256]::Create() + $digest = $hasher.ComputeHash($stream) + return [System.BitConverter]::ToString($digest).Replace("-", "").ToLowerInvariant() + } + finally { + if ($null -ne $digest) { + [Array]::Clear($digest, 0, $digest.Length) + } + if ($null -ne $hasher) { + $hasher.Dispose() + } + if ($null -ne $stream) { + $stream.Dispose() + } + } } function Read-Gate13JsonFile { @@ -2814,7 +2844,9 @@ function Invoke-Gate13WindowsPackagedLifecycle { })) [void]$phases.Add((Measure-Gate13Phase -Name "signed_bootstrap" -Action { + $script:LifecycleFailureOperation = "bootstrap_command" $state.Bootstrap = Invoke-Gate13Bootstrap + $script:LifecycleFailureOperation = "bootstrap_binding" if ( $state.Bootstrap.CatalogId -cne $state.Audit.PublicationCatalogId -or [int64]$state.Bootstrap.CatalogSequence -ne @@ -2826,8 +2858,11 @@ function Invoke-Gate13WindowsPackagedLifecycle { ) { throw "installed bootstrap did not match release provenance" } + $script:LifecycleFailureOperation = "product_start" Start-Gate13Product + $script:LifecycleFailureOperation = "product_readiness" $state.ProductStatus = Wait-Gate13ProductStatus -TimeoutSeconds 300 + $script:LifecycleFailureOperation = "profile_binding" $state.Profile = $state.ProductStatus.Profile if ( $state.Profile.ModelId -cne $state.Audit.ExpectedModelId -or @@ -2835,6 +2870,7 @@ function Invoke-Gate13WindowsPackagedLifecycle { ) { throw "operator-bound selected model identity rejected" } + $script:LifecycleFailureOperation = "selected_manifest_context" $state.Context = Get-Gate13SelectedManifestContext -Profile $state.Profile return [ordered]@{ catalog_id = $state.Bootstrap.CatalogId @@ -3186,6 +3222,8 @@ function Invoke-Gate13WindowsPackagedLifecycle { } })) + $script:LifecycleFailurePhase = "evidence_validation" + $script:LifecycleFailureOperation = "evidence_validation" $document = [ordered]@{ schema_version = 1 run_id = $state.Audit.RunId @@ -3252,10 +3290,24 @@ function Start-Gate13WindowsPackagedLifecycle { return 0 } catch { + $failurePhase = [string]$script:LifecycleFailurePhase + if ($failurePhase -notmatch '^[a-z_]{1,64}$') { + $failurePhase = "initialization" + } + $failureOperation = [string]$script:LifecycleFailureOperation + if ($failureOperation -notmatch '^[a-z_]{1,64}$') { + $failureOperation = $failurePhase + } Invoke-Gate13FailureCleanup - [Console]::Out.WriteLine( - '{"failure_code":"windows_packaged_lifecycle_failed","result":"failed","schema_version":1}' - ) + [Console]::Out.WriteLine(( + [ordered]@{ + failure_code = "windows_packaged_lifecycle_failed" + failure_operation = $failureOperation + failure_phase = $failurePhase + result = "failed" + schema_version = 1 + } | ConvertTo-Json -Compress + )) return 2 } finally { diff --git a/scripts/gate14_hardware_acceptance.py b/scripts/gate14_hardware_acceptance.py new file mode 100644 index 000000000..9b43a56d9 --- /dev/null +++ b/scripts/gate14_hardware_acceptance.py @@ -0,0 +1,861 @@ +"""Validate privacy-safe Gate 14 packaged hardware evidence. + +The real host probes are deliberately separate from this verifier. They may use private +paths and provider details while running, but only the strict bounded documents accepted +here can enter the evidence archive. The aggregate binds the exact controller source, +production packages, manifests, Gate 9 envelopes, device profiles, and final cleanup. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +import stat +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Mapping, Sequence + +SCHEMA_VERSION = 1 +PLATFORM_SCOPE = "gate14-packaged-hardware" +CLEANUP_SCOPE = "gate14-provider-cleanup" +AGGREGATE_SCOPE = "gate14-hardware-acceptance" +MAX_INPUT_BYTES = 262_144 +MAX_DURATION_SECONDS = 300.0 +MAX_BYTES = 1 << 50 +MAX_BLOCKS = 512 +PROTECTED_INSTANCE = "communityai-bootstrap-1" + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") +_NAME_RE = re.compile(r"[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?") +_PROJECT_RE = re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]") +_ZONE_RE = re.compile(r"[a-z]+(?:-[a-z0-9]+)+-[a-z]") +_OS_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._+()/-]{0,127}") + +MODEL_PROFILES = { + "Qwen3.5 2B": { + "manifest_digest": "sha256:3ba8528cb3c0d85e1ed048e0438a0d64cfbbc298944ed674caa6950d415f8e33", + "revision_commit": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "selected_artifact_count": 8, + "selected_artifact_bytes": 4_571_197_320, + "total_blocks": 24, + }, + "Gemma 4 E2B IT": { + "manifest_digest": "sha256:2f8debbe0fcdf5af8d4c56c982210fa50aa584314968ae2617e2ccc2de9eafdd", + "revision_commit": "3e22461f65e89153144f8adb70e3b8c2cc9845a7", + "selected_artifact_count": 5, + "selected_artifact_bytes": 10_278_818_149, + "total_blocks": 35, + }, +} +EXPECTED_PLATFORM_MODELS = {"windows": "Qwen3.5 2B", "linux": "Gemma 4 E2B IT"} +EXPECTED_PLATFORM_OS = {"windows": "Windows Server 2022", "linux": "Ubuntu 24.04"} +EXPECTED_GATE9_ENVELOPES = { + "windows": "sha256:cd68afb67d9b0f3cb8c82db0d3314ad89b558c20880998ea4d8c4493e9f4bc9f", + "linux": "sha256:2eb0bcf6419ba085665fad34310453a1b9dc2e89d90e9177f41566df012996c8", +} +EXPECTED_GATE13_EVIDENCE_SHA256 = "sha256:ad4f892f4af9a9aee0dd428d74695981d0cca6241f79c0270c9fcea3a229b72e" + +_DOCUMENT_FIELDS = { + "schema_version", + "scope", + "run_id", + "platform", + "result", + "source_commit", + "gate13_evidence_sha256", + "package", + "model", + "hardware", + "cache", + "placement", + "limits", + "suspensions", + "recovery", + "pause", + "restart", + "unsupported_telemetry", + "privacy", + "qualification_temporaries_removed", +} +_PACKAGE_FIELDS = { + "source_commit", + "archive_sha256", + "archive_bytes", + "release_metadata_sha256", +} +_MODEL_FIELDS = { + "id", + "manifest_digest", + "revision_commit", + "gate9_envelope_sha256", + "selected_artifact_count", + "selected_artifact_bytes", + "total_blocks", +} +_HARDWARE_FIELDS = { + "os_name", + "accelerator", + "accelerator_count", + "accelerator_memory_bytes", +} +_CACHE_FIELDS = { + "verified_bytes_before", + "verified_bytes_after", + "transfer_bytes_during_gate", + "digest_mismatch_count", + "forbidden_model_acquired", +} +_PLACEMENT_FIELDS = { + "automatic", + "worker_count", + "block_start", + "block_end", + "intent_published", + "remote_acknowledged", +} +_LIMIT_FIELDS = { + "disk_bytes", + "vram_bytes", + "bandwidth_mbps", + "power_watts", + "schedule_timezone", + "resource_limit_count", + "configured_and_resolved_match", + "low_vram_rejected", +} +_SUSPENSION_FIELDS = { + "kind", + "suspended", + "resumed", + "desired_intent_preserved", + "worker_count_during", + "duration_seconds", +} +_RECOVERY_FIELDS = { + "worker_crash_observed", + "worker_restarted", + "restart_seconds", + "previous_worker_absent", + "manifest_unchanged", + "automatic_block_range_valid", + "desired_intent_preserved", +} +_PAUSE_FIELDS = { + "requested", + "completed", + "duration_seconds", + "worker_count_after", + "descendant_count_after", +} +_RESTART_FIELDS = { + "node_restarted", + "policy_persisted", + "desired_intent_persisted", + "worker_resumed", + "duration_seconds", + "cache_reused", +} +_UNSUPPORTED_FIELDS = { + "device", + "configured_limit", + "start_rejected", + "reason_code", + "private_detail_retained", +} +_PRIVACY_FIELDS = { + "prompt_retained", + "response_retained", + "token_identifiers_retained", + "credentials_retained", + "paths_retained", + "endpoints_retained", + "provider_output_retained", +} +_CLEANUP_FIELDS = { + "schema_version", + "scope", + "run_id", + "result", + "provider", + "controller_source_commit", + "provider_plan_digest", + "project", + "zone", + "deleted_instances", + "deleted_disks", + "controller_terminal_state_sha256", + "native_auth_revalidated", + "expected_instances", + "remaining_instances", + "expected_disks", + "remaining_disks", + "remaining_firewalls", + "l4_usage", + "protected_bootstrap_running", + "product_processes_remaining", + "temporary_credentials_remaining", +} +_TERMINAL_STATE_FIELDS = { + "schema_version", + "run_id", + "authorization_sha256", + "provider_plan_digest", + "revision", + "phase", + "failure_code", + "windows_evidence_digest", + "linux_evidence_digest", + "windows_consumed", + "linux_consumed", + "cleanup_verified", + "next_action", +} +_AUTH_FIELDS = { + "schema_version", + "gate", + "result", + "run_id", + "source_commit", + "provider_plan_digest", + "provider_plan", + "authorization", + "prohibited", +} +_AUTHORIZATION_FIELDS = { + "combined_cloud_ceiling_usd", + "ledger_committed_before_run_usd", + "maximum_estimate_usd", + "remaining_after_run_maximum_usd", + "reservation_recorded", + "native_auth_revalidated", + "provisioning_authorized_after_fail_closed_preflight", +} +_PLAN_FIELDS = {"project", "zone", "clients", "sequencing"} +_CLIENT_PLAN_FIELDS = { + "platform", + "instance", + "disk", + "source_commit", + "termination_unix", + "package_sha256", + "model_id", + "manifest_digest", +} +_SEQUENCING_FIELDS = { + "clients_may_run_concurrently", + "windows_first", + "fresh_host_per_platform", +} + + +class Gate14EvidenceError(ValueError): + """A Gate 14 input was malformed, unsafe, incomplete, or inconsistent.""" + + +def _reject_constant(_value: str) -> None: + raise Gate14EvidenceError("invalid JSON") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Gate14EvidenceError("duplicate JSON field") + result[key] = value + return result + + +def _regular_bytes(path: Path) -> bytes: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise Gate14EvidenceError("required evidence is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if ( + reparse + or path.is_symlink() + or not stat.S_ISREG(metadata.st_mode) + or not 1 <= metadata.st_size <= MAX_INPUT_BYTES + ): + raise Gate14EvidenceError("required evidence is unsafe") + try: + return path.read_bytes() + except OSError as exc: + raise Gate14EvidenceError("required evidence is unreadable") from exc + + +def _strict_json(payload: bytes) -> Mapping[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= MAX_INPUT_BYTES: + raise Gate14EvidenceError("JSON size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Gate14EvidenceError("invalid JSON") from exc + if not isinstance(value, dict): + raise Gate14EvidenceError("JSON root is invalid") + return value + + +def _mapping(value: Any, fields: set[str]) -> Mapping[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise Gate14EvidenceError("evidence schema is invalid") + return value + + +def _true(value: Any) -> None: + if value is not True: + raise Gate14EvidenceError("required proof is absent") + + +def _false(value: Any) -> None: + if value is not False: + raise Gate14EvidenceError("forbidden retention or result is present") + + +def _integer(value: Any, minimum: int = 0, maximum: int = 2**63 - 1) -> int: + if type(value) is not int or not minimum <= value <= maximum: + raise Gate14EvidenceError("integer evidence is invalid") + return value + + +def _number(value: Any, minimum: float, maximum: float) -> float: + if type(value) not in (int, float): + raise Gate14EvidenceError("numeric evidence is invalid") + rendered = float(value) + if not math.isfinite(rendered) or not minimum <= rendered <= maximum: + raise Gate14EvidenceError("numeric evidence is invalid") + return rendered + + +def _string(value: Any, pattern: re.Pattern[str]) -> str: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise Gate14EvidenceError("string evidence is invalid") + return value + + +def _validate_package(value: Any, source_commit: str) -> Mapping[str, Any]: + package = _mapping(value, _PACKAGE_FIELDS) + if package["source_commit"] != source_commit: + raise Gate14EvidenceError("package source is inconsistent") + _string(package["source_commit"], _COMMIT_RE) + _string(package["archive_sha256"], _DIGEST_RE) + _integer(package["archive_bytes"], 1, 8 * 1024**3) + _string(package["release_metadata_sha256"], _DIGEST_RE) + return package + + +def _validate_model(value: Any, platform: str) -> Mapping[str, Any]: + model = _mapping(value, _MODEL_FIELDS) + model_id = model["id"] + if model_id != EXPECTED_PLATFORM_MODELS[platform]: + raise Gate14EvidenceError("platform model is invalid") + profile = MODEL_PROFILES[model_id] + for field in ( + "manifest_digest", + "revision_commit", + "selected_artifact_count", + "selected_artifact_bytes", + "total_blocks", + ): + if model[field] != profile[field]: + raise Gate14EvidenceError("model identity is inconsistent") + if model["gate9_envelope_sha256"] != EXPECTED_GATE9_ENVELOPES[platform]: + raise Gate14EvidenceError("Gate 9 envelope is inconsistent") + return model + + +def _validate_hardware(value: Any, platform: str) -> Mapping[str, Any]: + hardware = _mapping(value, _HARDWARE_FIELDS) + _string(hardware["os_name"], _OS_RE) + if hardware["os_name"] != EXPECTED_PLATFORM_OS[platform]: + raise Gate14EvidenceError("platform operating system is inconsistent") + if hardware["accelerator"] != "NVIDIA L4": + raise Gate14EvidenceError("real L4 hardware is required") + if hardware["accelerator_count"] != 1: + raise Gate14EvidenceError("exactly one accelerator is required") + _integer(hardware["accelerator_memory_bytes"], 20 * 1024**3, 32 * 1024**3) + return hardware + + +def _validate_cache(value: Any, selected_bytes: int) -> None: + cache = _mapping(value, _CACHE_FIELDS) + if ( + cache["verified_bytes_before"] != selected_bytes + or cache["verified_bytes_after"] != selected_bytes + or cache["transfer_bytes_during_gate"] != 0 + or cache["digest_mismatch_count"] != 0 + ): + raise Gate14EvidenceError("verified cache reuse is inconsistent") + _false(cache["forbidden_model_acquired"]) + + +def _validate_placement(value: Any, total_blocks: int) -> tuple[int, int]: + placement = _mapping(value, _PLACEMENT_FIELDS) + _true(placement["automatic"]) + if placement["worker_count"] != 1: + raise Gate14EvidenceError("exactly one automatic worker is required") + start = _integer(placement["block_start"], 0, total_blocks - 1) + end = _integer(placement["block_end"], 1, total_blocks) + if end <= start: + raise Gate14EvidenceError("automatic block range is empty") + _true(placement["intent_published"]) + _true(placement["remote_acknowledged"]) + return start, end + + +def _validate_limits(value: Any, selected_bytes: int, accelerator_memory: int) -> None: + limits = _mapping(value, _LIMIT_FIELDS) + disk = _integer(limits["disk_bytes"], selected_bytes, MAX_BYTES) + vram = _integer(limits["vram_bytes"], 1, accelerator_memory) + if disk < selected_bytes or vram >= accelerator_memory: + raise Gate14EvidenceError("resource ceilings are not bounded") + _number(limits["bandwidth_mbps"], 0.001, 1_000_000.0) + _number(limits["power_watts"], 0.001, 1_000.0) + if limits["schedule_timezone"] != "UTC" or limits["resource_limit_count"] != 5: + raise Gate14EvidenceError("all five resource classes are required") + _true(limits["configured_and_resolved_match"]) + _true(limits["low_vram_rejected"]) + + +def _validate_suspensions(value: Any) -> None: + if not isinstance(value, list) or len(value) != 3: + raise Gate14EvidenceError("three suspension classes are required") + seen: set[str] = set() + for raw in value: + item = _mapping(raw, _SUSPENSION_FIELDS) + kind = item["kind"] + if kind not in {"bandwidth", "power", "schedule"} or kind in seen: + raise Gate14EvidenceError("suspension class is invalid") + seen.add(kind) + _true(item["suspended"]) + _true(item["resumed"]) + _true(item["desired_intent_preserved"]) + if item["worker_count_during"] != 0: + raise Gate14EvidenceError("worker remained active while suspended") + _number(item["duration_seconds"], 0.0, MAX_DURATION_SECONDS) + + +def _validate_recovery(value: Any) -> None: + recovery = _mapping(value, _RECOVERY_FIELDS) + for field in _RECOVERY_FIELDS - {"restart_seconds"}: + _true(recovery[field]) + _number(recovery["restart_seconds"], 0.0, MAX_DURATION_SECONDS) + + +def _validate_pause(value: Any) -> None: + pause = _mapping(value, _PAUSE_FIELDS) + _true(pause["requested"]) + _true(pause["completed"]) + _number(pause["duration_seconds"], 0.0, MAX_DURATION_SECONDS) + if pause["worker_count_after"] != 0 or pause["descendant_count_after"] != 0: + raise Gate14EvidenceError("pause cleanup is incomplete") + + +def _validate_restart(value: Any) -> None: + restart = _mapping(value, _RESTART_FIELDS) + for field in _RESTART_FIELDS - {"duration_seconds"}: + _true(restart[field]) + _number(restart["duration_seconds"], 0.0, MAX_DURATION_SECONDS) + + +def _validate_unsupported(value: Any) -> None: + unsupported = _mapping(value, _UNSUPPORTED_FIELDS) + if ( + unsupported["device"] != "cpu" + or unsupported["configured_limit"] != "power_watts" + or unsupported["reason_code"] != "power-telemetry-unavailable" + ): + raise Gate14EvidenceError("unsupported telemetry classification is invalid") + _true(unsupported["start_rejected"]) + _false(unsupported["private_detail_retained"]) + + +def _validate_privacy(value: Any) -> None: + privacy = _mapping(value, _PRIVACY_FIELDS) + for field in _PRIVACY_FIELDS: + _false(privacy[field]) + + +def validate_platform_document(value: Mapping[str, Any]) -> Mapping[str, Any]: + document = _mapping(value, _DOCUMENT_FIELDS) + if ( + document["schema_version"] != SCHEMA_VERSION + or document["scope"] != PLATFORM_SCOPE + or document["result"] != "passed" + ): + raise Gate14EvidenceError("platform evidence header is invalid") + run_id = _string(document["run_id"], _RUN_RE) + platform = document["platform"] + if platform not in EXPECTED_PLATFORM_MODELS: + raise Gate14EvidenceError("platform is invalid") + source_commit = _string(document["source_commit"], _COMMIT_RE) + gate13_evidence_sha256 = _string(document["gate13_evidence_sha256"], _DIGEST_RE) + if gate13_evidence_sha256 != EXPECTED_GATE13_EVIDENCE_SHA256: + raise Gate14EvidenceError("Gate 13 lifecycle evidence is inconsistent") + package = _validate_package(document["package"], source_commit) + model = _validate_model(document["model"], platform) + hardware = _validate_hardware(document["hardware"], platform) + _validate_cache(document["cache"], model["selected_artifact_bytes"]) + block_start, block_end = _validate_placement(document["placement"], model["total_blocks"]) + _validate_limits( + document["limits"], + model["selected_artifact_bytes"], + hardware["accelerator_memory_bytes"], + ) + _validate_suspensions(document["suspensions"]) + _validate_recovery(document["recovery"]) + _validate_pause(document["pause"]) + _validate_restart(document["restart"]) + _validate_unsupported(document["unsupported_telemetry"]) + _validate_privacy(document["privacy"]) + _true(document["qualification_temporaries_removed"]) + return { + "run_id": run_id, + "platform": platform, + "source_commit": source_commit, + "gate13_evidence_sha256": gate13_evidence_sha256, + "package_sha256": package["archive_sha256"], + "model_id": model["id"], + "manifest_digest": model["manifest_digest"], + "gate9_envelope_sha256": model["gate9_envelope_sha256"], + "accelerator": hardware["accelerator"], + "block_start": block_start, + "block_end": block_end, + } + + +def _digest(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _resource_names(value: Sequence[str], field: str) -> tuple[str, str]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence) or len(value) != 2: + raise Gate14EvidenceError(f"{field} inventory is invalid") + names = tuple(_string(item, _NAME_RE) for item in value) + if len(set(names)) != 2: + raise Gate14EvidenceError(f"{field} inventory is not unique") + if PROTECTED_INSTANCE in names: + raise Gate14EvidenceError("protected resource is targeted") + return names + + +def validate_authorization_document( + value: Mapping[str, Any], + *, + run_id: str, + source_commit: str, + provider_plan_digest: str, + project: str, + zone: str, + expected_instances: Sequence[str], + expected_disks: Sequence[str], + package_sha256: Mapping[str, str], +) -> Mapping[str, Any]: + authorization = _mapping(value, _AUTH_FIELDS) + if ( + authorization["schema_version"] != SCHEMA_VERSION + or authorization["gate"] != 14 + or authorization["result"] != "authorized" + or authorization["run_id"] != run_id + or authorization["source_commit"] != source_commit + or authorization["provider_plan_digest"] != provider_plan_digest + ): + raise Gate14EvidenceError("authorization binding is invalid") + + provider_plan = _mapping(authorization["provider_plan"], _PLAN_FIELDS) + canonical_plan = json.dumps(provider_plan, sort_keys=True, separators=(",", ":")).encode("utf-8") + if ( + _digest(canonical_plan) != provider_plan_digest + or provider_plan["project"] != project + or provider_plan["zone"] != zone + ): + raise Gate14EvidenceError("authorized provider plan is inconsistent") + sequencing = _mapping(provider_plan["sequencing"], _SEQUENCING_FIELDS) + if ( + sequencing["clients_may_run_concurrently"] is not False + or sequencing["windows_first"] is not True + or sequencing["fresh_host_per_platform"] is not True + ): + raise Gate14EvidenceError("authorized sequencing is inconsistent") + clients = provider_plan["clients"] + if not isinstance(clients, list) or len(clients) != 2: + raise Gate14EvidenceError("authorized client plan is invalid") + by_platform = { + item.get("platform"): item + for item in clients + if isinstance(item, dict) and isinstance(item.get("platform"), str) + } + if set(by_platform) != {"windows", "linux"}: + raise Gate14EvidenceError("authorized platform plan is invalid") + for index, platform in enumerate(("windows", "linux")): + client = _mapping(by_platform[platform], _CLIENT_PLAN_FIELDS) + model_id = EXPECTED_PLATFORM_MODELS[platform] + if ( + client["platform"] != platform + or client["instance"] != expected_instances[index] + or client["disk"] != expected_disks[index] + or client["source_commit"] != source_commit + or client["package_sha256"] != package_sha256[platform] + or client["model_id"] != model_id + or client["manifest_digest"] != MODEL_PROFILES[model_id]["manifest_digest"] + ): + raise Gate14EvidenceError("authorized client binding is inconsistent") + _integer(client["termination_unix"], 1) + + cost = _mapping(authorization["authorization"], _AUTHORIZATION_FIELDS) + try: + ceiling = Decimal(str(cost["combined_cloud_ceiling_usd"])) + before = Decimal(str(cost["ledger_committed_before_run_usd"])) + maximum = Decimal(str(cost["maximum_estimate_usd"])) + remaining = Decimal(str(cost["remaining_after_run_maximum_usd"])) + except (InvalidOperation, TypeError, ValueError) as exc: + raise Gate14EvidenceError("cost authorization is invalid") from exc + if ( + (ceiling, before, maximum, remaining) + != (Decimal("100.00"), Decimal("56.00"), Decimal("44.00"), Decimal("0.00")) + or not all(item.is_finite() for item in (ceiling, before, maximum, remaining)) + or cost["reservation_recorded"] is not True + or cost["native_auth_revalidated"] is not True + or cost["provisioning_authorized_after_fail_closed_preflight"] is not True + ): + raise Gate14EvidenceError("cost authorization is inconsistent") + prohibited = authorization["prohibited"] + if ( + not isinstance(prohibited, dict) + or set(prohibited) != {"credits", "macos", "fly_gpu"} + or any(type(item) is not int or item != 0 for item in prohibited.values()) + ): + raise Gate14EvidenceError("prohibited work is present") + return authorization + + +def validate_terminal_state( + value: Mapping[str, Any], + *, + run_id: str, + authorization_sha256: str, + provider_plan_digest: str, + windows_evidence_sha256: str, + linux_evidence_sha256: str, +) -> Mapping[str, Any]: + state = _mapping(value, _TERMINAL_STATE_FIELDS) + if ( + state["schema_version"] != SCHEMA_VERSION + or state["run_id"] != run_id + or state["authorization_sha256"] != authorization_sha256 + or state["provider_plan_digest"] != provider_plan_digest + or state["phase"] != "CLEANED_PASS" + or state["failure_code"] is not None + or state["windows_evidence_digest"] != windows_evidence_sha256 + or state["linux_evidence_digest"] != linux_evidence_sha256 + or state["windows_consumed"] is not True + or state["linux_consumed"] is not True + or state["cleanup_verified"] is not True + or state["next_action"] != "none" + ): + raise Gate14EvidenceError("controller terminal state is inconsistent") + _integer(state["revision"], 1) + return state + + +def validate_cleanup_document( + value: Mapping[str, Any], + *, + run_id: str, + controller_source_commit: str, + provider_plan_digest: str, + project: str, + zone: str, + expected_instances: Sequence[str], + expected_disks: Sequence[str], + terminal_state_sha256: str, +) -> Mapping[str, Any]: + cleanup = _mapping(value, _CLEANUP_FIELDS) + if ( + cleanup["schema_version"] != SCHEMA_VERSION + or cleanup["scope"] != CLEANUP_SCOPE + or cleanup["run_id"] != run_id + or cleanup["result"] != "passed" + or cleanup["provider"] != "GCP" + or cleanup["controller_source_commit"] != controller_source_commit + or cleanup["provider_plan_digest"] != provider_plan_digest + or cleanup["project"] != project + or cleanup["zone"] != zone + or cleanup["deleted_instances"] != list(expected_instances) + or cleanup["deleted_disks"] != list(expected_disks) + or cleanup["controller_terminal_state_sha256"] != terminal_state_sha256 + ): + raise Gate14EvidenceError("cleanup evidence binding is invalid") + _true(cleanup["native_auth_revalidated"]) + if cleanup["expected_instances"] != 2 or cleanup["expected_disks"] != 2: + raise Gate14EvidenceError("cleanup target count is invalid") + for field in ( + "remaining_instances", + "remaining_disks", + "remaining_firewalls", + "l4_usage", + "product_processes_remaining", + "temporary_credentials_remaining", + ): + if cleanup[field] != 0 or type(cleanup[field]) is not int: + raise Gate14EvidenceError("cleanup is incomplete") + _true(cleanup["protected_bootstrap_running"]) + return cleanup + + +def validate_files( + windows_path: Path, + linux_path: Path, + cleanup_path: Path, + controller_source_commit: str, + *, + provider_plan_digest: str, + project: str, + zone: str, + expected_instances: Sequence[str], + expected_disks: Sequence[str], + terminal_state_path: Path, + authorization_path: Path, +) -> Mapping[str, Any]: + controller_source_commit = _string(controller_source_commit, _COMMIT_RE) + provider_plan_digest = _string(provider_plan_digest, _DIGEST_RE) + project = _string(project, _PROJECT_RE) + zone = _string(zone, _ZONE_RE) + expected_instances = _resource_names(expected_instances, "instance") + expected_disks = _resource_names(expected_disks, "disk") + payloads = { + "windows": _regular_bytes(windows_path), + "linux": _regular_bytes(linux_path), + "cleanup": _regular_bytes(cleanup_path), + "terminal_state": _regular_bytes(terminal_state_path), + "authorization": _regular_bytes(authorization_path), + } + windows = validate_platform_document(_strict_json(payloads["windows"])) + linux = validate_platform_document(_strict_json(payloads["linux"])) + if windows["platform"] != "windows" or linux["platform"] != "linux": + raise Gate14EvidenceError("platform evidence ordering is invalid") + if windows["run_id"] != linux["run_id"]: + raise Gate14EvidenceError("run identity is inconsistent") + if windows["source_commit"] != linux["source_commit"] or windows["source_commit"] != controller_source_commit: + raise Gate14EvidenceError("package source identity is inconsistent") + authorization_sha256 = _digest(payloads["authorization"]) + validate_authorization_document( + _strict_json(payloads["authorization"]), + run_id=windows["run_id"], + source_commit=controller_source_commit, + provider_plan_digest=provider_plan_digest, + project=project, + zone=zone, + expected_instances=expected_instances, + expected_disks=expected_disks, + package_sha256={ + "windows": windows["package_sha256"], + "linux": linux["package_sha256"], + }, + ) + terminal_state_sha256 = _digest(payloads["terminal_state"]) + validate_terminal_state( + _strict_json(payloads["terminal_state"]), + run_id=windows["run_id"], + authorization_sha256=authorization_sha256, + provider_plan_digest=provider_plan_digest, + windows_evidence_sha256=_digest(payloads["windows"]), + linux_evidence_sha256=_digest(payloads["linux"]), + ) + cleanup = validate_cleanup_document( + _strict_json(payloads["cleanup"]), + run_id=windows["run_id"], + controller_source_commit=controller_source_commit, + provider_plan_digest=provider_plan_digest, + project=project, + zone=zone, + expected_instances=expected_instances, + expected_disks=expected_disks, + terminal_state_sha256=terminal_state_sha256, + ) + return { + "schema_version": SCHEMA_VERSION, + "scope": AGGREGATE_SCOPE, + "run_id": windows["run_id"], + "result": "passed", + "controller_source_commit": controller_source_commit, + "package_source_commit": windows["source_commit"], + "provider_plan_digest": provider_plan_digest, + "authorization_sha256": authorization_sha256, + "platforms": [ + { + **windows, + "evidence_sha256": _digest(payloads["windows"]), + }, + { + **linux, + "evidence_sha256": _digest(payloads["linux"]), + }, + ], + "cleanup": { + "evidence_sha256": _digest(payloads["cleanup"]), + "provider": cleanup["provider"], + "project": project, + "zone": zone, + "deleted_instances": list(expected_instances), + "deleted_disks": list(expected_disks), + "terminal_state_sha256": terminal_state_sha256, + "resource_absence_proved": True, + "protected_bootstrap_running": True, + }, + "credits_in_scope": False, + "macos_in_scope": False, + "privacy_safe": True, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--windows", type=Path, required=True) + parser.add_argument("--linux", type=Path, required=True) + parser.add_argument("--cleanup", type=Path, required=True) + parser.add_argument("--controller-state", type=Path, required=True) + parser.add_argument("--authorization", type=Path, required=True) + parser.add_argument("--controller-source-commit", required=True) + parser.add_argument("--provider-plan-digest", required=True) + parser.add_argument("--project", required=True) + parser.add_argument("--zone", required=True) + parser.add_argument("--instances", nargs=2, required=True) + parser.add_argument("--disks", nargs=2, required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + result = validate_files( + args.windows, + args.linux, + args.cleanup, + args.controller_source_commit, + provider_plan_digest=args.provider_plan_digest, + project=args.project, + zone=args.zone, + expected_instances=args.instances, + expected_disks=args.disks, + terminal_state_path=args.controller_state, + authorization_path=args.authorization, + ) + except Gate14EvidenceError as exc: + raise SystemExit(str(exc)) from exc + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gate14_run_controller.py b/scripts/gate14_run_controller.py new file mode 100644 index 000000000..cf6c6f9d0 --- /dev/null +++ b/scripts/gate14_run_controller.py @@ -0,0 +1,881 @@ +"""Durable, source-bound controller for one bounded Gate 14 GCP run. + +The controller never invokes a provider. Every start, status, collect, or cleanup +operation first consumes an exact provider observation, persists its decision, and +returns one allowlisted action. A provider adapter may execute only that action. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import stat +import tempfile +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Mapping, Sequence + +import gate14_hardware_acceptance as acceptance +import qualification_cost_guard as cost_guard + +SCHEMA_VERSION = 1 +STATE_SCHEMA_VERSION = 1 +MAX_JSON_BYTES = 262_144 +PROTECTED_INSTANCE = "communityai-bootstrap-1" +ALLOWED_CEILING_USD = 100.0 +CURRENT_EPOCH_ANCHOR_RUN_ID = "gate13-20260901-a" +CURRENT_EPOCH_ANCHOR_MAXIMUM_USD = Decimal("56.00") + +_RUN_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +_NAME_RE = re.compile(r"[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?") +_COMMIT_RE = re.compile(r"[0-9a-f]{40}") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}") + +PHASES = { + "ABSENT", + "WINDOWS_RUNNING", + "WINDOWS_DELETING", + "LINUX_RUNNING", + "LINUX_DELETING", + "CLEANING_FAILED", + "CLEANED_PASS", + "CLEANED_FAILURE", +} +TERMINAL_PHASES = {"CLEANED_PASS", "CLEANED_FAILURE"} +ACTIONS = { + "start_windows", + "collect_windows", + "delete_windows", + "start_linux", + "collect_linux", + "delete_linux", + "cleanup_failure", + "none", +} +JOB_STATES = {"absent", "starting", "running", "passed", "failed", "ambiguous"} + +_AUTH_FIELDS = { + "schema_version", + "gate", + "result", + "run_id", + "source_commit", + "provider_plan_digest", + "provider_plan", + "authorization", + "prohibited", +} +_AUTHORIZATION_FIELDS = { + "combined_cloud_ceiling_usd", + "ledger_committed_before_run_usd", + "maximum_estimate_usd", + "remaining_after_run_maximum_usd", + "reservation_recorded", + "native_auth_revalidated", + "provisioning_authorized_after_fail_closed_preflight", +} +_PLAN_FIELDS = {"project", "zone", "clients", "sequencing"} +_CLIENT_PLAN_FIELDS = { + "platform", + "instance", + "disk", + "source_commit", + "termination_unix", + "package_sha256", + "model_id", + "manifest_digest", +} +_SEQUENCING_FIELDS = { + "clients_may_run_concurrently", + "windows_first", + "fresh_host_per_platform", +} +_STATE_FIELDS = { + "schema_version", + "run_id", + "authorization_sha256", + "provider_plan_digest", + "revision", + "phase", + "failure_code", + "windows_evidence_digest", + "linux_evidence_digest", + "windows_consumed", + "linux_consumed", + "cleanup_verified", + "next_action", +} +_OBSERVATION_FIELDS = { + "schema_version", + "run_id", + "observed_at_unix", + "instances", + "disks", + "clients", + "l4_usage", + "protected_bootstrap_running", +} +_INSTANCE_FIELDS = {"present", "run_id", "source_commit", "termination_unix"} +_CLIENT_FIELDS = {"job_state", "attempt_ordinal", "evidence_digest"} + + +class Gate14ControllerError(ValueError): + """The plan, state, observation, or transition failed closed.""" + + +@dataclass(frozen=True) +class ClientPlan: + platform: str + instance: str + disk: str + source_commit: str + termination_unix: int + package_sha256: str + model_id: str + manifest_digest: str + + +@dataclass(frozen=True) +class RunPlan: + run_id: str + authorization_sha256: str + provider_plan_digest: str + source_commit: str + ledger_state: str + project: str + zone: str + windows: ClientPlan + linux: ClientPlan + + @property + def instances(self) -> tuple[str, str]: + return (self.windows.instance, self.linux.instance) + + @property + def disks(self) -> tuple[str, str]: + return (self.windows.disk, self.linux.disk) + + +def _reject_constant(_value: str) -> None: + raise Gate14ControllerError("invalid JSON") + + +def _unique_object(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise Gate14ControllerError("duplicate JSON field") + result[key] = value + return result + + +def _regular_bytes(path: Path, maximum: int = MAX_JSON_BYTES) -> bytes: + path = Path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise Gate14ControllerError("required file is unavailable") from exc + reparse = bool(getattr(metadata, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if reparse or path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= maximum: + raise Gate14ControllerError("required file is unsafe") + try: + return path.read_bytes() + except OSError as exc: + raise Gate14ControllerError("required file is unreadable") from exc + + +def _strict_json(payload: bytes) -> Mapping[str, Any]: + if not isinstance(payload, bytes) or not 1 <= len(payload) <= MAX_JSON_BYTES: + raise Gate14ControllerError("JSON size is invalid") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Gate14ControllerError("invalid JSON") from exc + if not isinstance(value, dict): + raise Gate14ControllerError("JSON root is invalid") + return value + + +def _mapping(value: Any, fields: set[str]) -> Mapping[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise Gate14ControllerError("schema is invalid") + return value + + +def _string(value: Any, pattern: re.Pattern[str]) -> str: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise Gate14ControllerError("string is invalid") + return value + + +def _integer(value: Any, minimum: int = 0, maximum: int = 2**63 - 1) -> int: + if type(value) is not int or not minimum <= value <= maximum: + raise Gate14ControllerError("integer is invalid") + return value + + +def _canonical_digest(value: Mapping[str, Any]) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _client_plan(value: Any, platform: str, source_commit: str) -> ClientPlan: + raw = _mapping(value, _CLIENT_PLAN_FIELDS) + if raw["platform"] != platform or raw["source_commit"] != source_commit: + raise Gate14ControllerError("client source binding is invalid") + instance = _string(raw["instance"], _NAME_RE) + disk = _string(raw["disk"], _NAME_RE) + if instance == PROTECTED_INSTANCE or disk == PROTECTED_INSTANCE: + raise Gate14ControllerError("protected resource is targeted") + package_sha256 = _string(raw["package_sha256"], _DIGEST_RE) + expected_model = acceptance.EXPECTED_PLATFORM_MODELS[platform] + if raw["model_id"] != expected_model: + raise Gate14ControllerError("client model is invalid") + expected_manifest = acceptance.MODEL_PROFILES[expected_model]["manifest_digest"] + if raw["manifest_digest"] != expected_manifest: + raise Gate14ControllerError("client manifest is invalid") + return ClientPlan( + platform=platform, + instance=instance, + disk=disk, + source_commit=source_commit, + termination_unix=_integer(raw["termination_unix"], 1), + package_sha256=package_sha256, + model_id=expected_model, + manifest_digest=expected_manifest, + ) + + +def load_plan(authorization_path: Path, ledger_path: Path) -> RunPlan: + authorization_payload = _regular_bytes(authorization_path) + raw = _mapping(_strict_json(authorization_payload), _AUTH_FIELDS) + if raw["schema_version"] != SCHEMA_VERSION or raw["gate"] != 14 or raw["result"] != "authorized": + raise Gate14ControllerError("authorization scope is invalid") + run_id = _string(raw["run_id"], _RUN_RE) + source_commit = _string(raw["source_commit"], _COMMIT_RE) + provider_plan = _mapping(raw["provider_plan"], _PLAN_FIELDS) + provider_digest = _canonical_digest(provider_plan) + if raw["provider_plan_digest"] != provider_digest: + raise Gate14ControllerError("provider plan digest changed") + project = _string(provider_plan["project"], re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]")) + zone = _string(provider_plan["zone"], re.compile(r"[a-z]+(?:-[a-z0-9]+)+-[a-z]")) + sequencing = _mapping(provider_plan["sequencing"], _SEQUENCING_FIELDS) + if ( + sequencing["clients_may_run_concurrently"] is not False + or sequencing["windows_first"] is not True + or sequencing["fresh_host_per_platform"] is not True + ): + raise Gate14ControllerError("client sequencing is invalid") + clients = provider_plan["clients"] + if not isinstance(clients, list) or len(clients) != 2: + raise Gate14ControllerError("client plan is invalid") + by_platform = { + item.get("platform"): item + for item in clients + if isinstance(item, dict) and isinstance(item.get("platform"), str) + } + if set(by_platform) != {"windows", "linux"}: + raise Gate14ControllerError("client platform plan is invalid") + windows = _client_plan(by_platform["windows"], "windows", source_commit) + linux = _client_plan(by_platform["linux"], "linux", source_commit) + if windows.instance == linux.instance or windows.disk == linux.disk: + raise Gate14ControllerError("client resources overlap") + + cost = _mapping(raw["authorization"], _AUTHORIZATION_FIELDS) + try: + ceiling = Decimal(str(cost["combined_cloud_ceiling_usd"])) + before = Decimal(str(cost["ledger_committed_before_run_usd"])) + maximum = Decimal(str(cost["maximum_estimate_usd"])) + remaining = Decimal(str(cost["remaining_after_run_maximum_usd"])) + except (InvalidOperation, TypeError, ValueError) as exc: + raise Gate14ControllerError("cost authorization is invalid") from exc + if ( + not all(value.is_finite() for value in (ceiling, before, maximum, remaining)) + or ceiling != Decimal(str(ALLOWED_CEILING_USD)) + or before < 0 + or maximum <= 0 + or before + maximum > ceiling + or ceiling - before - maximum != remaining + or cost["reservation_recorded"] is not True + or cost["native_auth_revalidated"] is not True + or cost["provisioning_authorized_after_fail_closed_preflight"] is not True + ): + raise Gate14ControllerError("cost authorization is inconsistent") + prohibited = raw["prohibited"] + if ( + not isinstance(prohibited, dict) + or set(prohibited) != {"credits", "macos", "fly_gpu"} + or any(type(value) is not int or value != 0 for value in prohibited.values()) + ): + raise Gate14ControllerError("prohibited work is present") + try: + entries = cost_guard.load_spend_ledger(ledger_path) + except cost_guard.CostGuardError as exc: + raise Gate14ControllerError("spend ledger is invalid") from exc + anchors = [entry for entry in entries if entry.run_id == CURRENT_EPOCH_ANCHOR_RUN_ID] + if len(anchors) != 1 or anchors[0].maximum_usd != CURRENT_EPOCH_ANCHOR_MAXIMUM_USD: + raise Gate14ControllerError("current accounting epoch anchor is invalid") + anchor_index = entries.index(anchors[0]) + historical_entries = entries[anchor_index + 1 :] + if any(entry.state not in {"CANCELED", "CLEANED-COMMITTED", "CLEANED-RELEASED"} for entry in historical_entries): + raise Gate14ControllerError("active reservation is hidden below the epoch anchor") + current_epoch_entries = entries[: anchor_index + 1] + matches = [entry for entry in current_epoch_entries if entry.run_id == run_id] + if ( + len(matches) != 1 + or matches[0].provider != "GCP" + or matches[0].maximum_usd != maximum + or matches[0].state != "RESERVED" + or provider_digest not in matches[0].purpose + ): + raise Gate14ControllerError("spend ledger reservation is invalid") + ledger_committed = sum( + (entry.committed_usd for entry in current_epoch_entries), + Decimal("0"), + ) + committed_before = ledger_committed - matches[0].committed_usd + if committed_before != before or ledger_committed > ceiling: + raise Gate14ControllerError("spend ledger exceeds the authorized ceiling") + return RunPlan( + run_id=run_id, + authorization_sha256="sha256:" + hashlib.sha256(authorization_payload).hexdigest(), + provider_plan_digest=provider_digest, + source_commit=source_commit, + ledger_state=matches[0].state, + project=project, + zone=zone, + windows=windows, + linux=linux, + ) + + +def initial_state(plan: RunPlan) -> dict[str, Any]: + return { + "schema_version": STATE_SCHEMA_VERSION, + "run_id": plan.run_id, + "authorization_sha256": plan.authorization_sha256, + "provider_plan_digest": plan.provider_plan_digest, + "revision": 0, + "phase": "ABSENT", + "failure_code": None, + "windows_evidence_digest": None, + "linux_evidence_digest": None, + "windows_consumed": False, + "linux_consumed": False, + "cleanup_verified": False, + "next_action": "none", + } + + +def validate_state(value: Mapping[str, Any], plan: RunPlan) -> dict[str, Any]: + state = dict(_mapping(value, _STATE_FIELDS)) + if ( + state["schema_version"] != STATE_SCHEMA_VERSION + or state["run_id"] != plan.run_id + or state["authorization_sha256"] != plan.authorization_sha256 + or state["provider_plan_digest"] != plan.provider_plan_digest + or state["phase"] not in PHASES + or state["next_action"] not in ACTIONS + ): + raise Gate14ControllerError("state binding is invalid") + _integer(state["revision"]) + for field in ("windows_consumed", "linux_consumed", "cleanup_verified"): + if type(state[field]) is not bool: + raise Gate14ControllerError("state boolean is invalid") + for field in ("windows_evidence_digest", "linux_evidence_digest"): + if state[field] is not None: + _string(state[field], _DIGEST_RE) + if state["failure_code"] is not None: + _string(state["failure_code"], re.compile(r"[a-z0-9][a-z0-9-]{0,63}")) + allowed_actions = { + "ABSENT": {"none", "start_windows"}, + "WINDOWS_RUNNING": {"none", "collect_windows"}, + "WINDOWS_DELETING": {"delete_windows", "start_linux"}, + "LINUX_RUNNING": {"none", "collect_linux"}, + "LINUX_DELETING": {"delete_linux"}, + "CLEANING_FAILED": {"cleanup_failure"}, + "CLEANED_PASS": {"none"}, + "CLEANED_FAILURE": {"none"}, + } + if state["next_action"] not in allowed_actions[state["phase"]]: + raise Gate14ControllerError("state action is inconsistent") + if state["windows_evidence_digest"] is not None and not state["windows_consumed"]: + raise Gate14ControllerError("Windows evidence state is inconsistent") + if state["linux_evidence_digest"] is not None and not state["linux_consumed"]: + raise Gate14ControllerError("Linux evidence state is inconsistent") + phase = state["phase"] + failed_phase = phase in {"CLEANING_FAILED", "CLEANED_FAILURE"} + if failed_phase is (state["failure_code"] is None): + raise Gate14ControllerError("failure state is inconsistent") + if state["cleanup_verified"] is not (phase in TERMINAL_PHASES): + raise Gate14ControllerError("cleanup state is inconsistent") + windows_evidence = state["windows_evidence_digest"] is not None + linux_evidence = state["linux_evidence_digest"] is not None + if phase == "ABSENT" and any( + (state["windows_consumed"], state["linux_consumed"], windows_evidence, linux_evidence) + ): + raise Gate14ControllerError("initial state is inconsistent") + if phase == "WINDOWS_RUNNING" and ( + not state["windows_consumed"] + or state["linux_consumed"] + or linux_evidence + or (state["next_action"] == "collect_windows") is not windows_evidence + ): + raise Gate14ControllerError("Windows running state is inconsistent") + if phase == "WINDOWS_DELETING" and ( + not state["windows_consumed"] or state["linux_consumed"] or not windows_evidence or linux_evidence + ): + raise Gate14ControllerError("Windows deletion state is inconsistent") + if phase == "LINUX_RUNNING" and ( + not state["windows_consumed"] + or not state["linux_consumed"] + or not windows_evidence + or (state["next_action"] == "collect_linux") is not linux_evidence + ): + raise Gate14ControllerError("Linux running state is inconsistent") + if phase in {"LINUX_DELETING", "CLEANED_PASS"} and ( + not state["windows_consumed"] or not state["linux_consumed"] or not windows_evidence or not linux_evidence + ): + raise Gate14ControllerError("completed evidence state is inconsistent") + return state + + +def validate_observation(value: Mapping[str, Any], plan: RunPlan) -> dict[str, Any]: + observation = dict(_mapping(value, _OBSERVATION_FIELDS)) + if observation["schema_version"] != SCHEMA_VERSION or observation["run_id"] != plan.run_id: + raise Gate14ControllerError("observation binding is invalid") + now = _integer(observation["observed_at_unix"], 1) + if observation["protected_bootstrap_running"] is not True: + raise Gate14ControllerError("protected bootstrap is not healthy") + _integer(observation["l4_usage"], 0, 1) + instances = observation["instances"] + disks = observation["disks"] + clients = observation["clients"] + if ( + not isinstance(instances, dict) + or set(instances) != set(plan.instances) + or not isinstance(disks, dict) + or set(disks) != set(plan.disks) + or not isinstance(clients, dict) + or set(clients) != {"windows", "linux"} + ): + raise Gate14ControllerError("provider inventory is not exact") + for client in (plan.windows, plan.linux): + instance = _mapping(instances[client.instance], _INSTANCE_FIELDS) + if type(instance["present"]) is not bool or type(disks[client.disk]) is not bool: + raise Gate14ControllerError("provider inventory type is invalid") + if instance["present"]: + if ( + instance["run_id"] != plan.run_id + or instance["source_commit"] != client.source_commit + or _integer(instance["termination_unix"], 1) != client.termination_unix + or disks[client.disk] is not True + ): + raise Gate14ControllerError("provider resource binding is invalid") + elif any( + value is not None + for value in ( + instance["run_id"], + instance["source_commit"], + instance["termination_unix"], + ) + ): + raise Gate14ControllerError("absent instance metadata is invalid") + job = _mapping(clients[client.platform], _CLIENT_FIELDS) + job_state = job["job_state"] + if job_state not in JOB_STATES: + raise Gate14ControllerError("host job state is invalid") + attempt = _integer(job["attempt_ordinal"], 0, 1) + evidence_digest = job["evidence_digest"] + if job_state == "absent": + if attempt != 0 or evidence_digest is not None: + raise Gate14ControllerError("absent host job evidence is inconsistent") + else: + if attempt != 1: + raise Gate14ControllerError("host job attempt is inconsistent") + if job_state == "passed": + _string(evidence_digest, _DIGEST_RE) + elif evidence_digest is not None: + raise Gate14ControllerError("unfinished host job exposed evidence") + expected_l4_usage = sum(int(instances[client.instance]["present"]) for client in (plan.windows, plan.linux)) + if observation["l4_usage"] != expected_l4_usage: + raise Gate14ControllerError("accelerator inventory is inconsistent") + return observation + + +def _next(state: Mapping[str, Any], **changes: Any) -> dict[str, Any]: + result = dict(state) + result.update(changes) + result["revision"] = int(state["revision"]) + 1 + return result + + +def _resources_absent(observation: Mapping[str, Any], plan: RunPlan) -> bool: + return ( + all(not observation["instances"][name]["present"] for name in plan.instances) + and all(observation["disks"][name] is False for name in plan.disks) + and observation["l4_usage"] == 0 + ) + + +def _observed_evidence_matches(state: Mapping[str, Any], observation: Mapping[str, Any], platform: str) -> bool: + job = observation["clients"][platform] + return job["job_state"] == "passed" and job["evidence_digest"] == state[f"{platform}_evidence_digest"] + + +def reconcile( + state_value: Mapping[str, Any], + observation_value: Mapping[str, Any], + plan: RunPlan, +) -> dict[str, Any]: + state = validate_state(state_value, plan) + observation = validate_observation(observation_value, plan) + phase = state["phase"] + windows_present = observation["instances"][plan.windows.instance]["present"] + linux_present = observation["instances"][plan.linux.instance]["present"] + windows_job = observation["clients"]["windows"]["job_state"] + linux_job = observation["clients"]["linux"]["job_state"] + observed_at = observation["observed_at_unix"] + + if phase in TERMINAL_PHASES: + if not _resources_absent(observation, plan): + raise Gate14ControllerError("resources returned after terminal cleanup") + if phase == "CLEANED_PASS" and not ( + _observed_evidence_matches(state, observation, "windows") + and _observed_evidence_matches(state, observation, "linux") + ): + raise Gate14ControllerError("terminal evidence binding is inconsistent") + return state + if phase != "CLEANING_FAILED": + deadline = ( + plan.windows.termination_unix if phase in {"ABSENT", "WINDOWS_RUNNING"} else plan.linux.termination_unix + ) + if observed_at >= deadline: + if _resources_absent(observation, plan): + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="run-expired", + cleanup_verified=True, + next_action="none", + ) + return _next( + state, + phase="CLEANING_FAILED", + failure_code="run-expired", + next_action="cleanup_failure", + ) + if phase == "CLEANING_FAILED": + if _resources_absent(observation, plan): + return _next( + state, + phase="CLEANED_FAILURE", + cleanup_verified=True, + next_action="none", + ) + return _next(state, next_action="cleanup_failure") + if phase == "ABSENT": + orphan_disk = (observation["disks"][plan.windows.disk] and not windows_present) or ( + observation["disks"][plan.linux.disk] and not linux_present + ) + if orphan_disk: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="orphaned-planned-disk", + next_action="cleanup_failure", + ) + if linux_present or linux_job != "absent": + if _resources_absent(observation, plan): + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="unexpected-linux-state", + cleanup_verified=True, + next_action="none", + ) + return _next( + state, + phase="CLEANING_FAILED", + failure_code="unexpected-linux-state", + next_action="cleanup_failure", + ) + if windows_present: + return _next( + state, + phase="WINDOWS_RUNNING", + windows_consumed=True, + next_action="none", + ) + if windows_job != "absent": + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="stale-windows-job", + cleanup_verified=True, + next_action="none", + ) + return _next(state, next_action="start_windows") + if phase == "WINDOWS_RUNNING": + if linux_present or observation["disks"][plan.linux.disk] or not windows_present: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="windows-inventory-lost", + next_action="cleanup_failure", + ) + if state["next_action"] == "collect_windows" and not _observed_evidence_matches(state, observation, "windows"): + raise Gate14ControllerError("reported Windows evidence changed") + if windows_job in {"starting", "running"}: + return _next(state, next_action="none") + if windows_job == "passed": + return _next( + state, + windows_evidence_digest=observation["clients"]["windows"]["evidence_digest"], + next_action="collect_windows", + ) + return _next( + state, + phase="CLEANING_FAILED", + failure_code="windows-job-failed", + next_action="cleanup_failure", + ) + if phase == "WINDOWS_DELETING": + if not _observed_evidence_matches(state, observation, "windows"): + raise Gate14ControllerError("validated Windows evidence is unavailable") + if windows_present or observation["disks"][plan.windows.disk]: + if linux_present: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="clients-overlapped", + next_action="cleanup_failure", + ) + return _next(state, next_action="delete_windows") + if linux_present: + return _next( + state, + phase="LINUX_RUNNING", + linux_consumed=True, + next_action="none", + ) + if observation["disks"][plan.linux.disk]: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="orphaned-linux-disk", + next_action="cleanup_failure", + ) + if linux_job != "absent": + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="stale-linux-job", + cleanup_verified=True, + next_action="none", + ) + return _next(state, next_action="start_linux") + if phase == "LINUX_RUNNING": + if observation["disks"][plan.windows.disk]: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="orphaned-windows-disk", + next_action="cleanup_failure", + ) + if not _observed_evidence_matches(state, observation, "windows"): + raise Gate14ControllerError("validated Windows evidence is unavailable") + if windows_present or not linux_present: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="linux-inventory-lost", + next_action="cleanup_failure", + ) + if state["next_action"] == "collect_linux" and not _observed_evidence_matches(state, observation, "linux"): + raise Gate14ControllerError("reported Linux evidence changed") + if linux_job in {"starting", "running"}: + return _next(state, next_action="none") + if linux_job == "passed": + return _next( + state, + linux_evidence_digest=observation["clients"]["linux"]["evidence_digest"], + next_action="collect_linux", + ) + return _next( + state, + phase="CLEANING_FAILED", + failure_code="linux-job-failed", + next_action="cleanup_failure", + ) + if phase == "LINUX_DELETING": + if windows_present or observation["disks"][plan.windows.disk]: + return _next( + state, + phase="CLEANING_FAILED", + failure_code="windows-resources-returned", + next_action="cleanup_failure", + ) + if not ( + _observed_evidence_matches(state, observation, "windows") + and _observed_evidence_matches(state, observation, "linux") + ): + raise Gate14ControllerError("validated platform evidence is unavailable") + if not _resources_absent(observation, plan): + return _next(state, next_action="delete_linux") + if state["windows_evidence_digest"] is None or state["linux_evidence_digest"] is None: + return _next( + state, + phase="CLEANED_FAILURE", + failure_code="evidence-missing", + cleanup_verified=True, + next_action="none", + ) + return _next( + state, + phase="CLEANED_PASS", + cleanup_verified=True, + next_action="none", + ) + raise Gate14ControllerError("unhandled lifecycle phase") + + +def collect_platform( + state_value: Mapping[str, Any], + plan: RunPlan, + platform: str, + evidence_path: Path, +) -> dict[str, Any]: + state = validate_state(state_value, plan) + expected_phase = "WINDOWS_RUNNING" if platform == "windows" else "LINUX_RUNNING" + expected_action = f"collect_{platform}" + if state["phase"] != expected_phase or state["next_action"] != expected_action: + raise Gate14ControllerError("collect is out of sequence") + payload = _regular_bytes(evidence_path) + summary = acceptance.validate_platform_document(_strict_json(payload)) + client = plan.windows if platform == "windows" else plan.linux + if ( + summary["run_id"] != plan.run_id + or summary["platform"] != platform + or summary["source_commit"] != client.source_commit + or summary["package_sha256"] != client.package_sha256 + or summary["model_id"] != client.model_id + or summary["manifest_digest"] != client.manifest_digest + ): + raise Gate14ControllerError("platform evidence does not match the plan") + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + if digest != state[f"{platform}_evidence_digest"]: + raise Gate14ControllerError("collected evidence changed after host completion") + return _next( + state, + **{ + f"{platform}_evidence_digest": digest, + "phase": f"{platform.upper()}_DELETING", + "next_action": f"delete_{platform}", + }, + ) + + +def begin_cleanup( + state_value: Mapping[str, Any], + plan: RunPlan, + failure_code: str, +) -> dict[str, Any]: + state = validate_state(state_value, plan) + if state["phase"] in TERMINAL_PHASES: + return state + _string(failure_code, re.compile(r"[a-z0-9][a-z0-9-]{0,63}")) + return _next( + state, + phase="CLEANING_FAILED", + failure_code=failure_code, + next_action="cleanup_failure", + ) + + +def load_state(path: Path, plan: RunPlan) -> dict[str, Any]: + return validate_state(_strict_json(_regular_bytes(path)), plan) + + +def save_state(path: Path, state_value: Mapping[str, Any], plan: RunPlan) -> None: + state = validate_state(state_value, plan) + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + payload = (json.dumps(state, sort_keys=True, separators=(",", ":")) + os.linesep).encode("utf-8") + handle = tempfile.NamedTemporaryFile( + mode="wb", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) + temporary = Path(handle.name) + try: + with handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except BaseException: + try: + temporary.unlink() + except OSError: + pass + raise + + +def _common_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("start", "status", "collect", "cleanup")) + parser.add_argument("--authorization", type=Path, required=True) + parser.add_argument("--ledger", type=Path, required=True) + parser.add_argument("--state", type=Path, required=True) + parser.add_argument("--observation", type=Path) + parser.add_argument("--platform", choices=("windows", "linux")) + parser.add_argument("--evidence", type=Path) + parser.add_argument("--failure-code", default="operator-cleanup") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _common_parser().parse_args(argv) + try: + plan = load_plan(args.authorization, args.ledger) + state = load_state(args.state, plan) if args.state.exists() else initial_state(plan) + if args.operation in {"start", "status"}: + if args.observation is None: + raise Gate14ControllerError("observation is required") + state = reconcile( + state, + _strict_json(_regular_bytes(args.observation)), + plan, + ) + elif args.operation == "collect": + if args.platform is None or args.evidence is None: + raise Gate14ControllerError("platform evidence is required") + state = collect_platform(state, plan, args.platform, args.evidence) + else: + state = begin_cleanup(state, plan, args.failure_code) + if args.observation is not None: + state = reconcile( + state, + _strict_json(_regular_bytes(args.observation)), + plan, + ) + save_state(args.state, state, plan) + except (Gate14ControllerError, acceptance.Gate14EvidenceError) as exc: + raise SystemExit(str(exc)) from exc + print(json.dumps(state, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/drift/model_manifest.py b/src/drift/model_manifest.py index 293458d55..0c9cf9d9d 100644 --- a/src/drift/model_manifest.py +++ b/src/drift/model_manifest.py @@ -486,6 +486,26 @@ def _artifact_path_below_root(root: Path | str, relative_path: str) -> Path: return candidate +def _windows_safe_path(path: Path) -> Path: + """Opt long manifest-cache paths into the Win32 extended namespace. + + Full manifest and artifact SHA-256 identifiers make resumable lock and + partial paths exceed the legacy Win32 path limit under an ordinary user + profile. Python then reports a misleading ``FileNotFoundError`` even when + the parent directory exists. Keep the audited on-disk layout unchanged, + but use an extended-length spelling for filesystem operations. + """ + absolute = path.absolute() + if os.name != "nt": + return absolute + rendered = str(absolute) + if rendered.startswith("\\\\?\\") or len(rendered) < 248: + return absolute + if rendered.startswith("\\\\"): + return Path("\\\\?\\UNC\\" + rendered[2:]) + return Path("\\\\?\\" + rendered) + + def _validate_artifact_file(artifact: ManifestArtifact, candidate: Path) -> os.stat_result: try: stat_result = candidate.stat() @@ -670,9 +690,9 @@ def _resumable_paths(self, artifact: ManifestArtifact) -> Tuple[Path, Path, Path cache_root = Path(self.cache_dir).absolute() manifest_root = cache_root / "manifest-artifacts" / self.manifest.digest name_digest = hashlib.sha256(artifact.path.encode("utf-8")).hexdigest() - partial = manifest_root / "partial" / f"{name_digest}.part" - final = _artifact_path_below_root(manifest_root / "snapshot", artifact.path) - lock = manifest_root / "locks" / f"{name_digest}.lock" + partial = _windows_safe_path(manifest_root / "partial" / f"{name_digest}.part") + final = _windows_safe_path(_artifact_path_below_root(manifest_root / "snapshot", artifact.path)) + lock = _windows_safe_path(manifest_root / "locks" / f"{name_digest}.lock") return partial, final, lock def _resumable_hub_download(self, artifact: ManifestArtifact, *, destination: Optional[Path] = None) -> str: diff --git a/tests/test_gate13_automated_playthrough.py b/tests/test_gate13_automated_playthrough.py new file mode 100644 index 000000000..09bb79642 --- /dev/null +++ b/tests/test_gate13_automated_playthrough.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate13_automated_playthrough as replay + +MODEL_ID = "Qwen3.5 2B" +DIGEST = "sha256:" + "a" * 64 + + +def config_document(root: Path, platform: str = "windows") -> dict: + executable = root / ("CommunityAI.exe" if platform == "windows" else "CommunityAI") + executable.write_bytes(b"packaged-desktop") + archive = root / f"communityai-desktop-{platform}.zip" + archive.write_bytes(b"verified-production-archive") + return { + "schema_version": 2, + "run_id": "gate13-automated-a", + "platform": platform, + "source_commit": "1" * 40, + "package_archive": str(archive.resolve()), + "package_sha256": "sha256:" + hashlib.sha256(archive.read_bytes()).hexdigest(), + "package_bytes": archive.stat().st_size, + "desktop_executable": str(executable.resolve()), + "work_root": str((root / ".gate13-playthrough-gate13-automated-a").resolve()), + "model_id": MODEL_ID, + "manifest_digest": DIGEST, + "total_blocks": 24, + "policy": { + "sharing_enabled": True, + "allowed_models": [MODEL_ID], + "preferred_models": [MODEL_ID], + "denied_models": [], + "max_disk_space": "32GB", + "max_vram": "20GB", + "max_bandwidth_mbps": 100.0, + "max_power_watts": None, + "pause_timeout": 120.0, + "schedule": { + "timezone": "UTC", + "windows": [ + { + "days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"], + "start": "00:00", + "end": "23:59", + } + ], + }, + }, + "session_timeout_seconds": 30.0, + "inference_timeout_seconds": 10.0, + } + + +def session_evidence(plan: dict) -> dict: + stage = plan["stage"] + platform = plan["platform"] + inference_required = (platform, stage) in { + ("windows", "initial"), + ("linux", "initial"), + ("linux", "restart"), + } + policy_session = (platform, stage) in { + ("windows", "restart"), + ("linux", "initial"), + } + resumed_session = platform == "linux" and stage == "restart" + pause_session = stage == "restart" + return { + "schema_version": 2, + "scope": "gate13-packaged-desktop-playthrough", + "run_id": plan["run_id"], + "platform": platform, + "stage": stage, + "result": "passed", + "model_id": plan["model_id"], + "manifest_digest": plan["manifest_digest"], + "duration_seconds": 1.25, + "route": { + "rendered_in_real_window": True, + "complete": True, + "covered_blocks": plan["total_blocks"], + "total_blocks": plan["total_blocks"], + }, + "inference": { + "passed": True, + "model_id": plan["model_id"], + "manifest_digest": plan["manifest_digest"], + "completion_count": 1, + "generated_token_count": 1, + "response_content_retained": False, + "token_identifiers_retained": False, + "temporary_key_removed": True, + } + if inference_required + else None, + "ui": { + "real_window_opened": True, + "policy_dialog_saved": policy_session, + "start_clicked": policy_session, + "pause_control_observed": policy_session or resumed_session, + "pause_clicked": pause_session, + "restart_resume_observed": resumed_session, + "sharing_intent_enabled_observed": policy_session or resumed_session, + "sharing_intent_disabled_observed": pause_session, + }, + "limits": { + "storage": policy_session, + "memory_or_vram": policy_session, + "bandwidth": policy_session, + "power": False, + "pause_timeout": policy_session, + "schedule": policy_session, + }, + "timing": { + "start_observation_seconds": 25.0 + if platform == "windows" and stage == "restart" + else (20.0 if platform == "linux" and stage == "initial" else 0.0), + "restart_observation_seconds": 15.0 if resumed_session else 0.0, + }, + "privacy": { + "prompt_retained": False, + "response_content_retained": False, + "token_identifiers_retained": False, + "credentials_retained": False, + "paths_retained": False, + "endpoints_retained": False, + }, + } + + +@pytest.mark.parametrize("platform,expected_inferences,expected_resume", [("windows", 1, False), ("linux", 2, True)]) +def test_replay_runs_real_desktop_contract_twice_and_removes_temporaries( + tmp_path, platform, expected_inferences, expected_resume +): + document = config_document(tmp_path, platform) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(document), encoding="utf-8") + config = replay.load_config(config_path) + stages = [] + self_tests = [] + + def runner(argv, **kwargs): + assert kwargs["stdin"] is subprocess.DEVNULL + assert kwargs["stdout"] is subprocess.DEVNULL + assert kwargs["stderr"] is subprocess.DEVNULL + if "--gate13-ui-playthrough" not in argv: + self_tests.append(argv[1]) + return subprocess.CompletedProcess(argv, 0) + plan_path = Path(argv[argv.index("--gate13-ui-playthrough") + 1]) + evidence_path = Path(argv[argv.index("--gate13-ui-evidence") + 1]) + plan = json.loads(plan_path.read_text(encoding="utf-8")) + stages.append(plan["stage"]) + evidence_path.write_text(json.dumps(session_evidence(plan)), encoding="utf-8") + return subprocess.CompletedProcess(argv, 0) + + result = replay.run_replay(config, runner=runner) + + assert stages == ["initial", "restart"] + assert self_tests == ["--check-runtime", "--self-test", "--ui-self-test", "--onboarding-ui-self-test"] + assert result["result"] == "passed" + assert result["real_window_sessions"] == 2 + assert result["localhost_inference_count"] == expected_inferences + assert result["restart_resume_observed"] is expected_resume + assert result["pause_control_observed"] is True + assert result["sharing_intent_paused"] is True + assert result["sequence_profile"] == replay.SEQUENCE_PROFILES[platform] + assert result["policy_profile"] == replay.POLICY_PROFILE + assert result["qualification_temporaries_removed"] is True + assert not config.work_root.exists() + + +def test_config_and_session_evidence_fail_closed(tmp_path): + document = config_document(tmp_path) + document["work_root"] = str((tmp_path / "wrong-root").resolve()) + config_path = tmp_path / "invalid.json" + config_path.write_text(json.dumps(document), encoding="utf-8") + with pytest.raises(replay.ReplayError): + replay.load_config(config_path) + + invalid_power = config_document(tmp_path) + invalid_power["policy"]["max_power_watts"] = 250.0 + invalid_power_path = tmp_path / "invalid-power.json" + invalid_power_path.write_text(json.dumps(invalid_power), encoding="utf-8") + with pytest.raises(replay.ReplayError): + replay.load_config(invalid_power_path) + + valid = config_document(tmp_path) + valid_path = tmp_path / "valid.json" + valid_path.write_text(json.dumps(valid), encoding="utf-8") + config = replay.load_config(valid_path) + config.work_root.mkdir() + evidence = session_evidence({**valid, "stage": "restart"}) + evidence["ui"]["start_clicked"] = False + evidence_path = config.work_root / "evidence.json" + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + with pytest.raises(replay.ReplayError): + replay._validate_session(evidence_path, config, "restart") + + evidence = session_evidence({**valid, "stage": "initial"}) + evidence["inference"]["generated_token_count"] = 2 + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + with pytest.raises(replay.ReplayError): + replay._validate_session(evidence_path, config, "initial") diff --git a/tests/test_gate13_client_startup.py b/tests/test_gate13_client_startup.py new file mode 100644 index 000000000..41b933697 --- /dev/null +++ b/tests/test_gate13_client_startup.py @@ -0,0 +1,86 @@ +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +WINDOWS = ROOT / "scripts" / "gate13_windows_client_startup.ps1" +LINUX = ROOT / "scripts" / "gate13_linux_client_startup.sh" +GIT_BASH = Path(r"C:\Program Files\Git\bin\bash.exe") +BASH = str(GIT_BASH) if GIT_BASH.is_file() else shutil.which("bash") + + +def test_windows_bootstrap_preserves_the_proven_interactive_boundary(): + source = WINDOWS.read_text(encoding="utf-8") + + assert "RandomNumberGenerator]::Create()" in source + assert "RandomNumberGenerator]::Fill" not in source + assert 'if ((Get-Service -Name sshd).Status -ne "Running") { Start-Service -Name sshd }' in source + assert source.count('Set-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -Enabled True -Profile Any') == 2 + assert 'New-LocalUser -Name "M"' in source + assert 'Remove-LocalGroupMember -Group "Administrators" -Member "M"' in source + assert 'New-LocalUser -Name "Gate13Admin"' in source + assert 'Set-ItemProperty -Path $winlogon -Name AutoAdminLogon -Value "1"' in source + assert 'New-ScheduledTaskTrigger -AtLogOn -User "M"' in source + assert "Remove-ItemProperty -Path $p -Name DefaultPassword" in source + assert "Restart-Computer -Force" in source + assert "2a52993092a19cfdffe126e2eeac46a4265e25705614546604ad44988e040c0f" in source + assert "communityai_gate13_m_authorized_keys" in source + + +@pytest.mark.skipif(sys.platform != "win32", reason="requires Windows PowerShell parser") +def test_windows_bootstrap_parses_natively(): + probe = ( + f"$source=Get-Content -Raw -LiteralPath '{WINDOWS}';" + "$tokens=$null;$errors=$null;" + "[Management.Automation.Language.Parser]::ParseInput($source,[ref]$tokens,[ref]$errors)|Out-Null;" + "if($errors.Count -ne 0){$errors|ForEach-Object{$_.Message};exit 2}" + ) + result = subprocess.run( + [ + r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe", + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + probe, + ], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_linux_bootstrap_contains_the_proven_x11_runtime_and_display(): + source = LINUX.read_text(encoding="utf-8") + for package in ( + "xvfb", + "dbus-x11", + "gnome-keyring", + "libsecret-tools", + "libxcb-cursor0", + "libxcb-icccm4", + "libxcb-keysyms1", + "libxcb-shape0", + "libxkbcommon-x11-0", + ): + assert package in source + assert "/usr/bin/Xvfb :99" in source + assert "-nolisten tcp" in source + assert "DISPLAY=:99 xdpyinfo" in source + + +@pytest.mark.skipif(BASH is None, reason="requires bash parser") +def test_linux_bootstrap_parses_natively(): + result = subprocess.run( + [BASH, "-n", str(LINUX)], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr diff --git a/tests/test_gate13_host_job.py b/tests/test_gate13_host_job.py new file mode 100644 index 000000000..90a66837c --- /dev/null +++ b/tests/test_gate13_host_job.py @@ -0,0 +1,657 @@ +import hashlib +import io +import json +import subprocess +import sys +import threading +from dataclasses import replace +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate13_host_job as host_job # noqa: E402 + + +def sha256(path): + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +@pytest.fixture +def config_factory(tmp_path, monkeypatch): + def make(platform="linux"): + root = tmp_path / platform + root.mkdir() + adapter = root / "gate13_host_job.py" + adapter.write_bytes(host_job.ADAPTER_PATH.read_bytes()) + entrypoint = root / ( + "gate13_windows_packaged_lifecycle.ps1" if platform == "windows" else "gate13_linux_packaged_lifecycle.py" + ) + entrypoint.write_text("# bound lifecycle\n", encoding="utf-8") + lifecycle_config = root / ("gate13-windows-run.json" if platform == "windows" else "gate13-linux-run.json") + lifecycle_config.write_text('{"bound":true}\n', encoding="utf-8") + python = Path(sys.executable).resolve() + + monkeypatch.setitem(host_job.HOST_ROOTS, platform, root) + monkeypatch.setitem(host_job.HOST_PYTHON, platform, python) + monkeypatch.setattr(host_job, "ADAPTER_PATH", adapter.resolve()) + + run_id = "gate13-test-a" + raw = { + "schema_version": 1, + "run_id": run_id, + "lifecycle_run_id": f"{run_id}-{platform}", + "platform": platform, + "attempt_ordinal": 1, + "source_commit": "a" * 40, + "job_name": f"communityai-gate13-{run_id}-{platform}", + "host_user": "gate13", + "adapter_path": str(adapter.resolve()), + "adapter_sha256": sha256(adapter), + "config_path": str((root / "host-job.json").resolve()), + "entrypoint_path": str(entrypoint.resolve()), + "entrypoint_sha256": sha256(entrypoint), + "lifecycle_config_path": str(lifecycle_config.resolve()), + "lifecycle_config_sha256": sha256(lifecycle_config), + "evidence_path": str((root / "evidence.json").resolve()), + "stderr_path": str((root / "stderr.log").resolve()), + "status_path": str((root / "status.json").resolve()), + "terminal_path": str((root / "terminal.json").resolve()), + "working_directory": str(root.resolve()), + "python_executable": str(python), + "max_run_seconds": 3600, + } + path = root / "host-job.json" + path.write_text(json.dumps(raw), encoding="utf-8") + return path, raw + + return make + + +def test_load_config_binds_exact_files_paths_and_single_attempt(config_factory): + path, raw = config_factory() + + config = host_job.load_config(path) + + assert config.attempt_ordinal == 1 + assert config.job_name == "communityai-gate13-gate13-test-a-linux" + assert config.adapter_sha256 == raw["adapter_sha256"] + assert config.entrypoint_sha256 == raw["entrypoint_sha256"] + assert config.lifecycle_config_sha256 == raw["lifecycle_config_sha256"] + assert config.host_user == "gate13" + + +def test_windows_environment_keeps_standard_user_runtime_and_drops_secrets(config_factory, monkeypatch): + path, _raw = config_factory("windows") + config = host_job.load_config(path) + expected = { + "APPDATA": r"C:\\Users\\M\\AppData\\Roaming", + "LOCALAPPDATA": r"C:\\Users\\M\\AppData\\Local", + "PATH": r"C:\\Windows\\System32", + "USERPROFILE": r"C:\\Users\\M", + } + for key, value in expected.items(): + monkeypatch.setenv(key, value) + monkeypatch.setenv("GH_TOKEN", "must-not-cross-the-host-boundary") + monkeypatch.setenv("COMMUNITYAI_CONTROL_TOKEN", "must-not-cross-the-host-boundary") + + environment = host_job._bounded_environment(config) + + assert all(environment[key] == value for key, value in expected.items()) + assert set(environment).issubset(set(host_job.WINDOWS_RUNTIME_ENVIRONMENT)) + assert "GH_TOKEN" not in environment + assert "COMMUNITYAI_CONTROL_TOKEN" not in environment + + +def test_linux_environment_keeps_display_and_secret_service_session(config_factory, monkeypatch): + path, _raw = config_factory("linux") + config = host_job.load_config(path) + expected = { + "DISPLAY": ":99", + "DBUS_SESSION_BUS_ADDRESS": "unix:path=/run/user/1000/bus", + "GNOME_KEYRING_CONTROL": "/run/user/1000/keyring", + "QT_QPA_PLATFORM": "offscreen", + } + for key, value in expected.items(): + monkeypatch.setenv(key, value) + monkeypatch.setenv("UNRELATED_SECRET", "must-not-cross-the-host-boundary") + + environment = host_job._bounded_environment(config) + + assert all(environment[key] == value for key, value in expected.items()) + assert set(environment).issubset(set(host_job.LINUX_RUNTIME_ENVIRONMENT)) + assert "UNRELATED_SECRET" not in environment + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("attempt_ordinal", 2), + ("job_name", "communityai-gate13-foreign-linux"), + ("lifecycle_run_id", "foreign-linux"), + ("max_run_seconds", 86_400), + ("host_user", "root"), + ], +) +def test_changed_execution_binding_fails_closed(config_factory, field, value): + path, raw = config_factory() + raw[field] = value + path.write_text(json.dumps(raw), encoding="utf-8") + + with pytest.raises(host_job.HostJobError): + host_job.load_config(path) + + +def test_path_escape_and_entrypoint_tampering_fail_closed(config_factory, tmp_path): + path, raw = config_factory() + raw["evidence_path"] = str((tmp_path / "escaped.json").resolve()) + path.write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(host_job.HostJobError, match="escapes"): + host_job.load_config(path) + + raw["evidence_path"] = str((path.parent / "evidence.json").resolve()) + Path(raw["entrypoint_path"]).write_text("# changed\n", encoding="utf-8") + path.write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(host_job.HostJobError, match="entrypoint digest changed"): + host_job.load_config(path) + + +def test_lifecycle_config_tampering_fails_closed(config_factory): + path, raw = config_factory() + Path(raw["lifecycle_config_path"]).write_text('{"changed":true}\n', encoding="utf-8") + + with pytest.raises(host_job.HostJobError, match="lifecycle config digest changed"): + host_job.load_config(path) + + +def test_windows_lifecycle_config_must_be_beside_entrypoint(config_factory): + path, raw = config_factory("windows") + nested = path.parent / "nested" + nested.mkdir() + nominated = nested / "gate13-windows-run.json" + nominated.write_text('{"bound":true}\n', encoding="utf-8") + raw["lifecycle_config_path"] = str(nominated.resolve()) + raw["lifecycle_config_sha256"] = sha256(nominated) + path.write_text(json.dumps(raw), encoding="utf-8") + + with pytest.raises(host_job.HostJobError, match="not beside"): + host_job.load_config(path) + + +def test_windows_task_is_bounded_interactive_ordinary_user_single_instance(config_factory): + path, _raw = config_factory("windows") + config = host_job.load_config(path) + + script = host_job._windows_register_script(config) + + assert "New-ScheduledTaskPrincipal -UserId $targetAccount.Value" in script + assert "-LogonType Interactive -RunLevel Limited" in script + assert "privileged task registration required" in script + assert "Get-LocalUser -Name 'gate13'" in script + assert "'SYSTEM'" not in script + assert "-MultipleInstances IgnoreNew" in script + assert "-ExecutionTimeLimit" in script + assert str(config.adapter_path) in script + assert str(config.config_path) in script + assert "password" not in script.lower() + assert "token" not in script.lower() + + snapshot = host_job._windows_snapshot_script(config) + assert "MultipleInstances -eq 'IgnoreNew'" in snapshot + assert "ExecutionTimeLimit -eq $expectedLimit" in snapshot + assert "LogonType -eq 'Interactive'" in snapshot + assert "$taskSid -eq $targetSid" in snapshot + assert "NTAccount]::new([string]$task.Principal.UserId)" in snapshot + assert "RunLevel -eq 'Limited'" in snapshot + + +@pytest.mark.skipif(sys.platform != "win32", reason="requires Windows PowerShell parser") +def test_windows_task_scripts_parse_natively(config_factory): + import base64 + + path, _raw = config_factory("windows") + config = host_job.load_config(path) + for source in ( + host_job._windows_register_script(config), + host_job._windows_snapshot_script(config), + ): + encoded = base64.b64encode(source.encode("utf-16le")).decode("ascii") + probe = ( + "$source=[Text.Encoding]::Unicode.GetString(" + f"[Convert]::FromBase64String('{encoded}'));" + "$tokens=$null;$errors=$null;" + "[Management.Automation.Language.Parser]::ParseInput(" + "$source,[ref]$tokens,[ref]$errors)|Out-Null;" + "if($errors.Count -ne 0){exit 2}" + ) + result = subprocess.run( + host_job._powershell_argv(probe), + check=False, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + + +def test_linux_unit_is_bounded_non_root_and_non_restarting(config_factory): + path, _raw = config_factory() + config = host_job.load_config(path) + + argv = host_job._linux_start_argv(config) + + assert argv[:4] == ["sudo", "-n", "/usr/bin/systemd-run", "--quiet"] + assert f"--unit" in argv + assert config.job_name in argv + assert "--property=User=gate13" in argv + assert "--property=Restart=no" in argv + assert "--property=KillMode=control-group" in argv + assert "--property=NoNewPrivileges=no" in argv + assert "--property=PrivateTmp=no" in argv + assert "--property=TimeoutStartSec=120" in argv + assert f"--property=RuntimeMaxSec={config.max_run_seconds + 2 * host_job.SUPERVISOR_GRACE_SECONDS}" in argv + assert "--setenv=DISPLAY=:99" in argv + assert "--setenv=HOME=/home/gate13" in argv + assert "--setenv=XDG_RUNTIME_DIR=/qualification/runtime" in argv + assert "/usr/bin/dbus-run-session" in argv + assert "execute-linux-desktop-session" in argv + assert "--wait" not in argv + assert host_job._entrypoint_argv(config)[-2:] == [ + "--config", + str(config.lifecycle_config_path), + ] + + +def test_linux_desktop_session_starts_secret_service_before_execute(tmp_path, monkeypatch): + config_path = tmp_path / "host-job.json" + observed = [] + monkeypatch.setattr(host_job.sys, "platform", "linux") + monkeypatch.setenv("DISPLAY", ":99") + monkeypatch.setenv("HOME", "/home/gate13") + monkeypatch.setenv("XDG_RUNTIME_DIR", "/qualification/runtime") + monkeypatch.setenv("DBUS_SESSION_BUS_ADDRESS", "unix:path=/qualification/runtime/bus") + + def run(argv, **kwargs): + observed.append((argv, kwargs)) + return subprocess.CompletedProcess( + argv, 0, stdout="GNOME_KEYRING_CONTROL=/qualification/runtime/keyring\n", stderr="" + ) + + monkeypatch.setattr(host_job.subprocess, "run", run) + monkeypatch.setattr(host_job, "execute", lambda path: {"result": "passed", "path": str(path)}) + + assert host_job._execute_linux_desktop_session(config_path)["result"] == "passed" + assert observed[0][0] == ["/usr/bin/gnome-keyring-daemon", "--unlock", "--components=secrets"] + assert observed[0][1]["input"] == "\n" + assert host_job.os.environ["GNOME_KEYRING_CONTROL"] == "/qualification/runtime/keyring" + + +def test_windows_automated_python_replay_uses_the_bound_python(config_factory): + path, _raw = config_factory("windows") + config = host_job.load_config(path) + automated = replace(config, entrypoint_path=config.entrypoint_path.with_suffix(".py")) + + assert host_job._entrypoint_argv(automated) == [ + str(config.python_executable), + str(automated.entrypoint_path), + "--config", + str(config.lifecycle_config_path), + ] + + +def test_bounded_copy_caps_private_diagnostics(tmp_path): + destination = tmp_path / "stderr.log" + overflow = threading.Event() + errors = [] + + host_job._bounded_copy( + io.BytesIO(b"x" * 257), + destination, + 256, + overflow, + errors, + ) + + assert overflow.is_set() + assert errors == [] + assert destination.stat().st_size == 256 + + +def test_real_entrypoint_output_is_capped(config_factory): + path, raw = config_factory() + entrypoint = Path(raw["entrypoint_path"]) + entrypoint.write_text( + "import sys\nsys.stdout.buffer.write(b'x' * 1048577)\n", + encoding="utf-8", + ) + raw["entrypoint_sha256"] = sha256(entrypoint) + path.write_text(json.dumps(raw), encoding="utf-8") + config = host_job.load_config(path) + + assert host_job._run_entrypoint(config) == 126 + assert config.evidence_path.stat().st_size == host_job.MAX_EVIDENCE_BYTES + assert config.stderr_path.stat().st_size == 0 + + +def test_linux_tree_shutdown_escalates_to_process_group(config_factory, monkeypatch): + path, _raw = config_factory() + config = host_job.load_config(path) + signals = [] + + class Process: + pid = 4321 + + def __init__(self): + self.waits = 0 + + def wait(self, timeout): + self.waits += 1 + if self.waits == 1: + raise subprocess.TimeoutExpired("entrypoint", timeout) + return -9 + + monkeypatch.setattr( + host_job.os, + "killpg", + lambda pid, requested: signals.append((pid, requested)), + raising=False, + ) + host_job._stop_process_tree(config, Process()) + + assert signals == [ + (4321, host_job.POSIX_SIGTERM), + (4321, host_job.POSIX_SIGKILL), + ] + + +def test_execute_persists_status_validates_evidence_and_never_relaunches(config_factory, monkeypatch): + path, _raw = config_factory() + calls = [] + + monkeypatch.setattr(host_job.lifecycle, "load_lifecycle_json", lambda _payload: {}) + monkeypatch.setattr( + host_job.lifecycle, + "validate_lifecycle_document", + lambda _document: { + "run_id": "gate13-test-a-linux", + "platform": "linux", + "source_commit": "a" * 40, + }, + ) + + def runner(config): + calls.append(config.job_name) + config.evidence_path.write_text('{"canonical":true}', encoding="utf-8") + config.stderr_path.write_bytes(b"") + return 0 + + terminal = host_job.execute(path, clock=lambda: 2_000_000_000, entrypoint_runner=runner) + repeated = host_job.execute(path, clock=lambda: 2_000_000_001, entrypoint_runner=runner) + + assert terminal["result"] == "passed" + assert terminal["failure_code"] is None + assert terminal["evidence_digest"].startswith("sha256:") + assert repeated == terminal + assert calls == ["communityai-gate13-gate13-test-a-linux"] + status = json.loads((path.parent / "status.json").read_text(encoding="utf-8")) + assert status["state"] == "running" + assert status["attempt_ordinal"] == 1 + + +def test_started_attempt_without_terminal_is_never_relaunched(config_factory): + path, _raw = config_factory() + config = host_job.load_config(path) + host_job._atomic_json( + config.status_path, + { + "schema_version": 1, + "run_id": config.run_id, + "platform": config.platform, + "attempt_ordinal": 1, + "state": "running", + "started_at_unix": 2_000_000_000, + }, + exclusive=True, + ) + called = False + + def runner(_config): + nonlocal called + called = True + return 0 + + with pytest.raises(host_job.HostJobError, match="already started"): + host_job.execute(path, entrypoint_runner=runner) + assert called is False + + +def test_observation_distinguishes_pristine_active_terminal_and_ambiguous(config_factory, monkeypatch): + path, _raw = config_factory() + config = host_job.load_config(path) + + assert host_job.observe_job(config, {"native_state": "absent", "binding_ok": False}) == { + "job_state": "absent", + "attempt_ordinal": 0, + "evidence_digest": None, + } + assert host_job.observe_job(config, {"native_state": "running", "binding_ok": True}) == { + "job_state": "starting", + "attempt_ordinal": 1, + "evidence_digest": None, + } + assert host_job.observe_job(config, {"native_state": "running", "binding_ok": False}) == { + "job_state": "ambiguous", + "attempt_ordinal": 1, + "evidence_digest": None, + } + + monkeypatch.setattr(host_job.lifecycle, "load_lifecycle_json", lambda _payload: {}) + monkeypatch.setattr( + host_job.lifecycle, + "validate_lifecycle_document", + lambda _document: { + "run_id": "gate13-test-a-linux", + "platform": "linux", + "source_commit": "a" * 40, + }, + ) + + def runner(bound): + bound.evidence_path.write_text("{}", encoding="utf-8") + bound.stderr_path.write_bytes(b"") + return 0 + + terminal = host_job.execute(path, clock=lambda: 2_000_000_000, entrypoint_runner=runner) + observed = host_job.observe_job(config, {"native_state": "absent", "binding_ok": False}) + assert observed["job_state"] == "passed" + assert observed["evidence_digest"] == terminal["evidence_digest"] + assert host_job.observe_job(config, {"native_state": "running", "binding_ok": False}) == { + "job_state": "ambiguous", + "attempt_ordinal": 1, + "evidence_digest": None, + } + + +def test_inactive_after_persisted_start_is_ambiguous(config_factory): + path, _raw = config_factory() + config = host_job.load_config(path) + host_job._atomic_json( + config.status_path, + { + "schema_version": 1, + "run_id": config.run_id, + "platform": config.platform, + "attempt_ordinal": 1, + "state": "running", + "started_at_unix": 2_000_000_000, + }, + exclusive=True, + ) + + assert host_job.observe_job(config, {"native_state": "inactive", "binding_ok": True}) == { + "job_state": "ambiguous", + "attempt_ordinal": 1, + "evidence_digest": None, + } + + +def test_collect_revalidates_terminal_digest_and_lifecycle_binding(config_factory, monkeypatch): + path, _raw = config_factory() + monkeypatch.setattr(host_job.lifecycle, "load_lifecycle_json", lambda _payload: {}) + monkeypatch.setattr( + host_job.lifecycle, + "validate_lifecycle_document", + lambda _document: { + "run_id": "gate13-test-a-linux", + "platform": "linux", + "source_commit": "a" * 40, + }, + ) + + def runner(config): + config.evidence_path.write_text('{"ok":true}', encoding="utf-8") + config.stderr_path.write_bytes(b"") + return 0 + + host_job.execute(path, clock=lambda: 2_000_000_000, entrypoint_runner=runner) + assert host_job.collect(path) == b'{"ok":true}' + + (path.parent / "evidence.json").write_text('{"ok":false}', encoding="utf-8") + with pytest.raises(host_job.HostJobError, match="digest changed"): + host_job.collect(path) + + +def test_start_reattaches_to_bound_native_job_without_mutation(config_factory, monkeypatch): + path, _raw = config_factory() + monkeypatch.setattr( + host_job, + "native_snapshot", + lambda _config, _runner: {"native_state": "running", "binding_ok": True}, + ) + + def forbidden(*_args, **_kwargs): + raise AssertionError("native start must not run") + + observed = host_job.start(path, runner=forbidden) + + assert observed == { + "job_state": "starting", + "attempt_ordinal": 1, + "evidence_digest": None, + } + + +def test_linux_snapshot_binds_exact_service_command(config_factory): + path, _raw = config_factory() + config = host_job.load_config(path) + stdout = "\n".join( + [ + "LoadState=loaded", + "ActiveState=active", + "SubState=running", + "User=gate13", + "Group=gate13", + ( + "ExecStart={ path=/usr/bin/dbus-run-session ; argv[]=/usr/bin/dbus-run-session " + f"{config.python_executable} {config.adapter_path} " + f"execute-linux-desktop-session --config {config.config_path} ; " + "ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; " + "pid=0 ; code=(null) ; status=0/0 }" + ), + f"WorkingDirectory={config.working_directory}", + "Restart=no", + "KillMode=control-group", + "UMask=0077", + "NoNewPrivileges=no", + "PrivateTmp=no", + 'Environment="DISPLAY=:99" "HOME=/home/gate13" "XDG_RUNTIME_DIR=/qualification/runtime"', + "TimeoutStartUSec=2min", + "RuntimeMaxUSec=1h 2min", + ] + ) + + def runner(_argv, timeout): + assert timeout == 60 + return subprocess.CompletedProcess([], 0, stdout=stdout, stderr="") + + assert host_job._linux_snapshot(config, runner) == { + "native_state": "running", + "binding_ok": True, + } + + foreign_stdout = stdout.replace( + f"execute-linux-desktop-session --config {config.config_path} ;", + f"execute-linux-desktop-session --config {config.config_path} --extra ;", + ) + + def foreign_runner(_argv, timeout): + assert timeout == 60 + return subprocess.CompletedProcess([], 0, stdout=foreign_stdout, stderr="") + + assert host_job._linux_snapshot(config, foreign_runner) == { + "native_state": "running", + "binding_ok": False, + } + + for foreign_stdout in ( + stdout.replace("ignore_errors=no", "ignore_errors=yes"), + stdout.replace("status=0/0 }", "status=0/0 ; arbitrary=value }"), + ): + + def foreign_metadata_runner(_argv, timeout): + assert timeout == 60 + return subprocess.CompletedProcess([], 0, stdout=foreign_stdout, stderr="") + + assert host_job._linux_snapshot(config, foreign_metadata_runner) == { + "native_state": "running", + "binding_ok": False, + } + + +def test_linux_snapshot_accepts_fresh_systemd_inventory_without_exec_start(config_factory): + path, _raw = config_factory() + config = host_job.load_config(path) + stdout = "\n".join( + [ + "Restart=no", + "TimeoutStartUSec=1min 30s", + "RuntimeMaxUSec=infinity", + "Environment=", + "UMask=0022", + "WorkingDirectory=", + "User=", + "Group=", + "PrivateTmp=no", + "NoNewPrivileges=no", + "KillMode=control-group", + "LoadState=not-found", + "ActiveState=inactive", + "SubState=dead", + ] + ) + + def runner(_argv, timeout): + assert timeout == 60 + return subprocess.CompletedProcess([], 0, stdout=stdout, stderr="") + + assert host_job._linux_snapshot(config, runner) == { + "native_state": "absent", + "binding_ok": False, + } + + +def test_public_cli_failure_is_bounded_and_path_free(capsys, tmp_path): + missing = tmp_path / "secret-token-config.json" + + exit_code = host_job.main(["status", "--config", str(missing)]) + + assert exit_code == 2 + output = capsys.readouterr().out + assert json.loads(output) == { + "failure_code": "host_job_rejected", + "result": "failed", + "schema_version": 1, + } + assert str(missing) not in output diff --git a/tests/test_gate13_linux_packaged_lifecycle.py b/tests/test_gate13_linux_packaged_lifecycle.py index 45e60baf7..b4a5b4315 100644 --- a/tests/test_gate13_linux_packaged_lifecycle.py +++ b/tests/test_gate13_linux_packaged_lifecycle.py @@ -742,6 +742,11 @@ def run(command, **kwargs): assert owner.owned == [] +def test_termination_signal_enters_lifecycle_cleanup_path(): + with pytest.raises(linux_lifecycle.LifecycleRunError, match="termination"): + linux_lifecycle._termination_requested(15, None) + + def test_main_failure_is_generic_and_does_not_echo_config(monkeypatch, capsys): marker = "/private/path/must-not-escape" monkeypatch.setattr(linux_lifecycle, "_disable_core_dumps", lambda: None) diff --git a/tests/test_gate13_packaged_lifecycle.py b/tests/test_gate13_packaged_lifecycle.py index ffad861dc..86fefcc7f 100644 --- a/tests/test_gate13_packaged_lifecycle.py +++ b/tests/test_gate13_packaged_lifecycle.py @@ -480,3 +480,50 @@ def test_public_summary_does_not_retain_raw_phase_only_fields(): "recovery_action_count", ): assert forbidden_value not in rendered + + +def test_current_gate13_automated_replay_is_accepted_by_the_host_evidence_boundary(): + document = { + "schema_version": 2, + "scope": "gate13-automated-desktop-replay", + "run_id": "gate13-automated-a", + "platform": "windows", + "result": "passed", + "source_commit": SOURCE_COMMIT, + "package": { + "sha256": "sha256:" + PACKAGE_DIGEST, + "bytes": 123_456_789, + "verified_before_run": True, + "self_test_count": 4, + }, + "model_id": "Qwen3.5 2B", + "manifest_digest": "sha256:" + MANIFEST_DIGEST, + "real_window_sessions": 2, + "localhost_inference_count": 1, + "policy_dialog_saved": True, + "start_clicked": True, + "pause_control_observed": True, + "restart_resume_observed": False, + "pause_clicked": True, + "sharing_intent_paused": True, + "policy_profile": "gate13-manual-cpu-v1", + "sequence_profile": "gate13-manual-windows-v1", + "start_observation_seconds": 25.0, + "session_duration_seconds": {"initial": 100.0, "restart": 80.0}, + "privacy_safe": True, + "qualification_temporaries_removed": True, + } + + evidence = lifecycle.validate_lifecycle_document(document) + + assert evidence["result"] == "passed" + assert evidence["source_commit"] == SOURCE_COMMIT + assert evidence["package_sha256"] == PACKAGE_DIGEST + assert evidence["manifest_digest"] == MANIFEST_DIGEST + assert evidence["lifecycle"]["real_window_sessions"] == 2 + assert evidence["lifecycle"]["restart_resume_observed"] is False + assert evidence["lifecycle"]["policy_profile"] == "gate13-manual-cpu-v1" + + document["pause_clicked"] = False + with pytest.raises(lifecycle.LifecycleEvidenceError): + lifecycle.validate_lifecycle_document(document) diff --git a/tests/test_gate13_route_fence.py b/tests/test_gate13_route_fence.py new file mode 100644 index 000000000..78a0083df --- /dev/null +++ b/tests/test_gate13_route_fence.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate13_route_fence as fence + + +class Response: + status = 200 + + class Headers: + @staticmethod + def get_content_type(): + return "application/json" + + headers = Headers() + + def __init__(self, document): + self.payload = json.dumps(document).encode() + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _maximum): + return self.payload + + +def profile(tmp_path: Path) -> fence.Profile: + local = tmp_path / "local-api.key" + control = tmp_path / "control-api.key" + local.write_text("local-secret\n", encoding="ascii") + control.write_text("control-secret\n", encoding="ascii") + return fence.Profile( + target="windows", + service="communityai-qwen.service", + other_service="communityai-gemma.service", + origin="http://127.0.0.1:8081", + local_key=local, + control_key=control, + model_id="Qwen3.5 2B", + manifest_digest="sha256:" + "a" * 64, + total_blocks=24, + ) + + +def ready_opener(item: fence.Profile): + models = { + "data": [ + { + "id": item.model_id, + "availability": "complete", + "manifest_digest": item.manifest_digest, + } + ] + } + status = { + "auto_selection": { + "status": "selected", + "model": item.model_id, + "manifest_digest": item.manifest_digest, + "covered_blocks": item.total_blocks, + "total_blocks": item.total_blocks, + "peer_count": 1, + } + } + opener = MagicMock() + opener.open.side_effect = [Response(models), Response(status), Response(models), Response(status)] + return opener + + +def test_fence_restarts_only_target_and_rechecks_exact_route_after_settle(tmp_path): + item = profile(tmp_path) + calls = [] + timeouts = [] + + def runner(argv, **kwargs): + assert kwargs["stdin"] is subprocess.DEVNULL + assert kwargs["stdout"] is subprocess.DEVNULL + assert kwargs["stderr"] is subprocess.DEVNULL + calls.append(tuple(argv[1:])) + timeouts.append(kwargs["timeout"]) + inactive_probe = argv[1:3] == ["is-active", "--quiet"] and argv[3] == item.other_service + return subprocess.CompletedProcess(argv, 3 if inactive_probe else 0) + + sleeps = [] + result = fence.fence_route( + item, + timeout_seconds=60, + settle_seconds=30, + runner=runner, + opener=ready_opener(item), + sleeper=sleeps.append, + ) + + assert calls[:2] == [("stop", item.other_service), ("restart", item.service)] + assert timeouts[:2] == [fence.SERVICE_ACTION_TIMEOUT_SECONDS] * 2 + assert ("is-active", "--quiet", item.other_service) in calls + assert sleeps == [30] + assert result == { + "schema_version": 1, + "scope": "gate13-route-client-fence", + "result": "passed", + "target": "windows", + "model_id": item.model_id, + "manifest_digest": item.manifest_digest, + "covered_blocks": 24, + "total_blocks": 24, + "peer_count_minimum": 1, + "target_service_restarted": True, + "standby_service_stopped": True, + "stable_rechecks": 2, + "settle_seconds": 30, + "privacy_safe": True, + } + + +def test_fence_fails_if_standby_is_still_active(tmp_path): + item = profile(tmp_path) + + def runner(argv, **_kwargs): + return subprocess.CompletedProcess(argv, 0) + + with pytest.raises(fence.FenceError, match="remain stable"): + fence.fence_route( + item, + timeout_seconds=60, + settle_seconds=30, + runner=runner, + opener=ready_opener(item), + sleeper=lambda _seconds: None, + ) + + +def test_fence_retries_when_stale_advertisement_expires_during_settle(tmp_path): + item = profile(tmp_path) + opener = ready_opener(item) + ready_responses = list(opener.open.side_effect) + incomplete_models = { + "data": [ + { + "id": item.model_id, + "availability": "incomplete", + "manifest_digest": item.manifest_digest, + } + ] + } + opener.open.side_effect = [ + *ready_responses[:2], + Response(incomplete_models), + ready_responses[3], + *ready_responses, + ] + + def runner(argv, **_kwargs): + inactive_probe = argv[1:3] == ["is-active", "--quiet"] and argv[3] == item.other_service + return subprocess.CompletedProcess(argv, 3 if inactive_probe else 0) + + sleeps = [] + result = fence.fence_route( + item, + timeout_seconds=60, + settle_seconds=30, + runner=runner, + opener=opener, + sleeper=sleeps.append, + ) + + assert result["result"] == "passed" + assert sleeps == [30, 5.0, 30] + + +def test_snapshot_rejects_wrong_model_manifest_even_when_control_coverage_is_complete(tmp_path): + item = profile(tmp_path) + opener = ready_opener(item) + wrong_manifest_models = { + "data": [ + { + "id": item.model_id, + "availability": "complete", + "manifest_digest": "sha256:" + "b" * 64, + } + ] + } + responses = list(opener.open.side_effect) + responses[0] = Response(wrong_manifest_models) + opener.open.side_effect = responses + + assert fence._snapshot(item, opener) is False + + +def test_secret_rejects_links(tmp_path): + target = tmp_path / "target" + target.write_text("secret", encoding="ascii") + link = tmp_path / "link" + try: + link.symlink_to(target) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + with pytest.raises(fence.FenceError, match="unsafe"): + fence._secret(link) diff --git a/tests/test_gate13_run_controller.py b/tests/test_gate13_run_controller.py new file mode 100644 index 000000000..d80d52119 --- /dev/null +++ b/tests/test_gate13_run_controller.py @@ -0,0 +1,625 @@ +import hashlib +import json +import sys +from dataclasses import replace +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate13_run_controller as controller # noqa: E402 + +AUTHORIZATION = ROOT / "docs" / "evidence" / "gate13-20260831-a-cost-authorization.json" +LEDGER = ROOT / "docs" / "RELEASE_READINESS.md" +NOW = 2_000_000_000 +ROUTE_DIGEST = "sha256:" + "a" * 64 +WINDOWS_DIGEST = "sha256:" + "b" * 64 +LINUX_DIGEST = "sha256:" + "c" * 64 + + +def reserved_ledger_text(*, old_digest, new_digest): + lines = LEDGER.read_text(encoding="utf-8").splitlines(keepends=True) + matches = [index for index, line in enumerate(lines) if line.startswith("| gate13-20260831-a |")] + assert len(matches) == 1 + index = matches[0] + assert old_digest in lines[index] + assert lines[index].rstrip().endswith("| CLEANED-COMMITTED |") + lines[index] = lines[index].replace(old_digest, new_digest, 1).replace("| CLEANED-COMMITTED |", "| RESERVED |", 1) + return "".join(lines) + + +@pytest.fixture +def plan(tmp_path): + raw = json.loads(AUTHORIZATION.read_text(encoding="utf-8")) + raw["provider_plan"]["sequencing"]["clients_may_run_concurrently"] = False + old_digest = raw["provider_plan_digest"] + new_digest = controller._provider_digest(raw["provider_plan"]) + raw["provider_plan_digest"] = new_digest + authorization = tmp_path / "authorization.json" + authorization.write_text(json.dumps(raw), encoding="utf-8") + + ledger = tmp_path / "ledger.md" + ledger.write_text( + reserved_ledger_text(old_digest=old_digest, new_digest=new_digest), + encoding="utf-8", + ) + return controller.load_plan(authorization, ledger) + + +def observation( + plan, *, route=False, windows=False, linux=False, route_job="absent", windows_job="absent", linux_job="absent" +): + present = { + plan.route_instance: (route, plan.route_source_commit), + plan.windows_instance: (windows, plan.windows_source_commit), + plan.linux_instance: (linux, plan.linux_source_commit), + } + return { + "schema_version": 1, + "run_id": plan.run_id, + "observed_at_unix": NOW, + "instances": { + name: { + "present": exists, + "run_id": plan.run_id if exists else None, + "source_commit": source if exists else None, + "termination_unix": NOW + 20_000 if exists else None, + } + for name, (exists, source) in present.items() + }, + "disks": { + plan.route_disk: route, + plan.windows_disk: windows, + plan.linux_disk: linux, + }, + "firewalls": { + plan.route_firewalls[0]: route, + plan.route_firewalls[1]: route, + }, + "protected_bootstrap_running": True, + "route_acceptance": { + "job_state": route_job, + "evidence_digest": ROUTE_DIGEST if route_job == "passed" else None, + }, + "clients": { + "windows": { + "job_state": windows_job, + "attempt_ordinal": 1 if windows_job != "absent" else 0, + "evidence_digest": WINDOWS_DIGEST if windows_job == "passed" else None, + }, + "linux": { + "job_state": linux_job, + "attempt_ordinal": 1 if linux_job != "absent" else 0, + "evidence_digest": LINUX_DIGEST if linux_job == "passed" else None, + }, + }, + } + + +def test_load_plan_binds_exact_cost_and_resources(plan): + assert plan.run_id == "gate13-20260831-a" + assert plan.provider_plan_digest.startswith("sha256:") + assert plan.ledger_state == "RESERVED" + assert plan.instance_names == ( + "route-20260831-a-node", + "gate13-20260831-a-win", + "gate13-20260831-a-linux", + ) + assert controller.PROTECTED_INSTANCE not in plan.instance_names + assert plan.clients_may_run_concurrently is False + + +def test_load_plan_accepts_the_automated_replay_instead_of_legacy_16_phases(tmp_path): + raw = json.loads(AUTHORIZATION.read_text(encoding="utf-8")) + sequencing = raw["provider_plan"]["sequencing"] + sequencing["clients_may_run_concurrently"] = False + sequencing["all_16_phases_required_per_platform"] = False + sequencing["automated_gate13_replay_required"] = True + old_digest = raw["provider_plan_digest"] + new_digest = controller._provider_digest(raw["provider_plan"]) + raw["provider_plan_digest"] = new_digest + authorization = tmp_path / "authorization.json" + authorization.write_text(json.dumps(raw), encoding="utf-8") + ledger = tmp_path / "ledger.md" + ledger.write_text( + reserved_ledger_text(old_digest=old_digest, new_digest=new_digest), + encoding="utf-8", + ) + + replay_plan = controller.load_plan(authorization, ledger) + + assert replay_plan.clients_may_run_concurrently is False + + +def test_load_plan_accepts_only_documented_owner_ceiling(tmp_path): + raw = json.loads(AUTHORIZATION.read_text(encoding="utf-8")) + raw["provider_plan"]["sequencing"]["clients_may_run_concurrently"] = False + old_digest = raw["provider_plan_digest"] + new_digest = controller._provider_digest(raw["provider_plan"]) + raw["provider_plan_digest"] = new_digest + raw["authorization"].update( + { + "combined_cloud_ceiling_usd": "500.00", + "ledger_committed_before_run_usd": "52.00", + "maximum_estimate_usd": "56.00", + "remaining_after_run_maximum_usd": "392.00", + } + ) + authorization = tmp_path / "authorization.json" + authorization.write_text(json.dumps(raw), encoding="utf-8") + ledger = tmp_path / "ledger.md" + ledger.write_text( + reserved_ledger_text(old_digest=old_digest, new_digest=new_digest), + encoding="utf-8", + ) + + raised_plan = controller.load_plan(authorization, ledger) + assert raised_plan.ledger_state == "RESERVED" + assert controller.initial_state(raised_plan)["next_action"] == "start_route" + + raw["authorization"]["combined_cloud_ceiling_usd"] = "499.00" + raw["authorization"]["remaining_after_run_maximum_usd"] = "391.00" + authorization.write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(controller.RunControllerError, match="inconsistent"): + controller.load_plan(authorization, ledger) + + +def test_cleaned_committed_ledger_cannot_start_a_new_run(): + historical_plan = controller.load_plan(AUTHORIZATION, LEDGER) + + assert historical_plan.ledger_state == "CLEANED-COMMITTED" + with pytest.raises(controller.RunControllerError, match="not reserved"): + controller.initial_state(historical_plan) + + +def test_non_reserved_ledger_allows_cleanup_only(): + historical_plan = controller.load_plan(AUTHORIZATION, LEDGER) + reserved_plan = replace( + historical_plan, + ledger_state="RESERVED", + clients_may_run_concurrently=False, + ) + reserved_state = controller.initial_state(reserved_plan) + + forward_actions = controller.ACTION_STATES - controller.CLEANUP_ACTIONS - {"none"} + for action in forward_actions: + with pytest.raises(controller.RunControllerError, match="not reserved"): + controller.begin_action(reserved_state, historical_plan, action=action) + + cleanup_state = dict(reserved_state) + cleanup_state.update( + { + "phase": "CLEANING_FAILED", + "failure_code": "operator_cleanup", + "next_action": "cleanup_failure", + } + ) + cleaning = controller.begin_action( + cleanup_state, + historical_plan, + action="cleanup_failure", + ) + assert cleaning["phase"] == "CLEANING_FAILED" + assert cleaning["next_action"] == "none" + + +def test_reserved_parallel_client_plan_cannot_start(tmp_path): + ledger = tmp_path / "ledger.md" + digest = json.loads(AUTHORIZATION.read_text(encoding="utf-8"))["provider_plan_digest"] + ledger.write_text( + reserved_ledger_text(old_digest=digest, new_digest=digest), + encoding="utf-8", + ) + parallel = controller.load_plan(AUTHORIZATION, ledger) + + with pytest.raises(controller.RunControllerError, match="concurrent clients"): + controller.initial_state(parallel) + + +def test_changed_authorization_fails_closed(tmp_path): + raw = json.loads(AUTHORIZATION.read_text(encoding="utf-8")) + raw["provider_plan"]["route"]["machine_type"] = "e2-micro" + changed = tmp_path / "authorization.json" + changed.write_text(json.dumps(raw), encoding="utf-8") + + with pytest.raises(controller.RunControllerError, match="digest changed"): + controller.load_plan(changed, LEDGER) + + +def test_inventory_precedes_route_and_route_acceptance_precedes_clients(plan): + state = controller.initial_state(plan) + + absent = controller.reconcile(state, observation(plan), plan, now_unix=NOW) + assert absent["phase"] == "ABSENT" + assert absent["next_action"] == "start_route" + + accepting = controller.reconcile( + absent, + observation(plan, route=True, route_job="running"), + plan, + now_unix=NOW, + ) + assert accepting["phase"] == "ROUTE_ACCEPTING" + assert accepting["next_action"] == "accept_route" + + invalid = controller.reconcile( + accepting, + observation(plan, route=True, windows=True, route_job="running", windows_job="starting"), + plan, + now_unix=NOW, + ) + assert invalid["phase"] == "CLEANING_FAILED" + assert invalid["failure_code"] == "client_started_before_route_acceptance" + + +def test_exact_name_with_foreign_identity_fails_closed(plan): + raw = observation(plan, route=True, route_job="running") + raw["instances"][plan.route_instance]["run_id"] = "foreign-run" + + with pytest.raises(controller.RunControllerError, match="foreign exact-name"): + controller.reconcile(controller.initial_state(plan), raw, plan, now_unix=NOW) + + +def test_route_acceptance_starts_windows_before_linux(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, route_job="passed"), + plan, + now_unix=NOW, + ) + assert state["phase"] == "ROUTE_ACCEPTED" + assert state["next_action"] == "start_windows" + + invalid = controller.reconcile( + state, + observation(plan, route=True, linux=True, route_job="passed", linux_job="running"), + plan, + now_unix=NOW, + ) + assert invalid["phase"] == "CLEANING_FAILED" + assert invalid["failure_code"] == "linux_started_before_windows_evidence" + + +@pytest.mark.parametrize("job_state", ["failed", "ambiguous"]) +def test_failed_or_ambiguous_windows_is_consumed_and_never_resumed(plan, job_state): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, windows=True, route_job="passed", windows_job=job_state), + plan, + now_unix=NOW, + ) + + assert state["phase"] == "CLEANING_FAILED" + assert state["windows_consumed"] is True + assert state["next_action"] == "cleanup_failure" + + +def test_action_intent_is_persisted_before_mutation_and_cannot_relaunch(plan): + state = controller.initial_state(plan) + started = controller.begin_action(state, plan, action="start_route") + + assert started["phase"] == "ROUTE_STARTING" + assert started["next_action"] == "none" + missing = controller.reconcile(started, observation(plan), plan, now_unix=NOW) + assert missing["phase"] == "CLEANED_FAILURE" + assert missing["failure_code"] == "resources_disappeared_before_completion" + with pytest.raises(controller.RunControllerError, match="out of order"): + controller.begin_action(started, plan, action="start_route") + + +def test_route_acceptance_intent_cannot_rearm_after_dispatch(plan): + ready = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, route_job="absent"), + plan, + now_unix=NOW, + ) + dispatched = controller.begin_action(ready, plan, action="accept_route") + + running = controller.reconcile( + dispatched, + observation(plan, route=True, route_job="running"), + plan, + now_unix=NOW, + ) + assert running["phase"] == "ROUTE_ACCEPTING" + assert running["next_action"] == "none" + + missing = controller.reconcile( + dispatched, + observation(plan, route=True, route_job="absent"), + plan, + now_unix=NOW, + ) + + assert missing["phase"] == "CLEANING_FAILED" + assert missing["failure_code"] == "route_acceptance_disappeared" + assert missing["next_action"] == "cleanup_failure" + + +def test_client_start_intent_cannot_rearm_after_dispatch(plan): + route_ready = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, route_job="passed"), + plan, + now_unix=NOW, + ) + windows_dispatched = controller.begin_action(route_ready, plan, action="start_windows") + + provisioning = controller.reconcile( + windows_dispatched, + observation(plan, route=True, windows=True, route_job="passed", windows_job="absent"), + plan, + now_unix=NOW, + ) + assert provisioning["phase"] == "WINDOWS_RUNNING" + assert provisioning["next_action"] == "none" + + missing = controller.reconcile( + windows_dispatched, + observation(plan, route=True, route_job="passed"), + plan, + now_unix=NOW, + ) + assert missing["phase"] == "CLEANING_FAILED" + assert missing["failure_code"] == "windows_disappeared_after_start_intent" + + linux_ready = dict(route_ready) + linux_ready.update( + { + "phase": "WINDOWS_COLLECTED", + "windows_evidence_digest": WINDOWS_DIGEST, + "windows_consumed": True, + "next_action": "start_linux", + } + ) + linux_dispatched = controller.begin_action(linux_ready, plan, action="start_linux") + linux_provisioning = controller.reconcile( + linux_dispatched, + observation(plan, route=True, linux=True, route_job="passed", linux_job="absent"), + plan, + now_unix=NOW, + ) + assert linux_provisioning["phase"] == "LINUX_RUNNING" + assert linux_provisioning["next_action"] == "none" + + linux_missing = controller.reconcile( + linux_dispatched, + observation(plan, route=True, route_job="passed"), + plan, + now_unix=NOW, + ) + assert linux_missing["phase"] == "CLEANING_FAILED" + assert linux_missing["failure_code"] == "linux_disappeared_after_start_intent" + + +def test_observed_attempt_cannot_disappear_and_relaunch(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, route_job="passed"), + plan, + now_unix=NOW, + ) + disappeared = observation(plan, route=True, route_job="passed") + disappeared["clients"]["windows"]["attempt_ordinal"] = 1 + + failed = controller.reconcile(state, disappeared, plan, now_unix=NOW) + assert failed["phase"] == "CLEANING_FAILED" + assert failed["failure_code"] == "windows_attempt_disappeared" + + +def test_active_host_job_is_observed_not_relaunched(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, windows=True, route_job="passed", windows_job="running"), + plan, + now_unix=NOW, + ) + + assert state["phase"] == "WINDOWS_RUNNING" + assert state["next_action"] == "none" + + +def test_collect_binds_canonical_evidence_then_deletes_windows(monkeypatch, plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, windows=True, route_job="passed", windows_job="passed"), + plan, + now_unix=NOW, + ) + assert state["phase"] == "WINDOWS_COLLECTING" + + payload = b'{"bounded":true}' + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + monkeypatch.setattr(controller.lifecycle, "load_lifecycle_json", lambda _payload: {"validated": True}) + monkeypatch.setattr( + controller.lifecycle, + "validate_lifecycle_document", + lambda _raw: { + "source_commit": plan.windows_source_commit, + "package_sha256": plan.windows_package_sha256, + "package_bytes": plan.windows_package_bytes, + "model_id": "Qwen3.5 2B", + "manifest_digest": plan.qwen_manifest.removeprefix("sha256:"), + }, + ) + + collected = controller.collect_platform( + state, + plan, + platform="windows", + evidence_payload=payload, + observed_digest=digest, + ) + assert collected["phase"] == "WINDOWS_DELETING" + assert collected["next_action"] == "delete_windows" + assert collected["windows_consumed"] is True + + deleting = observation(plan, route=True, route_job="passed") + deleting["clients"]["windows"]["attempt_ordinal"] = 1 + deleting["disks"][plan.windows_disk] = True + still_present = controller.reconcile(collected, deleting, plan, now_unix=NOW) + assert still_present["phase"] == "WINDOWS_DELETING" + assert still_present["next_action"] == "delete_windows" + with pytest.raises(controller.RunControllerError, match="absence is not proved"): + controller.mark_client_absent( + collected, + plan, + platform="windows", + observation=deleting, + now_unix=NOW, + ) + + absent = observation(plan, route=True, route_job="passed") + absent["clients"]["windows"]["attempt_ordinal"] = 1 + after_delete = controller.mark_client_absent( + collected, + plan, + platform="windows", + observation=absent, + now_unix=NOW, + ) + assert after_delete["phase"] == "WINDOWS_COLLECTED" + assert after_delete["next_action"] == "start_linux" + + +def test_collect_accepts_current_automated_desktop_replay(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, windows=True, route_job="passed", windows_job="passed"), + plan, + now_unix=NOW, + ) + evidence = { + "schema_version": 2, + "scope": "gate13-automated-desktop-replay", + "run_id": f"{plan.run_id}-windows", + "platform": "windows", + "result": "passed", + "source_commit": plan.windows_source_commit, + "package": { + "sha256": "sha256:" + plan.windows_package_sha256.removeprefix("sha256:"), + "bytes": plan.windows_package_bytes, + "verified_before_run": True, + "self_test_count": 4, + }, + "model_id": "Qwen3.5 2B", + "manifest_digest": "sha256:" + plan.qwen_manifest.removeprefix("sha256:"), + "real_window_sessions": 2, + "localhost_inference_count": 1, + "policy_dialog_saved": True, + "start_clicked": True, + "pause_control_observed": True, + "restart_resume_observed": False, + "pause_clicked": True, + "sharing_intent_paused": True, + "policy_profile": "gate13-manual-cpu-v1", + "sequence_profile": "gate13-manual-windows-v1", + "start_observation_seconds": 25.0, + "session_duration_seconds": {"initial": 120.0, "restart": 90.0}, + "privacy_safe": True, + "qualification_temporaries_removed": True, + } + payload = (json.dumps(evidence, sort_keys=True, separators=(",", ":")) + "\n").encode() + digest = "sha256:" + hashlib.sha256(payload).hexdigest() + + collected = controller.collect_platform( + state, + plan, + platform="windows", + evidence_payload=payload, + observed_digest=digest, + ) + + assert collected["phase"] == "WINDOWS_DELETING" + assert collected["windows_consumed"] is True + + +def test_partial_or_wrong_digest_evidence_cannot_advance(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, route=True, windows=True, route_job="passed", windows_job="passed"), + plan, + now_unix=NOW, + ) + + with pytest.raises(controller.RunControllerError, match="digest changed"): + controller.collect_platform( + state, + plan, + platform="windows", + evidence_payload=b"{}", + observed_digest=WINDOWS_DIGEST, + ) + + +def test_success_requires_both_records_and_exact_absence(plan): + state = controller.initial_state(plan) + state.update( + { + "phase": "LINUX_COLLECTED", + "route_acceptance_digest": ROUTE_DIGEST, + "windows_evidence_digest": WINDOWS_DIGEST, + "linux_evidence_digest": LINUX_DIGEST, + "windows_consumed": True, + "linux_consumed": True, + "next_action": "delete_route", + } + ) + + complete = controller.reconcile(state, observation(plan), plan, now_unix=NOW) + assert complete["phase"] == "CLEANED_PASS" + assert complete["cleanup_verified"] is True + assert complete["next_action"] == "none" + + +def test_failure_cleanup_is_idempotent_and_never_becomes_pass(plan): + state = controller.initial_state(plan) + state.update( + { + "phase": "CLEANING_FAILED", + "failure_code": "windows_failed_or_ambiguous", + "windows_consumed": True, + "next_action": "cleanup_failure", + } + ) + + cleaned = controller.reconcile(state, observation(plan), plan, now_unix=NOW) + assert cleaned["phase"] == "CLEANED_FAILURE" + assert controller.reconcile(cleaned, observation(plan), plan, now_unix=NOW) == cleaned + + +def test_stale_observation_and_expired_deadline_fail_closed(plan): + stale = observation(plan) + stale["observed_at_unix"] = NOW - 301 + with pytest.raises(controller.RunControllerError, match="stale"): + controller.reconcile(controller.initial_state(plan), stale, plan, now_unix=NOW) + + expired = observation(plan, route=True, route_job="running") + expired["instances"][plan.route_instance]["termination_unix"] = NOW + with pytest.raises(controller.RunControllerError, match="deadline expired"): + controller.reconcile(controller.initial_state(plan), expired, plan, now_unix=NOW) + + +def test_atomic_state_round_trip_and_public_status_are_bounded(tmp_path, plan): + state_path = tmp_path / "state.json" + state = controller.initial_state(plan) + controller.persist(state_path, state, plan) + + assert controller.load_state(state_path, plan) == state + public = controller.public_status(state, plan) + assert set(public) == { + "schema_version", + "run_id", + "phase", + "next_action", + "failure_code", + "windows_consumed", + "linux_consumed", + "cleanup_verified", + } + rendered = json.dumps(public) + for forbidden in ("token", "password", "prompt", "endpoint", str(tmp_path)): + assert forbidden not in rendered.lower() diff --git a/tests/test_gate13_windows_packaged_lifecycle.py b/tests/test_gate13_windows_packaged_lifecycle.py index b850aade3..997bae485 100644 --- a/tests/test_gate13_windows_packaged_lifecycle.py +++ b/tests/test_gate13_windows_packaged_lifecycle.py @@ -81,6 +81,22 @@ def test_json_input_bound_accepts_production_scale_and_rejects_above_limit(tmp_p } +@pytest.mark.skipif(not POWERSHELL.is_file(), reason="native Windows PowerShell is required") +def test_sha256_does_not_depend_on_powershell_module_autoload(tmp_path): + target = tmp_path / "payload.bin" + target.write_bytes(b"clean-host-hash") + source = f""" +. {_ps_literal(LIFECYCLE)} +Remove-Module Microsoft.PowerShell.Utility -Force -ErrorAction Stop +$PSModuleAutoLoadingPreference = 'None' +[Console]::Out.WriteLine((Get-Gate13Sha256 -Path {_ps_literal(target)})) +""" + + result = _run_powershell(source, tmp_path) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == hashlib.sha256(target.read_bytes()).hexdigest() + + @pytest.mark.skipif(not POWERSHELL.is_file(), reason="native Windows PowerShell is required") def test_windows_build_platform_accepts_production_runner_and_rejects_spoofs(tmp_path): source = f""" @@ -588,6 +604,12 @@ def test_adapter_contains_exact_safety_and_lifecycle_contracts(): positions = [lifecycle.index(f'-Name "{phase}"') for phase in phases] assert positions == sorted(positions) assert lifecycle.count('-Name "') >= len(phases) + assert "$script:LifecycleFailurePhase = $Name" in lifecycle + assert "$script:LifecycleFailureOperation = $Name" in lifecycle + assert "failure_phase = $failurePhase" in lifecycle + assert "failure_operation = $failureOperation" in lifecycle + assert '"product_readiness"' in lifecycle + assert "ConvertTo-Json -Compress" in lifecycle for required in ( "CreateSuspended", diff --git a/tests/test_gate14_hardware_acceptance.py b/tests/test_gate14_hardware_acceptance.py new file mode 100644 index 000000000..ad20c2a53 --- /dev/null +++ b/tests/test_gate14_hardware_acceptance.py @@ -0,0 +1,492 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_hardware_acceptance as acceptance # noqa: E402 + +RUN_ID = "gate14-20260902-a" +CONTROLLER_SOURCE = "2" * 40 +SOURCE = CONTROLLER_SOURCE +DIGEST = "sha256:" + "a" * 64 +PROJECT = "community-ai-506321" +ZONE = "us-central1-a" +INSTANCES = ("gate14-20260902-a-windows", "gate14-20260902-a-linux") +DISKS = ("gate14-20260902-a-windows-disk", "gate14-20260902-a-linux-disk") + + +def provider_plan_document() -> dict: + return { + "project": PROJECT, + "zone": ZONE, + "clients": [ + { + "platform": platform, + "instance": INSTANCES[index], + "disk": DISKS[index], + "source_commit": SOURCE, + "termination_unix": 2_000_010_000 + index, + "package_sha256": DIGEST, + "model_id": acceptance.EXPECTED_PLATFORM_MODELS[platform], + "manifest_digest": acceptance.MODEL_PROFILES[acceptance.EXPECTED_PLATFORM_MODELS[platform]][ + "manifest_digest" + ], + } + for index, platform in enumerate(("windows", "linux")) + ], + "sequencing": { + "clients_may_run_concurrently": False, + "windows_first": True, + "fresh_host_per_platform": True, + }, + } + + +PROVIDER_PLAN = provider_plan_document() +PLAN_DIGEST = ( + "sha256:" + + hashlib.sha256(json.dumps(PROVIDER_PLAN, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() +) + + +def authorization_document() -> dict: + return { + "schema_version": 1, + "gate": 14, + "result": "authorized", + "run_id": RUN_ID, + "source_commit": CONTROLLER_SOURCE, + "provider_plan_digest": PLAN_DIGEST, + "provider_plan": PROVIDER_PLAN, + "authorization": { + "combined_cloud_ceiling_usd": "100.00", + "ledger_committed_before_run_usd": "56.00", + "maximum_estimate_usd": "44.00", + "remaining_after_run_maximum_usd": "0.00", + "reservation_recorded": True, + "native_auth_revalidated": True, + "provisioning_authorized_after_fail_closed_preflight": True, + }, + "prohibited": {"credits": 0, "macos": 0, "fly_gpu": 0}, + } + + +def platform_document(platform: str) -> dict: + model_id = acceptance.EXPECTED_PLATFORM_MODELS[platform] + profile = acceptance.MODEL_PROFILES[model_id] + return { + "schema_version": 1, + "scope": acceptance.PLATFORM_SCOPE, + "run_id": RUN_ID, + "platform": platform, + "result": "passed", + "source_commit": SOURCE, + "gate13_evidence_sha256": acceptance.EXPECTED_GATE13_EVIDENCE_SHA256, + "package": { + "source_commit": SOURCE, + "archive_sha256": DIGEST, + "archive_bytes": 1024, + "release_metadata_sha256": "sha256:" + "b" * 64, + }, + "model": { + "id": model_id, + "manifest_digest": profile["manifest_digest"], + "revision_commit": profile["revision_commit"], + "gate9_envelope_sha256": acceptance.EXPECTED_GATE9_ENVELOPES[platform], + "selected_artifact_count": profile["selected_artifact_count"], + "selected_artifact_bytes": profile["selected_artifact_bytes"], + "total_blocks": profile["total_blocks"], + }, + "hardware": { + "os_name": "Windows Server 2022" if platform == "windows" else "Ubuntu 24.04", + "accelerator": "NVIDIA L4", + "accelerator_count": 1, + "accelerator_memory_bytes": 24 * 1024**3, + }, + "cache": { + "verified_bytes_before": profile["selected_artifact_bytes"], + "verified_bytes_after": profile["selected_artifact_bytes"], + "transfer_bytes_during_gate": 0, + "digest_mismatch_count": 0, + "forbidden_model_acquired": False, + }, + "placement": { + "automatic": True, + "worker_count": 1, + "block_start": 0, + "block_end": min(4, profile["total_blocks"]), + "intent_published": True, + "remote_acknowledged": True, + }, + "limits": { + "disk_bytes": 16 * 1024**3, + "vram_bytes": 20 * 1024**3, + "bandwidth_mbps": 100.0, + "power_watts": 250.0, + "schedule_timezone": "UTC", + "resource_limit_count": 5, + "configured_and_resolved_match": True, + "low_vram_rejected": True, + }, + "suspensions": [ + { + "kind": kind, + "suspended": True, + "resumed": True, + "desired_intent_preserved": True, + "worker_count_during": 0, + "duration_seconds": 2.5, + } + for kind in ("bandwidth", "power", "schedule") + ], + "recovery": { + "worker_crash_observed": True, + "worker_restarted": True, + "restart_seconds": 4.5, + "previous_worker_absent": True, + "manifest_unchanged": True, + "automatic_block_range_valid": True, + "desired_intent_preserved": True, + }, + "pause": { + "requested": True, + "completed": True, + "duration_seconds": 1.5, + "worker_count_after": 0, + "descendant_count_after": 0, + }, + "restart": { + "node_restarted": True, + "policy_persisted": True, + "desired_intent_persisted": True, + "worker_resumed": True, + "duration_seconds": 8.0, + "cache_reused": True, + }, + "unsupported_telemetry": { + "device": "cpu", + "configured_limit": "power_watts", + "start_rejected": True, + "reason_code": "power-telemetry-unavailable", + "private_detail_retained": False, + }, + "privacy": { + "prompt_retained": False, + "response_retained": False, + "token_identifiers_retained": False, + "credentials_retained": False, + "paths_retained": False, + "endpoints_retained": False, + "provider_output_retained": False, + }, + "qualification_temporaries_removed": True, + } + + +def cleanup_document(terminal_state_sha256: str) -> dict: + return { + "schema_version": 1, + "scope": acceptance.CLEANUP_SCOPE, + "run_id": RUN_ID, + "result": "passed", + "provider": "GCP", + "controller_source_commit": CONTROLLER_SOURCE, + "provider_plan_digest": PLAN_DIGEST, + "project": PROJECT, + "zone": ZONE, + "deleted_instances": list(INSTANCES), + "deleted_disks": list(DISKS), + "controller_terminal_state_sha256": terminal_state_sha256, + "native_auth_revalidated": True, + "expected_instances": 2, + "remaining_instances": 0, + "expected_disks": 2, + "remaining_disks": 0, + "remaining_firewalls": 0, + "l4_usage": 0, + "protected_bootstrap_running": True, + "product_processes_remaining": 0, + "temporary_credentials_remaining": 0, + } + + +def write_documents(tmp_path: Path) -> tuple[Path, Path, Path, Path, Path]: + windows_path = tmp_path / "windows.json" + linux_path = tmp_path / "linux.json" + cleanup_path = tmp_path / "cleanup.json" + terminal_state_path = tmp_path / "state.json" + authorization_path = tmp_path / "authorization.json" + windows_path.write_text(json.dumps(platform_document("windows")), encoding="utf-8") + linux_path.write_text(json.dumps(platform_document("linux")), encoding="utf-8") + authorization_path.write_text(json.dumps(authorization_document()), encoding="utf-8") + windows_digest = "sha256:" + hashlib.sha256(windows_path.read_bytes()).hexdigest() + linux_digest = "sha256:" + hashlib.sha256(linux_path.read_bytes()).hexdigest() + authorization_digest = "sha256:" + hashlib.sha256(authorization_path.read_bytes()).hexdigest() + terminal_state = { + "schema_version": 1, + "run_id": RUN_ID, + "authorization_sha256": authorization_digest, + "provider_plan_digest": PLAN_DIGEST, + "revision": 10, + "phase": "CLEANED_PASS", + "failure_code": None, + "windows_evidence_digest": windows_digest, + "linux_evidence_digest": linux_digest, + "windows_consumed": True, + "linux_consumed": True, + "cleanup_verified": True, + "next_action": "none", + } + terminal_state_path.write_text(json.dumps(terminal_state), encoding="utf-8") + terminal_digest = "sha256:" + hashlib.sha256(terminal_state_path.read_bytes()).hexdigest() + cleanup_path.write_text( + json.dumps(cleanup_document(terminal_digest)), + encoding="utf-8", + ) + return windows_path, linux_path, cleanup_path, terminal_state_path, authorization_path + + +def validate_documents(paths: tuple[Path, Path, Path, Path, Path]) -> dict: + windows, linux, cleanup, terminal_state, authorization = paths + return acceptance.validate_files( + windows, + linux, + cleanup, + CONTROLLER_SOURCE, + provider_plan_digest=PLAN_DIGEST, + project=PROJECT, + zone=ZONE, + expected_instances=INSTANCES, + expected_disks=DISKS, + terminal_state_path=terminal_state, + authorization_path=authorization, + ) + + +def test_validate_platform_documents_cover_both_models_and_hardware_contract(): + windows = acceptance.validate_platform_document(platform_document("windows")) + linux = acceptance.validate_platform_document(platform_document("linux")) + + assert windows["model_id"] == "Qwen3.5 2B" + assert linux["model_id"] == "Gemma 4 E2B IT" + assert windows["accelerator"] == linux["accelerator"] == "NVIDIA L4" + assert windows["block_start"] == 0 + assert windows["block_end"] == 4 + + +def test_validate_files_emits_digest_bound_privacy_safe_aggregate(tmp_path): + paths = write_documents(tmp_path) + + result = validate_documents(paths) + + assert result["scope"] == acceptance.AGGREGATE_SCOPE + assert result["result"] == "passed" + assert result["controller_source_commit"] == CONTROLLER_SOURCE + assert result["package_source_commit"] == SOURCE + assert [item["platform"] for item in result["platforms"]] == ["windows", "linux"] + assert all(item["evidence_sha256"].startswith("sha256:") for item in result["platforms"]) + assert result["cleanup"]["resource_absence_proved"] is True + assert result["credits_in_scope"] is False + assert result["macos_in_scope"] is False + assert result["privacy_safe"] is True + + +@pytest.mark.parametrize( + "mutator", + [ + lambda value: value.update(gate13_evidence_sha256="sha256:" + "0" * 64), + lambda value: value["model"].update(gate9_envelope_sha256="sha256:" + "0" * 64), + lambda value: value["hardware"].update(os_name="Ubuntu 24.04"), + lambda value: value["cache"].update(transfer_bytes_during_gate=1), + lambda value: value["placement"].update(remote_acknowledged=False), + lambda value: value["limits"].update(low_vram_rejected=False), + lambda value: value["limits"].update(power_watts=None), + lambda value: value["suspensions"].pop(), + lambda value: value["suspensions"][0].update(resumed=False), + lambda value: value["recovery"].update(worker_restarted=False), + lambda value: value["pause"].update(descendant_count_after=1), + lambda value: value["restart"].update(policy_persisted=False), + lambda value: value["unsupported_telemetry"].update(start_rejected=False), + lambda value: value["privacy"].update(paths_retained=True), + lambda value: value.update(qualification_temporaries_removed=False), + ], +) +def test_platform_evidence_fails_closed(mutator): + value = platform_document("windows") + mutator(value) + + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_platform_document(value) + + +def test_wrong_model_and_unsafe_block_range_fail_closed(): + value = platform_document("windows") + value["model"] = platform_document("linux")["model"] + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_platform_document(value) + + value = platform_document("windows") + value["placement"]["block_end"] = value["placement"]["block_start"] + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_platform_document(value) + + +def test_aggregate_rejects_mismatched_run_source_and_incomplete_cleanup(tmp_path): + paths = write_documents(tmp_path) + linux = paths[1] + linux_value = json.loads(linux.read_text(encoding="utf-8")) + linux_value["run_id"] = "gate14-20260902-b" + linux.write_text(json.dumps(linux_value), encoding="utf-8") + with pytest.raises(acceptance.Gate14EvidenceError): + validate_documents(paths) + + paths = write_documents(tmp_path) + cleanup = paths[2] + cleanup_value = json.loads(cleanup.read_text(encoding="utf-8")) + cleanup_value["remaining_disks"] = 1 + cleanup.write_text(json.dumps(cleanup_value), encoding="utf-8") + with pytest.raises(acceptance.Gate14EvidenceError): + validate_documents(paths) + + +@pytest.mark.parametrize( + ("field", "replacement"), + [ + ("controller_source_commit", "3" * 40), + ("provider_plan_digest", "sha256:" + "3" * 64), + ("project", "different-project"), + ("zone", "us-east1-b"), + ("deleted_instances", list(reversed(INSTANCES))), + ("deleted_disks", list(reversed(DISKS))), + ("controller_terminal_state_sha256", "sha256:" + "3" * 64), + ], +) +def test_cleanup_must_bind_exact_plan_resources_and_terminal_state( + tmp_path, + field, + replacement, +): + paths = write_documents(tmp_path) + cleanup = paths[2] + value = json.loads(cleanup.read_text(encoding="utf-8")) + value[field] = replacement + cleanup.write_text(json.dumps(value), encoding="utf-8") + + with pytest.raises(acceptance.Gate14EvidenceError): + validate_documents(paths) + + +def test_terminal_state_must_be_a_real_digest_bound_pass(tmp_path): + paths = write_documents(tmp_path) + cleanup = paths[2] + terminal_state = paths[3] + state_value = json.loads(terminal_state.read_text(encoding="utf-8")) + state_value["cleanup_verified"] = False + terminal_state.write_text(json.dumps(state_value), encoding="utf-8") + terminal_digest = "sha256:" + hashlib.sha256(terminal_state.read_bytes()).hexdigest() + cleanup_value = json.loads(cleanup.read_text(encoding="utf-8")) + cleanup_value["controller_terminal_state_sha256"] = terminal_digest + cleanup.write_text(json.dumps(cleanup_value), encoding="utf-8") + + with pytest.raises(acceptance.Gate14EvidenceError): + validate_documents(paths) + + +def test_protected_bootstrap_cannot_enter_cleanup_inventory(tmp_path): + windows, linux, cleanup, terminal_state, authorization = write_documents(tmp_path) + + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance.validate_files( + windows, + linux, + cleanup, + CONTROLLER_SOURCE, + provider_plan_digest=PLAN_DIGEST, + project=PROJECT, + zone=ZONE, + expected_instances=(acceptance.PROTECTED_INSTANCE, INSTANCES[1]), + expected_disks=DISKS, + terminal_state_path=terminal_state, + authorization_path=authorization, + ) + + +def test_terminal_state_binds_exact_semantic_authorization_file(tmp_path): + paths = write_documents(tmp_path) + cleanup = paths[2] + terminal_state = paths[3] + authorization = paths[4] + authorization_value = json.loads(authorization.read_text(encoding="utf-8")) + authorization_value["source_commit"] = "3" * 40 + authorization.write_text(json.dumps(authorization_value), encoding="utf-8") + authorization_digest = "sha256:" + hashlib.sha256(authorization.read_bytes()).hexdigest() + + state_value = json.loads(terminal_state.read_text(encoding="utf-8")) + state_value["authorization_sha256"] = authorization_digest + terminal_state.write_text(json.dumps(state_value), encoding="utf-8") + terminal_digest = "sha256:" + hashlib.sha256(terminal_state.read_bytes()).hexdigest() + cleanup_value = json.loads(cleanup.read_text(encoding="utf-8")) + cleanup_value["controller_terminal_state_sha256"] = terminal_digest + cleanup.write_text(json.dumps(cleanup_value), encoding="utf-8") + + with pytest.raises(acceptance.Gate14EvidenceError): + validate_documents(paths) + + +def test_duplicate_and_non_finite_json_fail_closed(): + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance._strict_json(b'{"schema_version":1,"schema_version":1}') + with pytest.raises(acceptance.Gate14EvidenceError): + acceptance._strict_json(b'{"value":NaN}') + + +def test_cli_prints_canonical_aggregate(tmp_path): + windows, linux, cleanup, terminal_state, authorization = write_documents(tmp_path) + + completed = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "gate14_hardware_acceptance.py"), + "--windows", + str(windows), + "--linux", + str(linux), + "--cleanup", + str(cleanup), + "--controller-state", + str(terminal_state), + "--authorization", + str(authorization), + "--controller-source-commit", + CONTROLLER_SOURCE, + "--provider-plan-digest", + PLAN_DIGEST, + "--project", + PROJECT, + "--zone", + ZONE, + "--instances", + *INSTANCES, + "--disks", + *DISKS, + ], + check=True, + capture_output=True, + text=True, + ) + + result = json.loads(completed.stdout) + assert result["result"] == "passed" + assert completed.stdout.strip() == json.dumps( + result, + sort_keys=True, + separators=(",", ":"), + ) diff --git a/tests/test_gate14_run_controller.py b/tests/test_gate14_run_controller.py new file mode 100644 index 000000000..7ff2fb13c --- /dev/null +++ b/tests/test_gate14_run_controller.py @@ -0,0 +1,729 @@ +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import gate14_hardware_acceptance as acceptance # noqa: E402 +import gate14_run_controller as controller # noqa: E402 + +RUN_ID = "gate14-20260902-a" +SOURCE = "1" * 40 +NOW = 2_000_000_000 +PACKAGE_DIGESTS = { + "windows": "sha256:" + "a" * 64, + "linux": "sha256:" + "b" * 64, +} + + +def provider_plan() -> dict: + clients = [] + for index, platform in enumerate(("windows", "linux"), start=1): + model_id = acceptance.EXPECTED_PLATFORM_MODELS[platform] + clients.append( + { + "platform": platform, + "instance": f"gate14-20260902-a-{platform}", + "disk": f"gate14-20260902-a-{platform}-disk", + "source_commit": SOURCE, + "termination_unix": NOW + index * 10_000, + "package_sha256": PACKAGE_DIGESTS[platform], + "model_id": model_id, + "manifest_digest": acceptance.MODEL_PROFILES[model_id]["manifest_digest"], + } + ) + return { + "project": "community-ai-506321", + "zone": "us-central1-a", + "clients": clients, + "sequencing": { + "clients_may_run_concurrently": False, + "windows_first": True, + "fresh_host_per_platform": True, + }, + } + + +def write_plan( + tmp_path: Path, + *, + additional_current_maximum: str | None = None, + hide_additional_below_anchor: bool = False, +) -> controller.RunPlan: + provider = provider_plan() + digest = controller._canonical_digest(provider) + authorization = { + "schema_version": 1, + "gate": 14, + "result": "authorized", + "run_id": RUN_ID, + "source_commit": SOURCE, + "provider_plan_digest": digest, + "provider_plan": provider, + "authorization": { + "combined_cloud_ceiling_usd": "100.00", + "ledger_committed_before_run_usd": "56.00", + "maximum_estimate_usd": "44.00", + "remaining_after_run_maximum_usd": "0.00", + "reservation_recorded": True, + "native_auth_revalidated": True, + "provisioning_authorized_after_fail_closed_preflight": True, + }, + "prohibited": {"credits": 0, "macos": 0, "fly_gpu": 0}, + } + authorization_path = tmp_path / "authorization.json" + authorization_path.write_text(json.dumps(authorization), encoding="utf-8") + ledger_path = tmp_path / "ledger.md" + ledger_lines = [ + "## Cloud authorization and spend ledger", + "", + "| Run | Provider | Purpose | Maximum estimate | Observed cost | Cleanup proof | State |", + "| --- | --- | --- | ---: | ---: | --- | --- |", + f"| {RUN_ID} | GCP | Gate 14 packaged hardware [plan {digest}] | USD 44.00 | — | — | RESERVED |", + ] + additional_row = ( + "| gate14-prior-run | GCP | Unexpected same-epoch reservation | " + f"USD {additional_current_maximum} | — | — | RESERVED |" + if additional_current_maximum is not None + else None + ) + if additional_row is not None and not hide_additional_below_anchor: + ledger_lines.append(additional_row) + ledger_lines.append( + "| gate13-20260901-a | GCP | Current epoch anchor | " + "USD 56.00 | — | Existing cleanup proof | CLEANED-COMMITTED |" + ) + if additional_row is not None and hide_additional_below_anchor: + ledger_lines.append(additional_row) + ledger_lines.append("") + ledger_path.write_text("\n".join(ledger_lines), encoding="utf-8") + return controller.load_plan(authorization_path, ledger_path) + + +def observation( + plan: controller.RunPlan, + *, + windows: bool = False, + linux: bool = False, + windows_job: str = "absent", + linux_job: str = "absent", + windows_digest: str | None = None, + linux_digest: str | None = None, + windows_disk: bool | None = None, + linux_disk: bool | None = None, +) -> dict: + present = {"windows": windows, "linux": linux} + jobs = {"windows": windows_job, "linux": linux_job} + digests = {"windows": windows_digest, "linux": linux_digest} + disks = { + "windows": windows if windows_disk is None else windows_disk, + "linux": linux if linux_disk is None else linux_disk, + } + return { + "schema_version": 1, + "run_id": plan.run_id, + "observed_at_unix": NOW, + "instances": { + client.instance: { + "present": present[client.platform], + "run_id": plan.run_id if present[client.platform] else None, + "source_commit": client.source_commit if present[client.platform] else None, + "termination_unix": client.termination_unix if present[client.platform] else None, + } + for client in (plan.windows, plan.linux) + }, + "disks": {client.disk: disks[client.platform] for client in (plan.windows, plan.linux)}, + "clients": { + platform: { + "job_state": jobs[platform], + "attempt_ordinal": 1 if jobs[platform] != "absent" else 0, + "evidence_digest": digests[platform] if jobs[platform] == "passed" else None, + } + for platform in ("windows", "linux") + }, + "l4_usage": int(windows or linux), + "protected_bootstrap_running": True, + } + + +def platform_evidence(platform: str) -> dict: + model_id = acceptance.EXPECTED_PLATFORM_MODELS[platform] + profile = acceptance.MODEL_PROFILES[model_id] + selected = profile["selected_artifact_bytes"] + return { + "schema_version": 1, + "scope": acceptance.PLATFORM_SCOPE, + "run_id": RUN_ID, + "platform": platform, + "result": "passed", + "source_commit": SOURCE, + "gate13_evidence_sha256": acceptance.EXPECTED_GATE13_EVIDENCE_SHA256, + "package": { + "source_commit": SOURCE, + "archive_sha256": PACKAGE_DIGESTS[platform], + "archive_bytes": 1024, + "release_metadata_sha256": "sha256:" + "e" * 64, + }, + "model": { + "id": model_id, + "manifest_digest": profile["manifest_digest"], + "revision_commit": profile["revision_commit"], + "gate9_envelope_sha256": acceptance.EXPECTED_GATE9_ENVELOPES[platform], + "selected_artifact_count": profile["selected_artifact_count"], + "selected_artifact_bytes": selected, + "total_blocks": profile["total_blocks"], + }, + "hardware": { + "os_name": "Windows Server 2022" if platform == "windows" else "Ubuntu 24.04", + "accelerator": "NVIDIA L4", + "accelerator_count": 1, + "accelerator_memory_bytes": 24 * 1024**3, + }, + "cache": { + "verified_bytes_before": selected, + "verified_bytes_after": selected, + "transfer_bytes_during_gate": 0, + "digest_mismatch_count": 0, + "forbidden_model_acquired": False, + }, + "placement": { + "automatic": True, + "worker_count": 1, + "block_start": 0, + "block_end": 4, + "intent_published": True, + "remote_acknowledged": True, + }, + "limits": { + "disk_bytes": 16 * 1024**3, + "vram_bytes": 20 * 1024**3, + "bandwidth_mbps": 100.0, + "power_watts": 250.0, + "schedule_timezone": "UTC", + "resource_limit_count": 5, + "configured_and_resolved_match": True, + "low_vram_rejected": True, + }, + "suspensions": [ + { + "kind": kind, + "suspended": True, + "resumed": True, + "desired_intent_preserved": True, + "worker_count_during": 0, + "duration_seconds": 3.0, + } + for kind in ("bandwidth", "power", "schedule") + ], + "recovery": { + "worker_crash_observed": True, + "worker_restarted": True, + "restart_seconds": 3.0, + "previous_worker_absent": True, + "manifest_unchanged": True, + "automatic_block_range_valid": True, + "desired_intent_preserved": True, + }, + "pause": { + "requested": True, + "completed": True, + "duration_seconds": 3.0, + "worker_count_after": 0, + "descendant_count_after": 0, + }, + "restart": { + "node_restarted": True, + "policy_persisted": True, + "desired_intent_persisted": True, + "worker_resumed": True, + "duration_seconds": 3.0, + "cache_reused": True, + }, + "unsupported_telemetry": { + "device": "cpu", + "configured_limit": "power_watts", + "start_rejected": True, + "reason_code": "power-telemetry-unavailable", + "private_detail_retained": False, + }, + "privacy": { + "prompt_retained": False, + "response_retained": False, + "token_identifiers_retained": False, + "credentials_retained": False, + "paths_retained": False, + "endpoints_retained": False, + "provider_output_retained": False, + }, + "qualification_temporaries_removed": True, + } + + +@pytest.fixture +def plan(tmp_path): + return write_plan(tmp_path) + + +def test_load_plan_binds_budget_sequence_models_and_exact_resources(plan): + assert plan.run_id == RUN_ID + assert plan.ledger_state == "RESERVED" + assert plan.instances == ( + "gate14-20260902-a-windows", + "gate14-20260902-a-linux", + ) + assert plan.windows.model_id == "Qwen3.5 2B" + assert plan.linux.model_id == "Gemma 4 E2B IT" + assert controller.PROTECTED_INSTANCE not in plan.instances + + +def test_load_plan_rejects_spend_above_remaining_ceiling(tmp_path): + provider = provider_plan() + digest = controller._canonical_digest(provider) + authorization = { + "schema_version": 1, + "gate": 14, + "result": "authorized", + "run_id": RUN_ID, + "source_commit": SOURCE, + "provider_plan_digest": digest, + "provider_plan": provider, + "authorization": { + "combined_cloud_ceiling_usd": "100.00", + "ledger_committed_before_run_usd": "56.00", + "maximum_estimate_usd": "45.00", + "remaining_after_run_maximum_usd": "-1.00", + "reservation_recorded": True, + "native_auth_revalidated": True, + "provisioning_authorized_after_fail_closed_preflight": True, + }, + "prohibited": {"credits": 0, "macos": 0, "fly_gpu": 0}, + } + authorization_path = tmp_path / "bad.json" + authorization_path.write_text(json.dumps(authorization), encoding="utf-8") + ledger_path = tmp_path / "ledger.md" + ledger_path.write_text( + "\n".join( + ( + "## Cloud authorization and spend ledger", + "| Run | Provider | Purpose | Maximum estimate | Observed cost | Cleanup proof | State |", + "| --- | --- | --- | ---: | ---: | --- | --- |", + f"| {RUN_ID} | GCP | Gate 14 [plan {digest}] | USD 45.00 | — | — | RESERVED |", + ) + ), + encoding="utf-8", + ) + with pytest.raises(controller.Gate14ControllerError): + controller.load_plan(authorization_path, ledger_path) + + +def test_lifecycle_reattaches_collects_serially_and_cleanup_passes( + tmp_path, + plan, +): + state = controller.initial_state(plan) + assert state["next_action"] == "none" + + state = controller.reconcile(state, observation(plan), plan) + assert state["phase"] == "ABSENT" + assert state["next_action"] == "start_windows" + + state = controller.reconcile( + state, + observation(plan, windows=True, windows_job="running"), + plan, + ) + assert state["phase"] == "WINDOWS_RUNNING" + assert state["windows_consumed"] is True + assert state["next_action"] == "none" + + windows_path = tmp_path / "windows.json" + windows_path.write_text( + json.dumps(platform_evidence("windows")), + encoding="utf-8", + ) + windows_digest = "sha256:" + hashlib.sha256(windows_path.read_bytes()).hexdigest() + state = controller.reconcile( + state, + observation( + plan, + windows=True, + windows_job="passed", + windows_digest=windows_digest, + ), + plan, + ) + assert state["next_action"] == "collect_windows" + state = controller.collect_platform(state, plan, "windows", windows_path) + assert state["phase"] == "WINDOWS_DELETING" + assert state["next_action"] == "delete_windows" + + state = controller.reconcile( + state, + observation( + plan, + windows_job="passed", + windows_digest=windows_digest, + ), + plan, + ) + assert state["phase"] == "WINDOWS_DELETING" + assert state["next_action"] == "start_linux" + + state = controller.reconcile( + state, + observation( + plan, + linux=True, + windows_job="passed", + windows_digest=windows_digest, + linux_job="running", + ), + plan, + ) + assert state["phase"] == "LINUX_RUNNING" + assert state["linux_consumed"] is True + + linux_path = tmp_path / "linux.json" + linux_path.write_text( + json.dumps(platform_evidence("linux")), + encoding="utf-8", + ) + linux_digest = "sha256:" + hashlib.sha256(linux_path.read_bytes()).hexdigest() + state = controller.reconcile( + state, + observation( + plan, + linux=True, + windows_job="passed", + windows_digest=windows_digest, + linux_job="passed", + linux_digest=linux_digest, + ), + plan, + ) + assert state["next_action"] == "collect_linux" + state = controller.collect_platform(state, plan, "linux", linux_path) + assert state["phase"] == "LINUX_DELETING" + assert state["next_action"] == "delete_linux" + + state = controller.reconcile( + state, + observation( + plan, + windows_job="passed", + windows_digest=windows_digest, + linux_job="passed", + linux_digest=linux_digest, + ), + plan, + ) + assert state["phase"] == "CLEANED_PASS" + assert state["cleanup_verified"] is True + assert state["next_action"] == "none" + + +def test_failed_job_goes_directly_to_exact_cleanup(plan): + state = controller.initial_state(plan) + state = controller.reconcile( + state, + observation(plan, windows=True, windows_job="running"), + plan, + ) + state = controller.reconcile( + state, + observation(plan, windows=True, windows_job="failed"), + plan, + ) + assert state["phase"] == "CLEANING_FAILED" + assert state["next_action"] == "cleanup_failure" + + state = controller.reconcile(state, observation(plan), plan) + assert state["phase"] == "CLEANED_FAILURE" + assert state["cleanup_verified"] is True + + +def test_foreign_or_overlapping_resource_observations_fail_closed(plan): + state = controller.initial_state(plan) + value = observation(plan, windows=True, windows_job="running") + value["instances"][plan.windows.instance]["source_commit"] = "9" * 40 + with pytest.raises(controller.Gate14ControllerError): + controller.reconcile(state, value, plan) + + value = observation( + plan, + windows=True, + linux=True, + windows_job="running", + linux_job="running", + ) + state = controller.reconcile( + state, + observation(plan, windows=True, windows_job="running"), + plan, + ) + state = { + **state, + "phase": "WINDOWS_DELETING", + "next_action": "delete_windows", + } + with pytest.raises(controller.Gate14ControllerError): + controller.validate_observation( + { + **value, + "l4_usage": 2, + }, + plan, + ) + + +def test_collect_rejects_wrong_package_and_save_round_trips(tmp_path, plan): + state = controller.initial_state(plan) + state = controller.reconcile( + state, + observation(plan, windows=True, windows_job="running"), + plan, + ) + evidence = platform_evidence("windows") + evidence["package"]["archive_sha256"] = "sha256:" + "9" * 64 + evidence_path = tmp_path / "wrong.json" + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + evidence_digest = "sha256:" + hashlib.sha256(evidence_path.read_bytes()).hexdigest() + state = controller.reconcile( + state, + observation( + plan, + windows=True, + windows_job="passed", + windows_digest=evidence_digest, + ), + plan, + ) + with pytest.raises(controller.Gate14ControllerError): + controller.collect_platform(state, plan, "windows", evidence_path) + + state_path = tmp_path / "state.json" + controller.save_state(state_path, state, plan) + assert controller.load_state(state_path, plan) == state + + +def test_begin_cleanup_is_idempotent_after_terminal_state(plan): + state = controller.initial_state(plan) + state = controller.begin_cleanup(state, plan, "manual-stop") + assert state["phase"] == "CLEANING_FAILED" + state = controller.reconcile(state, observation(plan), plan) + assert state["phase"] == "CLEANED_FAILURE" + assert controller.begin_cleanup(state, plan, "manual-stop") == state + + +def test_forged_success_and_deletion_states_fail_closed(plan): + initial = controller.initial_state(plan) + forged_pass = { + **initial, + "phase": "CLEANED_PASS", + "cleanup_verified": True, + "next_action": "none", + } + with pytest.raises(controller.Gate14ControllerError): + controller.validate_state(forged_pass, plan) + + forged_windows_deleting = { + **initial, + "phase": "WINDOWS_DELETING", + "windows_consumed": True, + "next_action": "start_linux", + } + with pytest.raises(controller.Gate14ControllerError): + controller.validate_state(forged_windows_deleting, plan) + + forged_linux_deleting = { + **initial, + "phase": "LINUX_DELETING", + "windows_consumed": True, + "linux_consumed": True, + "windows_evidence_digest": "sha256:" + "c" * 64, + "linux_evidence_digest": "sha256:" + "d" * 64, + "next_action": "delete_linux", + } + with pytest.raises(controller.Gate14ControllerError): + controller.reconcile(forged_linux_deleting, observation(plan), plan) + + +def test_expired_run_never_returns_a_start_action(plan): + value = observation(plan) + value["observed_at_unix"] = plan.windows.termination_unix + + state = controller.reconcile(controller.initial_state(plan), value, plan) + + assert state["phase"] == "CLEANED_FAILURE" + assert state["failure_code"] == "run-expired" + assert state["cleanup_verified"] is True + assert state["next_action"] == "none" + + +def test_passed_job_requires_and_binds_exact_evidence_digest(tmp_path, plan): + missing = observation(plan, windows=True, windows_job="passed") + with pytest.raises(controller.Gate14ControllerError): + controller.validate_observation(missing, plan) + + stale = controller.reconcile( + controller.initial_state(plan), + observation( + plan, + windows_job="passed", + windows_digest="sha256:" + "c" * 64, + ), + plan, + ) + assert stale["phase"] == "CLEANED_FAILURE" + assert stale["failure_code"] == "stale-windows-job" + assert stale["next_action"] == "none" + + state = controller.reconcile( + controller.initial_state(plan), + observation(plan, windows=True, windows_job="running"), + plan, + ) + reported_digest = "sha256:" + "c" * 64 + state = controller.reconcile( + state, + observation( + plan, + windows=True, + windows_job="passed", + windows_digest=reported_digest, + ), + plan, + ) + evidence_path = tmp_path / "different.json" + evidence_path.write_text( + json.dumps(platform_evidence("windows")), + encoding="utf-8", + ) + assert "sha256:" + hashlib.sha256(evidence_path.read_bytes()).hexdigest() != reported_digest + with pytest.raises(controller.Gate14ControllerError): + controller.collect_platform(state, plan, "windows", evidence_path) + + +@pytest.mark.parametrize("platform", ["windows", "linux"]) +def test_initial_state_requires_all_planned_disks_absent_before_start(plan, platform): + value = observation( + plan, + **{f"{platform}_disk": True}, + ) + + state = controller.reconcile(controller.initial_state(plan), value, plan) + + assert state["phase"] == "CLEANING_FAILED" + assert state["failure_code"] == "orphaned-planned-disk" + assert state["next_action"] == "cleanup_failure" + + +def test_stale_passed_job_with_orphan_disk_cannot_claim_terminal_cleanup(plan): + state = controller.reconcile( + controller.initial_state(plan), + observation( + plan, + windows_job="passed", + windows_digest="sha256:" + "c" * 64, + windows_disk=True, + ), + plan, + ) + + assert state["phase"] == "CLEANING_FAILED" + assert state["cleanup_verified"] is False + assert state["next_action"] == "cleanup_failure" + + +def test_linux_start_requires_absent_disk_and_absent_stale_job(plan): + windows_digest = "sha256:" + "c" * 64 + state = { + **controller.initial_state(plan), + "revision": 2, + "phase": "WINDOWS_DELETING", + "windows_evidence_digest": windows_digest, + "windows_consumed": True, + "next_action": "delete_windows", + } + orphan = controller.reconcile( + state, + observation( + plan, + windows_job="passed", + windows_digest=windows_digest, + linux_disk=True, + ), + plan, + ) + assert orphan["phase"] == "CLEANING_FAILED" + assert orphan["failure_code"] == "orphaned-linux-disk" + assert orphan["next_action"] == "cleanup_failure" + + stale_job = controller.reconcile( + state, + observation( + plan, + windows_job="passed", + windows_digest=windows_digest, + linux_job="passed", + linux_digest="sha256:" + "d" * 64, + ), + plan, + ) + assert stale_job["phase"] == "CLEANED_FAILURE" + assert stale_job["failure_code"] == "stale-linux-job" + assert stale_job["cleanup_verified"] is True + assert stale_job["next_action"] == "none" + + +@pytest.mark.parametrize("returned_resource", [{"windows": True}, {"windows_disk": True}]) +def test_linux_deletion_escalates_returned_windows_resources(plan, returned_resource): + windows_digest = "sha256:" + "c" * 64 + linux_digest = "sha256:" + "d" * 64 + state = { + **controller.initial_state(plan), + "revision": 5, + "phase": "LINUX_DELETING", + "windows_evidence_digest": windows_digest, + "linux_evidence_digest": linux_digest, + "windows_consumed": True, + "linux_consumed": True, + "next_action": "delete_linux", + } + + state = controller.reconcile( + state, + observation( + plan, + windows_job="passed", + windows_digest=windows_digest, + linux_job="passed", + linux_digest=linux_digest, + **returned_resource, + ), + plan, + ) + + assert state["phase"] == "CLEANING_FAILED" + assert state["failure_code"] == "windows-resources-returned" + assert state["next_action"] == "cleanup_failure" + + +def test_load_plan_recomputes_total_ledger_commitment(tmp_path): + with pytest.raises(controller.Gate14ControllerError): + write_plan(tmp_path, additional_current_maximum="99.00") + + +def test_load_plan_rejects_hidden_active_reservation_below_epoch_anchor(tmp_path): + with pytest.raises(controller.Gate14ControllerError): + write_plan( + tmp_path, + additional_current_maximum="1.00", + hide_additional_below_anchor=True, + ) diff --git a/tests/test_model_manifest.py b/tests/test_model_manifest.py index ae42ec788..e205224b5 100644 --- a/tests/test_model_manifest.py +++ b/tests/test_model_manifest.py @@ -1,5 +1,6 @@ import hashlib import json +import os from pathlib import Path from types import SimpleNamespace @@ -461,6 +462,34 @@ def replace_after_release(source, destination): assert not partial.exists() +@pytest.mark.skipif(os.name != "nt", reason="Win32 extended-length paths are Windows-specific") +def test_resumable_manifest_paths_work_beyond_legacy_windows_max_path(tmp_path): + from drift.utils.file_lock import file_lock + + manifest = ModelManifest.from_dict(manifest_dict()) + cache = tmp_path / ("cache-" + "x" * 96) + verifier = ManifestArtifactVerifier( + manifest, + manifest.source.repository, + manifest.source.revision, + cache_dir=cache, + ) + + partial, final, lock = verifier._resumable_paths(manifest.get_artifact("weights.bin")) + assert len(str(cache.absolute() / "manifest-artifacts" / manifest.digest / "partial")) > 248 + assert str(partial).startswith("\\\\?\\") + assert str(lock).startswith("\\\\?\\") + + partial.parent.mkdir(parents=True, exist_ok=True) + partial.write_bytes(b"partial") + with file_lock(lock, exclusive=True): + final.parent.mkdir(parents=True, exist_ok=True) + final.write_bytes(b"final") + + assert partial.read_bytes() == b"partial" + assert final.read_bytes() == b"final" + + def test_mixed_cached_and_downloaded_artifacts_share_one_snapshot_root(tmp_path, monkeypatch): from huggingface_hub.utils import LocalEntryNotFoundError