diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce31694..81049b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 @@ -25,3 +25,44 @@ jobs: - name: Run test suite (CPU-only) run: python -m pytest tests/ -q + + coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install package with dev extras + run: pip install -e ".[dev]" + - name: Enforce 100% line and branch coverage + run: | + coverage run -m pytest tests/ -q + coverage report --fail-under=100 + coverage xml + - uses: actions/upload-artifact@v4 + with: + name: coverage-xml + path: coverage.xml + + docs-and-package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install package with dev extras + run: pip install -e ".[dev]" + - name: Build documentation strictly + run: mkdocs build --strict + - name: Build and inspect distributions + run: | + python -m build + python -m twine check --strict dist/* + - name: Smoke-test the wheel + run: | + python -m venv /tmp/wheeltest + /tmp/wheeltest/bin/pip install -q dist/*.whl + /tmp/wheeltest/bin/gpu-proof --help + /tmp/wheeltest/bin/python -m pytest -p pytest_gpu_proof --version diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..0cce259 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,29 @@ +# Agent guide + +This repository is a security-sensitive pytest plugin. Preserve fail-closed +behavior and treat receipt, fingerprint, signature, Git, merge, and policy code +as trust boundaries. + +Before changing behavior, read: + +- `README.md` +- `docs/architecture.md` +- `docs/security_model.md` +- `docs/policy.md` +- `CONTRIBUTING.md` + +Implementation rules: + +- New receipts are schema 3; keep explicit legacy verification isolated. +- Signer metadata belongs inside the signed payload. +- Default fingerprint scope is the full tracked repository; generated inputs + require explicit extra paths. +- Never turn receipt-emission failures into success unless the caller selected + `best_effort`. +- Never accept unknown policy fields or malformed receipt structure. +- Keep tests hermetic: mock GitHub key lookup and local identity. +- Maintain 100% line and branch coverage. Add adversarial cases for new trust + branches. +- Do not edit generated `site/` or release artifacts in `dist/`. + +Run the complete check list from `CONTRIBUTING.md` before proposing a PR. diff --git a/CHANGELOG.md b/CHANGELOG.md index 001943e..1e7ffa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,69 @@ All notable changes to pytest-gpu-proof are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/); versions follow [SemVer](https://semver.org/) (pre-1.0: minor bumps may break). +## [Unreleased] — 0.4.0 + +### Added + +- Schema `"3"`: signer username, key fingerprint, key algorithm, exact test + collection, session outcome, pytest arguments, and per-shard environment/time + are included in the signed payload. +- Open contributor and restricted signer policies. Restricted mode supports + username and exact SSH-key-fingerprint allowlists. +- Policy pinning for mode, global tracked/extra fingerprint scope, exact test + manifest, shard names, and per-shard tracked/extra scope. +- Explicit fingerprint paths for ignored/generated inputs and manifest support + for symlinks and submodule gitlinks. +- Explicit receipt-artifact exclusions avoid self-referential whole-tree + fingerprints and can be pinned by verification policy. +- `--gpu-proof-best-effort` as an explicit development escape hatch. +- `min_schema` policy field to refuse legacy schema-1/2 receipts. +- Python 3.13 CI, strict docs/package jobs, and enforced 100% line and branch + coverage. + +### Changed + +- The default fingerprint scope is the entire Git-tracked repository instead + of `src,tests`. +- Receipt creation now fails pytest on Git, fingerprint, key, serialization, or + write errors; it also removes stale output at session start. +- Schema-3 verification rejects dirty recording and verification trees by + default and accepts receipt commits only at the current commit or an ancestor. +- Setup/teardown failures, missing terminal reports, comparison exceptions, + skipped tests, and overall session failure are represented and verified. +- GPU metadata records all devices reported by `nvidia-smi`. +- Array comparison is shape-safe, uses tolerance only for float/complex data, + and uses exact equality for other dtypes. +- Receipt writes are atomic and strict JSON forbids NaN/non-JSON values. +- Receipt generation rejects xdist workers; use separate shard processes. +- Legacy schema-1/2 verification derives the policy-checked key fingerprint + from the key that actually verified the signature; an asserted + `signature.key_fingerprint` that disagrees is rejected. +- `signer_mode: restricted` policies reject unsigned receipts even with + `--allow-unsigned`, and `--github-user` must match the signed schema-3 + identity. +- The receipt under verification is excluded from the verification-tree dirty + check, so an untracked just-generated receipt verifies without gitignoring. +- Passphrase-protected SSH keys prompt correctly on current cryptography + releases (`ValueError` as well as `TypeError`) and fail closed with an + actionable message when no terminal is available. +- Uninitialized submodule checkouts fingerprint their index gitlink commit + instead of accidentally recording the parent repository's HEAD. +- Receipt/shard age limits are enforced to the exact day boundary. +- GitHub usernames are validated before key fetches and key responses are + size-capped. +- A stale receipt that cannot be cleared at session start raises a pytest + usage error (an exit-status write that early would be silently overwritten). +- Merging receipts without session timestamps is refused with a clear error. + +### Fixed + +- Carry-forward now recomputes the stored fingerprint algorithm and preserves + explicit generated/ignored shard inputs. +- Signed identity and algorithm substitution are rejected. +- Empty source scope, unmerged index entries, malformed policies, incomplete + collections, and stale success artifacts fail closed. + ## [0.3.0] — 2026-08-08 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 381672a..68a68a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,73 +1,14 @@ -# CLAUDE.md — orientation for AI agents (and humans) working on pytest-gpu-proof +# Claude orientation -pytest-gpu-proof is a **pytest plugin that emits a signed JSON receipt of a GPU -test run** (git SHA + source fingerprint + per-test outcomes + GPU info), plus a -**CPU-only verifier** (`gpu-proof verify`) that checks the signature against the -signer's public `github.com/.keys`. The point: run GPU tests on your own -hardware, let CI prove who attested them — no cloud-GPU fees, no secrets in CI. +Use [AGENTS.md](AGENTS.md) as the canonical agent guide. In particular: -**Trust model — never oversell it:** a receipt is a **signed attestation by a -keyholder, NOT cryptographic proof of GPU execution**. `gpu_info` is -self-reported; `--require-gpu` is modest hardening against accidents, not -against a dishonest signer. `docs/security_model.md` is the honest statement of -what is and isn't proven — keep every README/docs claim consistent with it. +- a receipt is signer attestation, not proof of GPU execution; +- receipt, fingerprint, signature, Git, merge, and policy code fail closed; +- new recordings use schema 3 and bind signer metadata inside the signature; +- the default manifest covers the full tracked repository; +- tests are hermetic and maintain 100% line and branch coverage; +- generated `site/`, `dist/`, coverage output, real keys, and local receipts are + not source changes. -## Source layout (`src/pytest_gpu_proof/`) - -| Module | Role | -|---|---| -| `plugin.py` | pytest hooks: options, `gpu_proof` marker, `gpu_proof_check` fixture, outcome+skip capture, receipt emission at session end | -| `receipt.py` | payload build (repo/fingerprint/tests/env), canonical JSON, sign+write. Signer resolution: flag/config → `gh` CLI login (keyholder) → origin-remote owner (warned — orgs have no SSH keys) | -| `verify.py` | the 7 verification checks (signature, fingerprint, commit SHA, outcomes+skip policy, gpu_info, freshness, dirty policy). Expected-skips baseline = EXACT set match | -| `cli.py` | `gpu-proof verify` argument surface | -| `config.py` | `GpuProofConfig`; CLI flags override `[tool.gpu_proof]` in pyproject.toml | -| `fingerprint.py` | SHA-256 digest over configured paths | -| `gitutils.py` | git/gh shell-outs, all failure-tolerant (return `None`) | -| `signers/` | `base.py` protocol + `ed25519.py` SSH-key signing / GitHub-keys verification; backend `none` emits unsigned receipts | -| `compare.py` | receipt diffing | - -Signing covers the canonical (compact, sorted-key) JSON **without** the -`signature` field; the sig block carries `signer`, key fingerprint, algorithm -(derived from actual key type — don't hardcode ed25519). - -## Behavioral invariants (test-enforced — don't regress) - -- **Skips prove nothing.** Verifier rejects receipts with skips unless - `--allow-skipped` (any skips) or `--expected-skips` (EXACT baseline: a new - skip fails, a stale baseline entry fails). The two are mutually exclusive. -- **Unsigned receipts** verify only with `--allow-unsigned`, loudly. -- **`--max-age-days 0`** means "today only", not "disabled". -- The signer recorded at signing time must be the **keyholder**, never - silently the repo owner. - -## Dev workflow - -```bash -.venv/bin/python -m pytest tests/ -q # full suite, ~2s, no GPU needed -.venv/bin/mkdocs build --strict # docs must stay warning-clean -``` - -- Tests are **hermetic**: `tests/conftest.py` has an autouse fixture nulling - `get_gh_cli_login` (a dev box with authenticated `gh` would otherwise hit the - network). Signer tests re-patch explicitly. Keep new shell-outs mockable and - wrapped in try/except like `gitutils._git`. -- `tests/conftest.py` uses `pytest_plugins = ["pytester"]`; keypairs are - generated in-memory (no real SSH keys touched). -- requires-python ≥ 3.11 (`datetime.UTC`). -- CI (`.github/workflows/`): tests on 3.11/3.12 + mkdocs gh-pages deploy. - -## Conventions & state - -- Short single-line commit messages; no Co-Authored-By footer. -- Flow: feature branch → PR → CI green → merge to `main`. Consumers install - from git (`pip install -e` on a submodule) or PyPI. **Releases**: OIDC - trusted publishing via `.github/workflows/publish.yml` (TestPyPI on manual - dispatch, PyPI on GitHub release) — process in `RELEASING.md`; keep - `CHANGELOG.md` current and bump `pyproject.toml` version in the same PR. -- Reference integration: **GLASS** (github.com/A2R-Lab/GLASS) — - `test/run_gpu_proof.sh`, `test/expected_skips.txt`, - `.github/workflows/verify-gpu-proof.yml`. If you change plugin/verifier - flags, check GLASS's usage still works and note it in the PR. -- Consumer-side gotcha worth remembering: a repo that submodules this project - under its pytest rootdir must `collect_ignore = ["pytest-gpu-proof"]` in its - conftest, or this repo's `tests/conftest.py` will shadow theirs. +Before implementation work, read `docs/architecture.md`, +`docs/security_model.md`, `docs/policy.md`, and `CONTRIBUTING.md`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d04dd43 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,41 @@ +# Contributing + +## Development setup + +```bash +python -m venv .venv +source .venv/bin/activate +python -m pip install -e ".[dev]" +``` + +## Required checks + +```bash +python -m pytest tests -q +coverage run -m pytest tests -q +coverage report --fail-under=100 +mkdocs build --strict +python -m build +python -m twine check --strict dist/* +``` + +CI runs the test suite on Python 3.11, 3.12, and 3.13, independently enforces +100% line and branch coverage, builds docs strictly, and smoke-tests the wheel. + +## Change expectations + +- Add adversarial tests for trust-boundary changes, not only happy paths. +- Keep receipt generation and verification fail-closed by default. +- Treat schema and policy changes as compatibility decisions; document them in + `CHANGELOG.md` and the relevant guide. +- Keep Git and network tests hermetic. Never depend on a developer's actual + GitHub login, SSH keys, GPU, or global Git configuration. +- Use separate pytest processes for shards. Receipt generation does not support + xdist workers. +- Do not commit generated `site/`, coverage files, caches, or local receipts. + +## Pull requests + +Describe the trust claim before and after the change, migration implications, +and the exact checks run. Small, reviewable commits are preferred. Security +reports should follow [SECURITY.md](SECURITY.md), not a public issue. diff --git a/README.md b/README.md index 8ba2c9c..3e6b315 100644 --- a/README.md +++ b/README.md @@ -1,367 +1,293 @@ # pytest-gpu-proof -A pytest plugin that lets you run GPU equivalence tests locally, sign the results with your existing SSH key, and have GitHub Actions verify the receipt — **without re-running the GPU tests in CI**. - -The trust model is simple: if you can push to GitHub, you can sign a receipt. Verification fetches your public keys from `github.com/{username}.keys`, exactly as SSH does. - -> **Requirements:** Python 3.11+ · pytest 7.0+ · cryptography 41.0+ -> The package is not yet on PyPI — install from source with `pip install -e .` (see [Installation](#installation)). - ---- - -## Why this exists - -GPU CI is expensive. For many teams, the typical workflow is: - -1. Run GPU correctness tests locally (or on a lab machine). -2. Push code and let CI run only CPU tests. -3. Hope the GPU tests still pass. - -This plugin closes that gap by producing a **cryptographically signed receipt** that attests that a specific signer ran specific tests against specific code at a specific time — verifiable in ordinary CPU-only CI with no GPU and no secrets. - ---- - -## How it works - +[![CI](https://github.com/A2R-Lab/pytest-gpu-proof/actions/workflows/ci.yml/badge.svg)](https://github.com/A2R-Lab/pytest-gpu-proof/actions/workflows/ci.yml) +[![Docs](https://github.com/A2R-Lab/pytest-gpu-proof/actions/workflows/docs.yml/badge.svg)](https://a2r-lab.github.io/pytest-gpu-proof/) +[![PyPI](https://img.shields.io/pypi/v/pytest-gpu-proof.svg)](https://pypi.org/project/pytest-gpu-proof/) +[![Python](https://img.shields.io/pypi/pyversions/pytest-gpu-proof.svg)](https://pypi.org/project/pytest-gpu-proof/) +[![Coverage: 100%](https://img.shields.io/badge/coverage-100%25-brightgreen.svg)](https://github.com/A2R-Lab/pytest-gpu-proof/actions/workflows/ci.yml) + +Signed pytest receipts for local GPU runs, verified in CPU-only CI. + +`pytest-gpu-proof` records which marked tests ran, their outcomes, the Git +commit and source fingerprint, the environment, and the run time. It signs +that payload with an existing SSH key. A normal CPU runner can then verify the +receipt against the signer's public keys on GitHub without rerunning CUDA. + +This is a practical bridge for projects with local or lab GPUs but no +always-on GPU CI. It is signer attestation—not hardware attestation. See the +[security model](docs/security_model.md) before making stronger claims. + +## The workflow + +```text +GPU machine CPU-only CI +────────────────────────────────── ─────────────────────────────── +pytest --gpu-proof-enable gpu-proof verify --receipt ... + run marked tests validate schema and signature + capture setup/call/teardown outcomes fetch current GitHub SSH keys + fingerprint the checked-out tree recompute source fingerprint + sign one schema-3 receipt enforce repository policy ``` -Local machine (GPU) GitHub Actions (CPU only) -───────────────────────────── ────────────────────────────────────── -pytest --gpu-proof-enable → → gpu-proof verify --receipt gpu-proof.json - runs your GPU tests fetches your public keys from - computes code fingerprint github.com/{you}.keys - signs receipt with SSH key verifies signature + fingerprint - writes gpu-proof.json exits 0 (pass) or 1 (fail) -``` - -**Zero new key management.** The plugin uses the SSH key you already have in `~/.ssh/` (the same one you use to push to GitHub). Your public key is already on GitHub. The verifier reads it from there. - ---- -## Installation +The default signer policy is **open**: a valid receipt from any GitHub user is +accepted. This supports contributor-signed pull requests. Repositories that +only trust maintainers or dedicated CI keys can use **restricted** mode with +username and/or key-fingerprint allowlists. -From PyPI (v0.1.0+): +## Install ```bash -pip install pytest-gpu-proof +python -m pip install pytest-gpu-proof ``` -Or from a clone / git submodule (how the A2R-Lab consumers pin exact versions): +For development: ```bash -git clone https://github.com/A2R-Lab/pytest-gpu-proof +git clone https://github.com/A2R-Lab/pytest-gpu-proof.git cd pytest-gpu-proof -pip install -e . # base install (pytest + cryptography) -pip install -e ".[dev]" # also installs numpy, pytest-cov +python -m pip install -e ".[dev]" ``` -Or use the helper script (checks your Python version first): - -```bash -bash install.sh # base install -bash install.sh dev # dev install -``` - -Maintainers: release process in [RELEASING.md](RELEASING.md). - ---- +Requirements: Python 3.11+, pytest 7+, and `cryptography` 41+. ## Quick start -### Step 1 — Install from source - -```bash -git clone -cd pytest-gpu-proof -pip install -e . -``` - -### Step 2 — Run the bundled demo (no GPU needed) - -The repo includes a no-GPU demo that works on any machine: - -```bash -cd examples/minimal_python_only -pytest test_minimal.py --gpu-proof-enable -v -``` - -Expected output: -``` -PASSED test_minimal.py::test_relu -PASSED test_minimal.py::test_softmax -... -[gpu-proof] Receipt written to gpu-proof.json -[gpu-proof] Signed with key SHA256:... -``` - -### Step 3 — Verify the receipt - -```bash -gpu-proof verify --receipt gpu-proof.json -``` - -The verifier fetches your public keys from `github.com/{you}.keys` automatically — no secrets needed. - -### Step 4 — Use it in your own project - -Add the marker and fixture to your tests: +Mark a test, or use the comparison fixture: ```python import pytest + @pytest.mark.gpu_proof -def test_my_kernel(gpu_proof_check): +def test_rnea(gpu_proof_check): gpu_proof_check( - name="relu", - reference=python_relu, # your reference implementation - candidate=cuda_relu, # your GPU wrapper - args=([1.0, -2.0, 3.0],), - metadata={"kernel": "relu"}, + name="rnea", + reference=python_rnea, + candidate=cuda_rnea, + args=(model, q, qd, qdd), + metadata={"robot": "go2"}, ) ``` -Run from **your project's root** (not the pytest-gpu-proof source directory): +Run from the root of the project being attested: ```bash -pytest path/to/your/tests/ --gpu-proof-enable -v -# → writes gpu-proof.json in the current directory +pytest tests/gpu --gpu-proof-enable --gpu-proof-github-user YOUR_USER +gpu-proof verify --receipt gpu-proof.json --repo . ``` -> **Note:** The `--gpu-proof-enable` flag only writes a receipt if at least one test is marked with `@pytest.mark.gpu_proof` or uses the `gpu_proof_check` fixture. Running it against the plugin's own `tests/` directory (which tests the plugin internals) will not produce a receipt. - -### Step 5 — Commit and verify in CI - -```bash -git add gpu-proof.json -git commit -m "update GPU proof receipt" -git push -``` +Commit `gpu-proof.json` with the code, then add a CPU-only CI step: ```yaml -# .github/workflows/ci.yml -- name: Verify GPU proof - run: gpu-proof verify --receipt gpu-proof.json +- name: Verify local GPU test receipt + run: gpu-proof verify --receipt gpu-proof.json --repo . ``` ---- - -## The `gpu_proof_check` fixture - -```python -gpu_proof_check( - name="my_op", # unique name within the test - reference=python_fn, # callable: the ground truth - candidate=cuda_fn, # callable: the GPU implementation - args=(arg1, arg2), # positional arguments (tuple) - kwargs={"key": "val"}, # keyword arguments (dict, optional) - compare=my_compare_fn, # optional: (ref_out, cand_out) -> None, raises on mismatch - metadata={"info": "..."}, # optional: included verbatim in the receipt -) -``` +Receipt creation is fail-closed. A missing key, invalid Git state, empty +fingerprint scope, no selected tests, or write failure makes pytest fail and +leaves no stale receipt behind. `--gpu-proof-best-effort` is an explicit +development-only opt-out. -**Default comparison:** `numpy.allclose` for float arrays/tensors, `==` otherwise. +## Four interfaces -**Custom comparison:** any callable that raises `AssertionError` on mismatch and returns `None` on success. +The project exposes four small interfaces: ---- +1. **Markers** select receipt tests. +2. **`gpu_proof_check`** compares a reference callable with a candidate. +3. **`gpu-proof verify`** validates a receipt and repository policy. +4. **`gpu-proof merge`** combines separately executed shards and optionally + carries unchanged shards forward. -## Markers +### Markers -| Marker | Effect | +| Marker | Meaning | |---|---| -| `@pytest.mark.gpu_proof` | Include test outcomes in the receipt | +| `@pytest.mark.gpu_proof` | Include the test in the receipt | | `@pytest.mark.gpu_equivalence` | Alias for `gpu_proof` | -| `@pytest.mark.gpu_required` | Skip test if no GPU is detected (via `nvidia-smi` or `torch.cuda`) | +| `@pytest.mark.gpu_required` | Skip when neither `nvidia-smi` nor PyTorch reports a GPU | ---- +Skipped tests are recorded. Verification rejects them by default. Prefer an +exact `--expected-skips` baseline over the broad `--allow-skipped` escape +hatch. -## CLI options +### Comparison fixture -| Option | Default | Description | -|---|---|---| +```python +gpu_proof_check( + name="operation", + reference=reference_fn, + candidate=gpu_fn, + args=(arg1, arg2), + kwargs={"option": value}, + compare=custom_compare, + metadata={"case": "small"}, +) +``` + +The default comparator is shape-safe: float/complex NumPy-compatible arrays +use `numpy.allclose(..., equal_nan=True)` and other arrays use exact equality. +Provide a comparator for GPU tensors, domain-specific tolerances, or structured +outputs. A comparator should return normally on success and raise +`AssertionError` on mismatch. + +## Recording options + +| Option | Default | Purpose | +|---|---:|---| | `--gpu-proof-enable` | off | Enable receipt generation | -| `--gpu-proof-mode` | `local` | `local` or `ci-gpu` | -| `--gpu-proof-out` | `gpu-proof.json` | Receipt output path | -| `--gpu-proof-key` | auto | SSH private key path | -| `--gpu-proof-signing-backend` | `ed25519` | `ed25519` or `none` (writes an **unsigned** receipt with `"signature": null`; the verifier rejects it unless `--allow-unsigned` is passed) | -| `--gpu-proof-required-marker` | `gpu_proof` | Marker name that flags a test for the receipt | -| `--gpu-proof-fingerprint-paths` | `src,tests` | Comma-separated paths to fingerprint | -| `--gpu-proof-github-user` | auto | GitHub username (auto-detected from git remote) | -| `--gpu-proof-policy` | — | Path to policy YAML | -| `--gpu-proof-fail-on-skip` | off | Exit non-zero and write no receipt if any marked or `gpu_required` test is skipped | - -Defaults for most of these can also be set in your project's `pyproject.toml` -under `[tool.gpu_proof]` (CLI flags take precedence): +| `--gpu-proof-mode` | `local` | Record `local` or `ci-gpu` provenance | +| `--gpu-proof-out` | `gpu-proof.json` | Output artifact | +| `--gpu-proof-key` | discovered | SSH private-key file | +| `--gpu-proof-github-user` | discovered | GitHub account that owns the public key | +| `--gpu-proof-signing-backend` | `ed25519` | SSH signing, or explicit `none` | +| `--gpu-proof-required-marker` | `gpu_proof` | Custom receipt marker | +| `--gpu-proof-fail-on-skip` | off | Fail and suppress the receipt on selected skips | +| `--gpu-proof-fingerprint-paths` | `.` | Comma-separated Git-tracked scope | +| `--gpu-proof-fingerprint-extra-paths` | empty | Explicit ignored/generated inputs | +| `--gpu-proof-fingerprint-excluded-paths` | `gpu-proof.json` | Receipt artifacts omitted to avoid self-reference | +| `--gpu-proof-best-effort` | off | Warn instead of failing if emission fails | + +Most defaults can live in the consumer's `pyproject.toml`: ```toml [tool.gpu_proof] mode = "local" output = "gpu-proof.json" -fingerprint_paths = ["src", "tests"] +fingerprint_paths = ["."] +fingerprint_extra_paths = ["generated/kernel_table.cuh"] +fingerprint_excluded_paths = ["gpu-proof.json"] required_marker = "gpu_proof" -max_age_days = 30 # used by the verifier -require_gpu = false # used by the verifier (see below) +max_age_days = 30 +require_gpu = true ``` ---- +By default, the fingerprint covers every tracked file in the repository, +including symlink targets and submodule gitlinks. Ignored/generated artifacts +are excluded unless explicitly named as extra paths. An empty or unreadable +scope is an error. The receipt artifact itself is excluded because a signed +file cannot hash its own final contents; set the exclusion explicitly if your +committed receipt uses a different path. -## Verification CLI +## Verification and policy ```bash gpu-proof verify \ --receipt gpu-proof.json \ --repo . \ - --max-age-days 30 - -# With an explicit GitHub username (non-GitHub remotes): -gpu-proof verify --receipt gpu-proof.json --github-user myusername + --policy gpu-proof-policy.yaml ``` -Also callable as: +Schema-3 verification checks: -```bash -python -m pytest_gpu_proof verify --receipt gpu-proof.json -``` - -### What the verifier checks - -1. **Signature** — fetches `github.com/{signer}.keys`, verifies Ed25519/ECDSA/RSA signature -2. **Fingerprint** — recomputes SHA-256 digest of `src/` and `tests/`, compares to receipt -3. **Commit SHA** — compares receipt commit SHA to current HEAD -4. **Test outcomes** — all tests recorded in the receipt must have passed; skipped marked tests fail verification unless `--allow-skipped` is passed, or an `--expected-skips` baseline is given and the receipt's skip set matches it exactly -5. **Freshness** — receipt must be younger than `max_age_days` (default: 30) -6. **Dirty policy** — configurable via policy file -7. **GPU info** (optional) — with `--require-gpu` (or `require_gpu = true` in `[tool.gpu_proof]`), the receipt's `environment.gpu_info` must be present - -Additional flags: - -- `--allow-unsigned` — accept receipts with `"signature": null` (produced by `--gpu-proof-signing-backend=none`). This disables the entire trust story; the verifier prints a loud warning. -- `--allow-skipped` — accept receipts that contain skipped marked tests (any skips, no questions asked). -- `--expected-skips PATH` — the strict alternative to `--allow-skipped` (the two are mutually exclusive): a baseline file of node IDs (one per line, `#` comments) that the receipt's skipped tests must match **exactly**. A skip not in the baseline fails verification, and a baseline entry that no longer skips fails too (stale baseline — update it). Use this when a project has a small set of permanent, documented skips. Can also be set as an inline list via `expected_skips = [...]` in `[tool.gpu_proof]`. -- `--require-gpu` — reject receipts whose `environment.gpu_info` is null/absent. This is **modest hardening, not proof**: `gpu_info` is self-reported by the recording machine, so it only guards against accidentally signing on a GPU-less box, not against a dishonest signer. +- strict receipt structure, complete test collection, and session outcome; +- signature, signed username, key fingerprint, and key algorithm; +- the signer's current public SSH keys at `github.com/.keys`; +- tracked and explicit-extra source fingerprints; +- current/ancestor Git commit and clean recording/verification trees; +- test and comparison outcomes, exact skip policy, and optional test manifest; +- optional shard membership, shard fingerprints, and carry-forward policy; +- mode, GPU-information requirement, and freshness. ---- +Open policy, suitable for contributor-signed PRs: -## Key management - -**Local mode:** Uses your existing `~/.ssh/id_ed25519` (or whatever `git config user.signingKey` points to). No new keys to generate. +```yaml +signer_mode: open +max_age_days: 30 +require_mode: local +required_fingerprint_paths: ["."] +required_fingerprint_excluded_paths: [gpu-proof.json] +required_test_manifest: gpu-proof-tests.txt +``` -**CI-GPU mode:** Generate a dedicated CI signing key, store the private key as a GitHub Actions secret, and add the public key to your GitHub account. +Restricted policy, suitable for a maintainer or CI allowlist: -```bash -ssh-keygen -t ed25519 -f ci-signing-key -N "" -# Add ci-signing-key.pub to github.com/settings/keys -# Add contents of ci-signing-key to GitHub Actions secrets as GPU_PROOF_SIGNING_KEY +```yaml +signer_mode: restricted +allowed_signers: [alice, release-bot] +allowed_key_fingerprints: + - "SHA256:..." +max_age_days: 14 +require_mode: ci-gpu +required_fingerprint_paths: ["."] +required_fingerprint_extra_paths: [generated/kernel_table.cuh] +required_fingerprint_excluded_paths: [gpu-proof.json] +allow_dirty: false +allow_carried: false ``` ---- - -## Receipt format - -```json -{ - "schema_version": "1", - "mode": "local", - "repo": { - "remote_url": "git@github.com:you/myrepo.git", - "github_username": "you", - "commit_sha": "abc123...", - "branch": "main", - "dirty": false - }, - "fingerprint": { - "algorithm": "sha256", - "included_paths": ["src", "tests"], - "file_count": 12, - "digest": "deadbeef..." - }, - "session": { - "started_at": "2024-01-01T10:00:00Z", - "ended_at": "2024-01-01T10:01:30Z", - "node_ids": ["tests/test_relu.py::test_relu"] - }, - "tests": [ - { - "node_id": "tests/test_relu.py::test_relu", - "outcome": "passed", - "duration_s": 1.23, - "checks": [{"name": "relu", "outcome": "passed", "metadata": {}}] - } - ], - "environment": { - "python_version": "3.11.0", - "platform": "linux", - "pytest_version": "7.4.0", - "gpu_info": {"name": "NVIDIA RTX 3090", "driver_version": "535.104"} - }, - "signature": { - "algorithm": "ed25519", - "backend": "ssh-local", - "signer": "you", - "key_fingerprint": "SHA256:...", - "value": "" - } -} -``` +If both restricted allowlists are present, both must match. Unknown policy +fields fail verification so misspellings cannot silently weaken policy. +YAML support is available through `pytest-gpu-proof[yaml]`; JSON policy files +need no optional dependency. ---- +`--allow-unsigned` accepts `"signature": null` with a loud warning. It removes +signer authentication and should not be used as a merge gate. -## Security model +## Signing identity -A signed receipt proves that **an accepted signer attested to a specific test run over a specific code state**. It does not prove: +The GitHub username is resolved in this order: -- The local machine was fully trustworthy or uncompromised. -- The GPU execution environment was hardware-attested. -- The signing key was protected with a hardware security module. +1. `--gpu-proof-github-user` or `github_username` configuration; +2. the authenticated `gh` CLI user; +3. the origin owner, with a warning because organization owners normally do + not own an individual's SSH key. -This is appropriate for **team workflows where the signer is a trusted team member** and the goal is to avoid paying for GPU CI on every merge, not to provide adversarial security guarantees. +The private-key file is resolved in this order: -The optional `--require-gpu` verifier flag adds a modest extra check — the receipt must contain self-reported `environment.gpu_info` — but this is hardening against mistakes (signing on a GPU-less machine), not proof of GPU execution. +1. `--gpu-proof-key`; +2. `git config user.signingKey`; +3. `~/.ssh/id_ed25519`, `id_ecdsa`, then `id_rsa`. -**SSH key support caveats:** +Ed25519, ECDSA, and RSA-PSS keys are supported. Agent-only and +hardware-backed keys are not yet supported because signing currently requires +a readable private-key file. -- The plugin signs the raw receipt bytes with the key loaded from disk — it does **not** produce SSHSIG-format signatures, so `ssh-keygen -Y verify` cannot validate receipts. Use `gpu-proof verify` instead. -- Keys that live only in an SSH agent, and FIDO/hardware-backed `sk-ssh-ed25519`/`sk-ecdsa` keys, are **not** supported: signing needs direct access to a private key file readable by the `cryptography` library. +## Shards -See [docs/security_model.md](docs/security_model.md) for a full discussion. +Run shards as separate pytest processes; xdist workers are intentionally +rejected for receipt generation. ---- +```bash +pytest tests/gpu/a --gpu-proof-enable \ + --gpu-proof-shard=a \ + --gpu-proof-shard-fingerprint-paths=src/a,tests/gpu/a \ + --gpu-proof-out=receipts/a.json -## Examples +pytest tests/gpu/b --gpu-proof-enable \ + --gpu-proof-shard=b \ + --gpu-proof-shard-fingerprint-paths=src/b,tests/gpu/b \ + --gpu-proof-out=receipts/b.json -| Example | Location | What it shows | -|---|---|---| -| Minimal (no GPU needed) | `examples/minimal_python_only/` | Full plugin flow with pure-Python "fake GPU" | -| Wrapped CUDA via ctypes (fake library) | `examples/wrapped_cuda_ctypes/` | ctypes-style wrapper pattern that runs without CUDA | -| Real CUDA via ctypes | `examples/cuda_ctypes_matmul/` | C ABI CUDA shared library loaded with Python `ctypes` | -| Real CUDA via pybind11 | `examples/cuda_pybind11_matmul/` | CUDA-backed Python extension using `pybind11_add_module` | -| Real CUDA via nanobind | `examples/cuda_nanobind_matmul/` | CUDA-backed Python extension using `nanobind_add_module` | -| Real CUDA via JAX FFI | `examples/jax_ffi_cuda_matmul/` | Typed CUDA custom call registered with `jax.ffi` | -| Local sign, CI verify | `examples/local_receipt_verify/` | GitHub Actions workflow for CPU-only verification | -| CI-GPU execution | `examples/github_gpu_runner/` | GitHub Actions workflow on a GPU runner | +gpu-proof merge receipts/a.json receipts/b.json --out gpu-proof.json +``` ---- +The merger refuses mixed commits, schemas, global fingerprints, modes, +runtime environments, duplicate tests, and duplicate shard names. It records +input provenance but does not verify each input signature; the merger signs +for the union. See [sharding and carry-forward](docs/sharding.md). -## Development +## Receipt schema -```bash -python3 -m pip install -e ".[dev]" -pytest tests/ -v -``` +New receipts use schema `"3"`. Signer identity is inside the signed payload, +test node IDs exactly match the recorded collection, setup and teardown +failures are terminal outcomes, and writes are atomic. The verifier retains +schema-1/2 compatibility, but their legacy defaults are less strict. -If you do not want to install the package, run tests directly from the source -tree with: +## Development ```bash -PYTHONPATH=src pytest -q +python -m pytest tests -q +coverage run -m pytest tests -q +coverage report --fail-under=100 +mkdocs build --strict +python -m build +python -m twine check --strict dist/* ``` -Tests are CPU-only. No GPU or network access required. - ---- +CI tests Python 3.11–3.13 and enforces 100% line and branch coverage. See +[CONTRIBUTING.md](CONTRIBUTING.md), [SECURITY.md](SECURITY.md), and +[RELEASING.md](RELEASING.md). -## Compatibility +## License -- Python 3.11+ -- pytest 7.0+ -- `cryptography` 41.0+ -- `pytest-xdist` is **not** supported in v1 (parallel workers would write conflicting receipts) +MIT. diff --git a/ROADMAP.md b/ROADMAP.md index 303876f..ac570d5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,11 +1,15 @@ # Roadmap -- **PyPI publishing: infrastructure DONE 2026-07-07** — `publish.yml` (OIDC - trusted publishing: TestPyPI on manual dispatch, PyPI on GitHub release), - CHANGELOG.md, RELEASING.md, full metadata, wheel smoke-tested; name free on - PyPI. REMAINING (repo-owner web UI, ~5 min, steps in RELEASING.md): add the - pending trusted publishers on pypi.org + test.pypi.org and create the - `pypi`/`testpypi` GitHub environments — then dispatch the TestPyPI lane and - cut v0.1.0. -- SSHSIG-compatible signing (`ssh-keygen -Y verify` interop; agent-only + FIDO keys). -- CI-issued nonce / challenge mode for stronger replay protection. +The current focus is a small, auditable receipt format and strict local-to-CI +workflow. Candidate future work: + +- SSHSIG-compatible signing, SSH-agent signing, and hardware-backed keys; +- optional verification of every input shard signature before merge; +- CI-issued nonce/challenge mode for stronger replay resistance; +- Sigstore/OIDC provenance for controlled CI-GPU runs; +- archived signer-key evidence or transparency integration; +- versioned JSON Schema publication and external conformance fixtures; +- an explicit multi-machine merge model for heterogeneous GPU metadata. + +Hardware execution attestation is intentionally out of scope unless a concrete +backend and verifier can support claims stronger than self-reported GPU data. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..da957c3 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,33 @@ +# Security policy + +## Reporting + +Please report vulnerabilities through GitHub's private security advisory flow +for `A2R-Lab/pytest-gpu-proof`. Do not open a public issue for a bypass that +could cause an invalid receipt to verify. + +Include: + +- affected version or commit; +- receipt/policy sample with secrets removed; +- expected and observed verifier behavior; +- reproduction steps and impact. + +## In scope + +- signature or signed-identity bypasses; +- source fingerprint omissions or path escapes; +- incomplete test/session outcomes accepted as passing; +- policy fields that fail open; +- stale/carry-forward/ancestry bypasses; +- unsafe private-key handling or receipt writes. + +The documented limits in the [security model](docs/security_model.md)—including +the absence of hardware attestation and trust in the local signer—are not +vulnerabilities by themselves. + +## Supported versions + +Until 1.0, security fixes are made on the latest released minor version. Older +schema receipts may remain verifiable for migration, but new recordings use +the current schema and stricter defaults. diff --git a/docs/architecture.md b/docs/architecture.md index 89c30e8..a0da45c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,92 +1,99 @@ # Architecture -## Package layout - -``` -src/pytest_gpu_proof/ - __init__.py version - plugin.py pytest hooks, CLI option registration, gpu_proof_check fixture - config.py GpuProofConfig dataclass, load_config() - gitutils.py git helpers: commit SHA, branch, dirty state, remote URL, GitHub username - fingerprint.py deterministic SHA-256 fingerprint of source files - compare.py run_comparison(), default_compare() (numpy.allclose / ==) - receipt.py build_receipt_payload(), finalize_receipt(), write_receipt() - verify.py _verify() with all checks, verify_receipt() public API - cli.py argparse entry point for `gpu-proof verify` - __main__.py enables `python -m pytest_gpu_proof verify` - signers/ - base.py SignerBase ABC, VerifierError - ed25519.py SSHSigner, fetch_github_public_keys(), verify_with_github_keys() -``` - -## Data flow - -### Local mode (signing) - +## Components + +```text +pytest hooks + fixture + │ + ▼ +receipt payload ── source manifest ── Git metadata ── environment + │ + ▼ +canonical JSON + SSH signature + │ + ▼ +schema-3 receipt + │ + ├── verify: schema → signature → tree → outcomes → policy + └── merge: compatible shards → union → merger signature ``` -pytest session start - │ - ├── plugin.pytest_sessionstart() record started_at - │ - ├── [tests run] - │ gpu_proof_check fixture run_comparison(reference, candidate) - │ pytest_runtest_makereport hook collect outcome + checks per test - │ - └── plugin.pytest_sessionfinish() - build_receipt_payload() git state + fingerprint + test results + env - SSHSigner.sign(canonical_json) Ed25519 via ~/.ssh/id_ed25519 - finalize_receipt() embed signature block - write_receipt() → gpu-proof.json -``` - -### Verification - -``` -gpu-proof verify --receipt gpu-proof.json - │ - ├── load receipt JSON - ├── extract signature block, compute canonical payload - ├── fetch github.com/{signer}.keys - ├── verify signature against each key - ├── recompute fingerprint, compare digest - ├── compare commit SHA to current HEAD - ├── check all test outcomes == "passed" - ├── check receipt age ≤ max_age_days - └── exit 0 (pass) or 1 (fail) -``` - -## Receipt signing -The signature covers the **canonical JSON** of the receipt without the `signature` field: +| Module | Responsibility | +|---|---| +| `plugin.py` | pytest options, collection, phase outcomes, fixture checks | +| `config.py` | CLI/TOML precedence and runtime configuration | +| `gitutils.py` | strict Git queries and tracked-index entries | +| `fingerprint.py` | deterministic tracked/extra manifest | +| `compare.py` | reference/candidate execution and comparison | +| `receipt.py` | payload construction, canonicalization, atomic output | +| `signers/ed25519.py` | Ed25519/ECDSA/RSA signing and GitHub key lookup | +| `verify.py` | schema, signature, repository, test, shard, and policy checks | +| `merge.py` | compatible shard union and controlled carry-forward | + +## Recording sequence + +1. Session start records time and removes the old output. +2. Collection records the exact selected node-ID sequence. +3. `pytest_runtest_makereport` observes setup, call, and teardown. +4. `gpu_proof_check` stores named comparison outcomes on the test item. +5. Session finish fills any missing terminal reports as errors. +6. Receipt construction requires a Git repository and nonempty source scope. +7. Signer metadata is inserted into the payload before canonicalization. +8. The JSON file is flushed, fsynced, and atomically replaced. + +Receipt-generation failure changes pytest's exit status unless explicit +best-effort mode was requested. + +## Schema 3 + +The top-level blocks are: + +- `repo`: remote, signed GitHub identity, commit, branch, and dirty state; +- `fingerprint`: manifest algorithm, tracked paths, extra paths, count, digest; +- `session`: start/end, pass/fail, exact node IDs, pytest arguments; +- `tests`: terminal outcome, phase, duration, and comparison checks; +- `environment`: Python, platform, pytest, plugin, and self-reported GPU data; +- `shards`: optional narrow manifests and carry metadata; +- `signer`: signed username, algorithm, backend, and key fingerprint; +- `signature`: base64 signature value only. + +The signature covers canonical JSON for every field except `signature`: ```python -payload = {k: v for k, v in receipt.items() if k != "signature"} -canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() -signature = private_key.sign(canonical) +payload = {key: value for key, value in receipt.items() if key != "signature"} +canonical = json.dumps( + payload, sort_keys=True, separators=(",", ":"), allow_nan=False +).encode() ``` -This means the human-readable indented `gpu-proof.json` on disk is verifiable: -the verifier simply strips the `signature` key and re-canonicalizes before verifying. +Putting signer metadata inside this payload prevents identity or algorithm +substitution after signing. -## Key discovery order (signing) +## Fingerprint manifest -1. `--gpu-proof-key=PATH` CLI option -2. `git config user.signingKey` (expanded with `~`) -3. `~/.ssh/id_ed25519` -4. `~/.ssh/id_ecdsa` -5. `~/.ssh/id_rsa` +`sha256-manifest-v2` enumerates stage-0 Git index entries. Regular files bind +bytes and mode, symbolic links bind their target string, and submodules bind +the checked-out commit or index gitlink. Explicit extra paths add ignored or +generated files without sweeping unrelated build output into the claim. The +committed receipt path is an explicit exclusion because its final signed bytes +cannot recursively be an input to its own digest. -If none found, signing is skipped with a warning (tests still pass; just no receipt). +The digest is over canonical JSON for the file map, not a concatenation with +ambiguous boundaries. -## GitHub username discovery order +## Verification order -1. `--gpu-proof-github-user=USERNAME` CLI option -2. `config.github_username` (from pyproject.toml `[tool.gpu_proof]`) -3. Parsed from `git remote get-url origin`: - - `git@github.com:username/repo.git` → `username` - - `https://github.com/username/repo.git` → `username` +The verifier fails at the first invalid trust boundary: -## Key type support +1. repository and JSON readability; +2. supported schema and strict structure; +3. signature and signed identity; +4. repository policy and fingerprint; +5. Git ancestry and dirty-tree policy; +6. shard and carry-forward consistency; +7. session, test, comparison, and skip outcomes; +8. required test manifest and GPU metadata; +9. timestamp validity and freshness. -The signing layer handles Ed25519, ECDSA (P-256/P-384), and RSA-PSS keys transparently. -Ed25519 is recommended for new keys — it is the fastest and produces 64-byte signatures. +Legacy schema-1/2 receipts remain readable, but schema 3 uses stricter clean +tree and signed-identity defaults. diff --git a/docs/ci_gpu_mode.md b/docs/ci_gpu_mode.md index 11b21c7..8824147 100644 --- a/docs/ci_gpu_mode.md +++ b/docs/ci_gpu_mode.md @@ -1,100 +1,61 @@ -# CI-GPU Mode +# CI-GPU mode -CI-GPU mode runs the same tests on a GitHub-hosted GPU runner and emits the same -signed receipt. Use this when you need the receipt to come from a controlled -environment rather than a developer laptop. +CI-GPU mode uses the same receipt and verifier but records `mode: ci-gpu`. +Choose it when GPU tests must run on a controlled runner rather than a +developer machine. -## When to use CI-GPU mode +## Dedicated signing identity -- Compliance requirements specify where GPU tests must run. -- You cannot trust individual developer machines for signing. -- You want the receipt to be tied to a CI identity, not a personal SSH key. -- You want to validate that the code works on a fresh, clean environment. - -## Prerequisites - -- A GitHub plan that includes GPU-powered larger runners. - See [GitHub-hosted runners documentation](https://docs.github.com/en/actions/concepts/runners/github-hosted-runners). -- A dedicated CI signing key. - -## Setting up a CI signing key +Create a dedicated key and register its public half on the GitHub account named +in the receipt: ```bash -# Generate a dedicated Ed25519 key for CI signing ssh-keygen -t ed25519 -f ci-signing-key -N "" - -# Add the PUBLIC key to your GitHub account -# → github.com/settings/keys → "New SSH key" -cat ci-signing-key.pub - -# Add the PRIVATE key as a GitHub Actions secret -# → Repository → Settings → Secrets and variables → Actions → "New repository secret" -# Name: GPU_PROOF_SIGNING_KEY -cat ci-signing-key ``` -The verifier will fetch the public key from `github.com/{your-username}.keys` automatically. - -## Example GitHub Actions workflow +Store the private key as a protected CI secret. A simplified job is: ```yaml -name: GPU Tests (CI-GPU mode) - -on: - schedule: - - cron: "0 3 * * 1" # weekly — GPU runners cost money - workflow_dispatch: - jobs: - gpu-test: - runs-on: ubuntu-latest-gpu-4 # your plan's GPU runner label - + gpu-proof: + runs-on: YOUR_GPU_RUNNER steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.11" - - - name: Install dependencies - run: pip install pytest-gpu-proof - - - name: Run GPU tests + python-version: "3.12" + - run: pip install pytest-gpu-proof + - name: Record GPU receipt env: - GPU_PROOF_KEY_DATA: ${{ secrets.GPU_PROOF_SIGNING_KEY }} + GPU_PROOF_KEY: ${{ secrets.GPU_PROOF_SIGNING_KEY }} run: | - echo "$GPU_PROOF_KEY_DATA" > /tmp/ci-key - chmod 600 /tmp/ci-key - pytest tests/ \ - --gpu-proof-enable \ + install -m 600 /dev/null /tmp/gpu-proof-key + printf '%s' "$GPU_PROOF_KEY" > /tmp/gpu-proof-key + pytest tests/gpu --gpu-proof-enable \ --gpu-proof-mode=ci-gpu \ - --gpu-proof-key=/tmp/ci-key \ - --gpu-proof-out=gpu-proof.json \ - -v - rm -f /tmp/ci-key - - - name: Upload receipt - uses: actions/upload-artifact@v4 + --gpu-proof-key=/tmp/gpu-proof-key \ + --gpu-proof-github-user=gpu-ci \ + --gpu-proof-out=gpu-proof.json + - uses: actions/upload-artifact@v4 with: name: gpu-proof-receipt path: gpu-proof.json - retention-days: 90 ``` -## Key difference from local mode +Use your platform's secret-file mechanism where available, and delete the +temporary key in an `always()` cleanup step. -In local mode, the receipt is signed with the developer's personal SSH key. -In CI-GPU mode, it is signed with a dedicated CI key — but the verification -mechanism is identical: the verifier fetches `github.com/{username}.keys`. - -The receipt `mode` field will contain `"ci-gpu"` instead of `"local"`, which -policy files can use to enforce that only CI-produced receipts are accepted. - -## Policy enforcement example +## Enforce the origin mode and signer ```yaml -# gpu-proof-policy.yaml -allow_dirty: false +signer_mode: restricted +allowed_signers: [gpu-ci] +allowed_key_fingerprints: ["SHA256:..."] +require_mode: ci-gpu max_age_days: 7 +allow_dirty: false ``` -With `require_mode: ci-gpu` planned for a future version. +The mode field is signed, so it cannot be changed from `local` after the run. +It still does not itself prove that GitHub or a particular runner executed the +tests; the controlled workflow and key custody provide that operational trust. diff --git a/docs/index.md b/docs/index.md index 3cf3259..9a77062 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,33 +1,37 @@ # pytest-gpu-proof -A pytest plugin that lets you run GPU equivalence tests locally, sign the results with your existing SSH key, and have GitHub Actions verify the receipt — **without re-running the GPU tests in CI**. - -The trust model is simple: if you can push to GitHub, you can sign a receipt. Verification fetches your public keys from `github.com/{username}.keys`, exactly as SSH does. - -> **Requirements:** Python 3.11+ · pytest 7.0+ · cryptography 41.0+ -> The package is not yet on PyPI — install from source with `pip install -e .` or `pip install "git+https://github.com/A2R-Lab/pytest-gpu-proof.git"`. - -## How it works - -``` -Local machine (GPU) GitHub Actions (CPU only) -───────────────────────────── ────────────────────────────────────── -pytest --gpu-proof-enable → → gpu-proof verify --receipt gpu-proof.json - runs your GPU tests fetches your public keys from - computes code fingerprint github.com/{you}.keys - signs receipt with SSH key verifies signature + fingerprint - writes gpu-proof.json exits 0 (pass) or 1 (fail) +**Signed pytest receipts for local GPU runs, verified in CPU-only CI.** + +The plugin turns an ordinary marked pytest run into a signed, reviewable +artifact. The receipt binds a signer, exact test node IDs and outcomes, a Git +commit, a source manifest, timestamps, and environment metadata. CPU-only CI +validates those claims without importing CUDA or rerunning the GPU suite. + +```text +local or lab GPU ordinary CI runner +───────────────────────────────── ───────────────────────────── +pytest --gpu-proof-enable ───────▶ gpu-proof verify +run + fingerprint + sign authenticate + recompute + policy ``` -**Zero new key management.** The plugin uses the SSH key you already have in `~/.ssh/` (the same one you use to push to GitHub). Your public key is already on GitHub. The verifier reads it from there. +The default open signer mode supports contributor-signed pull requests. +Restricted policy can instead allowlist maintainers, dedicated CI accounts, +or exact SSH key fingerprints. + +!!! important + A receipt proves that a GitHub-key holder attested to the signed payload. + It does not prove that the GPU or local machine was trustworthy. Read the + [security model](security_model.md). -## Documentation +## Start here -- [Quickstart](quickstart.md) — local CUDA proof, GitHub verification, end to end -- [Local Mode](local_mode.md) — the default workflow: sign locally, verify in CI -- [CI-GPU Mode](ci_gpu_mode.md) — run the tests on a GitHub-hosted GPU runner instead -- [Architecture](architecture.md) — package layout and data flow -- [Security Model](security_model.md) — what a receipt does and does not establish -- [Landscape](landscape.md) — why this tool exists rather than an existing one +- [Quickstart](quickstart.md): add a test, record a receipt, verify it in CI. +- [Local mode](local_mode.md): signer discovery, source scope, and failures. +- [Policy](policy.md): open and restricted trust policies. +- [Sharding and merge](sharding.md): separate processes and carry-forward. +- [CI-GPU mode](ci_gpu_mode.md): produce receipts in controlled GPU CI. +- [Architecture](architecture.md): schema and data flow. +- [Security model](security_model.md): exact guarantees and limits. -See the [README on GitHub](https://github.com/A2R-Lab/pytest-gpu-proof#readme) for the full CLI reference, receipt format, and examples. +Requirements: Python 3.11+, pytest 7+, and an SSH private-key file whose public +key is registered on GitHub. diff --git a/docs/local_mode.md b/docs/local_mode.md index e5895d9..000138c 100644 --- a/docs/local_mode.md +++ b/docs/local_mode.md @@ -1,136 +1,87 @@ -# Local Mode +# Local mode -Local mode is the default. It is the primary value proposition of this plugin: -run GPU tests on your own hardware, sign the receipt, let CI verify it. +Local mode is the primary workflow: run tests on a developer or lab GPU, sign +the result, and let CPU-only CI verify it. -## Prerequisites +## Recording contract -- An SSH key registered on your GitHub account (`github.com/settings/keys`). - This is the same key you use to push to GitHub — no new key needed. -- Your code in a git repository with a GitHub remote. +With `--gpu-proof-enable`, receipt generation is fail-closed: -## Step-by-step +- no selected tests is an error; +- setup, call, and teardown failures are recorded; +- an interrupted test without a terminal report becomes an error; +- missing Git metadata, empty fingerprints, missing keys, and write failures + fail pytest; +- the previous output is removed at session start, so a failed run cannot + leave a stale success artifact. -### 1. Install the plugin +`--gpu-proof-best-effort` converts receipt-generation errors to warnings. It is +useful while integrating the plugin, but it should not be used in a release or +merge workflow. -```bash -pip install pytest-gpu-proof -``` - -### 2. Write tests using the fixture - -```python -import pytest - -@pytest.mark.gpu_proof -def test_rnea(gpu_proof_check): - gpu_proof_check( - name="rnea", - reference=pinocchio_rnea, - candidate=cuda_rnea, - args=(model, q, qd, qdd), - metadata={"robot": "go2", "algorithm": "rnea"}, - ) -``` +## Signer and key discovery -### 3. Run the tests locally (on your GPU machine) +The GitHub username is selected from: -```bash -pytest tests/ --gpu-proof-enable -v -``` +1. `--gpu-proof-github-user` or `[tool.gpu_proof].github_username`; +2. authenticated `gh api user` output; +3. the origin owner, with a warning. -The plugin: -- Runs every test normally. -- For tests that use `gpu_proof_check`, records the comparison outcome. -- At session end, detects your SSH key, computes a code fingerprint, and signs the receipt. -- Writes `gpu-proof.json` in the current directory. +Set the username explicitly for organization-owned repositories. The receipt +must name the account that actually owns the public SSH key. -Output: -``` -... -[gpu-proof] Receipt written to gpu-proof.json -[gpu-proof] Signed with key SHA256:abc123... -``` +The private-key file is selected from: -### 4. Commit the receipt +1. `--gpu-proof-key`; +2. `git config user.signingKey`; +3. `~/.ssh/id_ed25519`, `id_ecdsa`, or `id_rsa`. -```bash -git add gpu-proof.json -git commit -m "gpu proof receipt: update after rnea kernel fix" -git push -``` +Encrypted key files prompt for a passphrase. SSH-agent-only and hardware-backed +keys are not currently supported. -The receipt is a human-readable JSON file. It is safe to commit — it contains no secrets. +## Source fingerprint -### 5. Add a CI verification step +The safe default is the entire tracked repository: -```yaml -# .github/workflows/ci.yml -- name: Verify GPU proof receipt - run: gpu-proof verify --receipt gpu-proof.json +```toml +[tool.gpu_proof] +fingerprint_paths = ["."] ``` -No GPU, no secrets, no CUDA dependencies required in CI. - -If your suite has a small set of permanent, documented skips, pin them instead -of waving all skips through: keep a baseline file (one node ID per line, `#` -comments allowed) and verify with +The manifest binds tracked file bytes, symlink targets, executable modes, and +submodule gitlinks. Untracked build debris is excluded. Explicitly add ignored +or generated dependencies: -```yaml -- name: Verify GPU proof receipt - run: gpu-proof verify --receipt gpu-proof.json --expected-skips expected_skips.txt +```toml +fingerprint_extra_paths = [ + "generated/kernel_table.cuh", + "vendor/generated-config.json", +] +fingerprint_excluded_paths = ["gpu-proof.json"] ``` -Verification then fails on any skip *not* in the baseline (something new is -being silently skipped) **and** on any baseline entry that no longer skips -(the baseline is stale — update it). `--allow-skipped` remains the loose -alternative and the two are mutually exclusive. +The default receipt exclusion prevents a tracked `gpu-proof.json` from +hashing its previous contents and invalidating its replacement. Change this +list when the committed final receipt uses another path. Exclusions weaken the +scope like any omission, so repository policy should pin them. -## Who signs the receipt +Narrowing the tracked paths narrows the claim. Only do it when repository +policy independently pins the scope and the omitted files cannot influence +the run. -The signer identity recorded in the receipt (and whose `github.com/.keys` -CI verifies against) is resolved in order: -1. `--gpu-proof-github-user` / `github_username` in `[tool.gpu_proof]` -2. the authenticated GitHub CLI login (`gh api user`), if `gh` is available — - this is the actual keyholder -3. the origin remote owner, as a last-resort guess — with a warning, because - for org-owned repos this is the **org**, which has no SSH keys, and - verification would fail. Set option 1 or 2 up properly in that case. +## Clean trees and commit ancestry -## Controlling which key is used +Schema-3 verification rejects a dirty recording tree or dirty verification +tree by default. The receipt commit may equal the verified commit or be its +ancestor; the manifest must still match the checked-out tree. A policy may set +`allow_dirty: true`, but this weakens reproducibility. -By default the plugin tries these in order: -1. `git config user.signingKey` -2. `~/.ssh/id_ed25519` -3. `~/.ssh/id_ecdsa` -4. `~/.ssh/id_rsa` +## Skips -To specify explicitly: +Selected skips are recorded and rejected by default. -```bash -pytest tests/ --gpu-proof-enable --gpu-proof-key=~/.ssh/my_key -v -``` - -## Controlling the fingerprint scope - -By default the plugin fingerprints `src/` and `tests/`. To change this: - -```bash -pytest tests/ --gpu-proof-enable --gpu-proof-fingerprint-paths=src,lib,tests -v -``` +- `--gpu-proof-fail-on-skip` fails recording and writes no receipt. +- `--expected-skips FILE` accepts exactly the listed node IDs at verification. +- `--allow-skipped` accepts any selected skip and is intentionally broad. -## What happens when the repo is dirty - -The plugin records `"dirty": true` in the receipt but does not block signing. -The verifier warns about dirty receipts. You can enforce a clean-tree policy with -a policy file: - -```yaml -# gpu-proof-policy.yaml -allow_dirty: false -max_age_days: 14 -``` - -```bash -gpu-proof verify --receipt gpu-proof.json --policy gpu-proof-policy.yaml -``` +An exact baseline catches both new skips and stale entries that now run. diff --git a/docs/policy.md b/docs/policy.md new file mode 100644 index 0000000..f69c5b3 --- /dev/null +++ b/docs/policy.md @@ -0,0 +1,91 @@ +# Verification policy + +Policy is a repository-owned JSON or YAML object passed to `gpu-proof verify`. +Unknown fields are errors, preventing a misspelled control from silently doing +nothing. + +## Open contributor mode + +Open mode is the default. Any GitHub user whose current public SSH key verifies +the signed identity may submit a receipt: + +```yaml +signer_mode: open +max_age_days: 30 +require_mode: local +required_fingerprint_paths: ["."] +required_fingerprint_excluded_paths: [gpu-proof.json] +allow_dirty: false +allow_carried: false +``` + +This works well for pull requests: the author signs with their own key and the +reviewer sees exactly who attested to the run. + +## Restricted mode + +Restricted mode requires at least one allowlist: + +```yaml +signer_mode: restricted +allowed_signers: [alice, gpu-ci] +allowed_key_fingerprints: + - "SHA256:base64-fingerprint" +``` + +If both lists are present, the username and key must both be allowed. Key +fingerprints are useful for dedicated CI keys or explicit key rotation. +The checked fingerprint is always derived from the key that actually verified +the signature — never from the receipt's own (unsigned) envelope — so a +legacy receipt cannot satisfy the allowlist by asserting a fingerprint. +Restricted mode also rejects unsigned receipts even under `--allow-unsigned`. + +Legacy schema-1/2 receipts bind less identity into the signature than +schema 3. Repositories that have finished migrating should pin +`min_schema: 3` to refuse them outright. + +## Full field reference + +| Field | Meaning | +|---|---| +| `signer_mode` | `open` or `restricted` | +| `allowed_signers` | GitHub usernames accepted in restricted mode | +| `allowed_key_fingerprints` | SSH SHA-256 fingerprints accepted in restricted mode | +| `min_schema` | Minimum acceptable receipt schema version (1, 2, or 3) | +| `max_age_days` | Maximum age of the merged/session receipt | +| `require_mode` | Required receipt mode: `local` or `ci-gpu` | +| `allow_dirty` | Permit dirty recording or verification trees | +| `required_fingerprint_paths` | Exact tracked path list | +| `required_fingerprint_extra_paths` | Exact generated/ignored path list | +| `required_fingerprint_excluded_paths` | Exact receipt/self-reference exclusion list | +| `required_test_manifest` | Repository-relative file containing exact node IDs | +| `required_shard_fingerprints` | Exact paths/extras for every named shard | +| `allow_carried` | Permit carry-forward shards | +| `carried_max_age_days` | Maximum age of each carried shard's original run | + +Example shard scope: + +```yaml +required_shard_fingerprints: + l1: + paths: [src/l1, tests/gpu/test_l1.py] + extra_paths: [generated/l1_table.cuh] + excluded_paths: [gpu-proof.json] + solvers: + paths: [src/solvers, tests/gpu/test_solvers.py] + extra_paths: [] + excluded_paths: [gpu-proof.json] +``` + +The required shard names must exactly match the receipt. Within each scope, +only listed fields are pinned; list `paths`, `extra_paths`, and +`excluded_paths` when policy should lock the complete scope. + +## CLI-only controls + +`--allow-unsigned`, `--allow-skipped`, `--expected-skips`, and `--require-gpu` +are verifier flags. `max_age_days`, `require_gpu`, and inline +`expected_skips` can also be read from `[tool.gpu_proof]`. + +Unsigned acceptance authenticates no signer. Treat it as a development tool, +not a repository trust policy. diff --git a/docs/quickstart.md b/docs/quickstart.md index cef941d..5268127 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,377 +1,101 @@ -# Quickstart: local CUDA proof, GitHub verification +# Quickstart -This guide creates a separate GitHub repository that builds a tiny CUDA -matrix-multiply extension with nanobind, runs the GPU proof locally, commits the -signed receipt, and lets GitHub Actions verify it on an ordinary CPU runner. - -The CI job does not need CUDA, a GPU, or secrets. It only verifies the committed -`gpu-proof.json`. - -## What you need - -- A Linux machine with a CUDA-capable GPU, NVIDIA driver, CUDA toolkit, `nvcc`, - CMake, and Python 3.8+. -- A GitHub account with an SSH public key registered at - `github.com/settings/keys`. -- A new empty GitHub repository. The examples below use - `git@github.com:YOUR_USER/gpu-proof-nanobind-demo.git`. - -## 1. Create the demo repo +## 1. Install ```bash -mkdir gpu-proof-nanobind-demo -cd gpu-proof-nanobind-demo -git init -git remote add origin git@github.com:YOUR_USER/gpu-proof-nanobind-demo.git - -mkdir -p src tests .github/workflows -``` - -Create a virtual environment and install the runtime/build tools: - -```bash -python3 -m venv .venv -source .venv/bin/activate -python -m pip install -U pip -python -m pip install pytest-gpu-proof numpy nanobind -``` - -Add a `.gitignore`: - -```gitignore -.venv/ -build/ -*.so -*.egg-info/ -__pycache__/ -.pytest_cache/ -``` - -## 2. Add the CUDA nanobind extension - -Create `CMakeLists.txt`: - -```cmake -cmake_minimum_required(VERSION 3.18) -project(cuda_nanobind_matmul LANGUAGES CXX CUDA) - -if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) - set(CMAKE_CUDA_ARCHITECTURES 60) -endif() - -find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) - -execute_process( - COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir - OUTPUT_VARIABLE nanobind_ROOT - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET -) -find_package(nanobind CONFIG REQUIRED) - -nanobind_add_module(cuda_nanobind_matmul - src/bindings.cpp - src/matmul_kernel.cu -) - -target_compile_features(cuda_nanobind_matmul PRIVATE cxx_std_17) -target_include_directories(cuda_nanobind_matmul PRIVATE src) -set_target_properties(cuda_nanobind_matmul PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/src" -) -``` - -Create `src/matmul_cuda.h`: - -```cpp -#pragma once - -int cuda_matmul_square_f32(const float *a_host, const float *b_host, float *c_host, int n); -``` - -Create `src/matmul_kernel.cu`: - -```cpp -#include "matmul_cuda.h" - -#include - -__global__ void matmul_square_kernel(const float *a, const float *b, float *c, int n) { - int row = blockIdx.y * blockDim.y + threadIdx.y; - int col = blockIdx.x * blockDim.x + threadIdx.x; - - if (row >= n || col >= n) { - return; - } - - float acc = 0.0f; - for (int k = 0; k < n; ++k) { - acc += a[row * n + k] * b[k * n + col]; - } - c[row * n + col] = acc; -} - -int cuda_matmul_square_f32(const float *a_host, const float *b_host, float *c_host, int n) { - const size_t bytes = static_cast(n) * static_cast(n) * sizeof(float); - float *a_dev = nullptr; - float *b_dev = nullptr; - float *c_dev = nullptr; - - cudaError_t status = cudaMalloc(&a_dev, bytes); - if (status != cudaSuccess) { - return static_cast(status); - } - status = cudaMalloc(&b_dev, bytes); - if (status != cudaSuccess) { - cudaFree(a_dev); - return static_cast(status); - } - status = cudaMalloc(&c_dev, bytes); - if (status != cudaSuccess) { - cudaFree(a_dev); - cudaFree(b_dev); - return static_cast(status); - } - - status = cudaMemcpy(a_dev, a_host, bytes, cudaMemcpyHostToDevice); - if (status == cudaSuccess) { - status = cudaMemcpy(b_dev, b_host, bytes, cudaMemcpyHostToDevice); - } - if (status == cudaSuccess) { - dim3 block(16, 16); - dim3 grid((n + block.x - 1) / block.x, (n + block.y - 1) / block.y); - matmul_square_kernel<<>>(a_dev, b_dev, c_dev, n); - status = cudaGetLastError(); - } - if (status == cudaSuccess) { - status = cudaDeviceSynchronize(); - } - if (status == cudaSuccess) { - status = cudaMemcpy(c_host, c_dev, bytes, cudaMemcpyDeviceToHost); - } - - cudaFree(a_dev); - cudaFree(b_dev); - cudaFree(c_dev); - return static_cast(status); -} -``` - -Create `src/bindings.cpp`: - -```cpp -#include -#include -#include - -#include -#include - -#include "matmul_cuda.h" - -namespace nb = nanobind; - -std::vector matmul_square(const std::vector &a, - const std::vector &b, - int n) { - const size_t expected = static_cast(n) * static_cast(n); - if (n <= 0 || a.size() != expected || b.size() != expected) { - throw std::invalid_argument("expected two flat row-major n x n matrices"); - } - - std::vector out(expected); - int status = cuda_matmul_square_f32(a.data(), b.data(), out.data(), n); - if (status != 0) { - throw std::runtime_error("CUDA matmul failed with cudaError_t=" + std::to_string(status)); - } - return out; -} - -NB_MODULE(cuda_nanobind_matmul, m) { - m.doc() = "nanobind CUDA matrix multiplication example for pytest-gpu-proof"; - m.def("matmul_square", &matmul_square, nb::arg("a_flat"), nb::arg("b_flat"), nb::arg("n")); -} +python -m pip install pytest-gpu-proof ``` -## 3. Add the GPU proof test +Run the following commands from the root of the project whose code will be +attested. It must be a Git repository. -Create `tests/test_cuda_nanobind_matmul.py`: +## 2. Select and compare a GPU test ```python -import importlib.util - import numpy as np import pytest -def reference_matmul(a_flat, b_flat, n): - a = np.asarray(a_flat, dtype=np.float32).reshape(n, n) - b = np.asarray(b_flat, dtype=np.float32).reshape(n, n) - return (a @ b).reshape(n * n).tolist() - +def compare(ref, candidate): + np.testing.assert_allclose(candidate, ref, rtol=1e-5, atol=1e-6) -def compare_allclose(ref, cand): - np.testing.assert_allclose( - np.asarray(cand, dtype=np.float32), - np.asarray(ref, dtype=np.float32), - rtol=1e-5, - atol=1e-5, - ) - -@pytest.mark.gpu_required @pytest.mark.gpu_proof -def test_nanobind_cuda_matmul_square(gpu_proof_check): - if importlib.util.find_spec("cuda_nanobind_matmul") is None: - pytest.fail("cuda_nanobind_matmul is not built. Run `cmake -S . -B build && cmake --build build`.") - - import cuda_nanobind_matmul - - n = 4 - a = (np.arange(n * n, dtype=np.float32) / 17.0).tolist() - b = (np.flip(np.arange(n * n, dtype=np.float32)).copy() / 19.0).tolist() - +@pytest.mark.gpu_required +def test_matmul(gpu_proof_check): gpu_proof_check( - name="cuda_nanobind_matmul_4x4", - reference=reference_matmul, - candidate=cuda_nanobind_matmul.matmul_square, - args=(a, b, n), - compare=compare_allclose, - metadata={"binding": "nanobind", "shape": "4x4", "dtype": "float32"}, + name="matmul", + reference=numpy_matmul, + candidate=cuda_matmul, + args=(a, b), + compare=compare, + metadata={"dtype": "float32"}, ) ``` -## 4. Add CPU-only GitHub verification - -Create `.github/workflows/verify-gpu-proof.yml`: - -```yaml -name: Verify GPU proof - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - verify: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install verifier - run: python -m pip install pytest-gpu-proof - - - name: Verify committed GPU receipt - run: | - gpu-proof verify \ - --receipt gpu-proof.json \ - --repo . \ - --max-age-days 30 -``` - -This job verifies your signature, the current commit, the fingerprint of `src/` -`tests/`, and `CMakeLists.txt`, receipt freshness, and the recorded test -outcomes. - -## 5. Commit the initial repo +The fixture calls the reference and candidate with the same arguments. A +failed comparison fails pytest and is recorded. Plain marked tests are also +supported when comparison happens elsewhere. -Make an initial commit before generating the proof. The receipt records the -current commit SHA, so generate it from the commit you plan to push. +## 3. Record ```bash -git add . -git commit -m "add CUDA nanobind GPU proof demo" -git branch -M main +pytest tests/gpu \ + --gpu-proof-enable \ + --gpu-proof-github-user YOUR_GITHUB_USER ``` -## 6. Build and run the proof locally +The command writes `gpu-proof.json` only after a complete selected collection +has a terminal outcome. Setup failures, teardown failures, selected skips, and +session failure are represented honestly. -Build the extension with the same Python environment that has nanobind -installed: +By default every Git-tracked file is fingerprinted. If runtime behavior also +depends on generated or ignored inputs, name them explicitly: ```bash -cmake -S . -B build -DPython_EXECUTABLE="$PWD/.venv/bin/python" -cmake --build build +pytest tests/gpu --gpu-proof-enable \ + --gpu-proof-fingerprint-extra-paths=generated/kernel_table.cuh ``` -Run the proof test and write `gpu-proof.json`: +## 4. Verify locally ```bash -PYTHONPATH=src pytest tests/ \ - --gpu-proof-enable \ - --gpu-proof-fingerprint-paths=src,tests,CMakeLists.txt \ - --gpu-proof-github-user YOUR_USER \ - -v +gpu-proof verify --receipt gpu-proof.json --repo . ``` -You should see: - -```text -PASSED tests/test_cuda_nanobind_matmul.py::test_nanobind_cuda_matmul_square -[gpu-proof] Receipt written to gpu-proof.json -[gpu-proof] Signed with key SHA256:... -``` +Verification fetches the current public SSH keys for the signed GitHub user. +It then validates the signature, manifest, Git ancestry, clean-tree state, +complete outcomes, and freshness. -If your GitHub remote is already configured as `github.com/YOUR_USER/...`, the -plugin can usually infer the username and `--gpu-proof-github-user` is optional. - -## 7. Commit and push the receipt +## 5. Commit and verify in CPU-only CI ```bash git add gpu-proof.json -git commit -m "add local GPU proof receipt" -git push -u origin main +git commit -m "Record local GPU correctness receipt" ``` -Open the repository on GitHub and check the Actions tab. The workflow should -pass without a GPU runner because it only verifies the committed receipt. - -The receipt records the commit SHA that existed when you ran the GPU test. The -verifier accepts the follow-up commit that adds only `gpu-proof.json`, as long -as the recorded fingerprint still matches the checked-out files. - -## Updating the proof after code changes - -When you change files under `src/` or `tests/`, generate a new receipt for the -new commit: - -```bash -git add src tests CMakeLists.txt -git commit -m "change CUDA matmul demo" - -cmake --build build -PYTHONPATH=src pytest tests/ \ - --gpu-proof-enable \ - --gpu-proof-fingerprint-paths=src,tests,CMakeLists.txt \ - --gpu-proof-github-user YOUR_USER \ - -v - -git add gpu-proof.json -git commit -m "update GPU proof receipt" -git push +```yaml +- uses: actions/setup-python@v5 + with: + python-version: "3.12" +- run: pip install pytest-gpu-proof +- run: gpu-proof verify --receipt gpu-proof.json --repo . ``` -## Troubleshooting +For a repository trust policy: -- `No SSH private key found`: pass `--gpu-proof-key ~/.ssh/id_ed25519`, or set - `git config user.signingKey ~/.ssh/id_ed25519`. -- `Cannot determine GitHub username`: pass `--gpu-proof-github-user YOUR_USER`. -- `Fingerprint mismatch`: regenerate `gpu-proof.json` after committing the code - state you want CI to verify. -- `Commit SHA mismatch`: generate the proof after the commit that will be - pushed. -- `Signature does not match any SSH key`: make sure the public half of the key - used locally is registered on your GitHub account. +```yaml +# gpu-proof-policy.yaml +signer_mode: open +max_age_days: 30 +required_fingerprint_paths: ["."] +required_fingerprint_excluded_paths: [gpu-proof.json] +allow_dirty: false +``` -## Next steps +```yaml +- run: gpu-proof verify --receipt gpu-proof.json --repo . --policy gpu-proof-policy.yaml +``` -- [Local mode](local_mode.md) explains the workflow in general terms. -- [CI-GPU mode](ci_gpu_mode.md) shows how to run GPU tests on a GPU runner - instead of a developer machine. -- [Security model](security_model.md) explains what the receipt proves. +Continue with [local mode](local_mode.md), [policy](policy.md), or the bundled +CUDA examples in the repository's `examples/` directory. diff --git a/docs/security_model.md b/docs/security_model.md index 7b8e58b..ebe2e20 100644 --- a/docs/security_model.md +++ b/docs/security_model.md @@ -1,87 +1,111 @@ -# Security Model +# Security model -## What a signed receipt establishes +## What verification establishes -A receipt signed with `pytest-gpu-proof` establishes that: +A valid schema-3 receipt establishes that a private-key holder corresponding +to a current SSH public key on the named GitHub account signed a payload that: -1. **A specific signer** (identified by their GitHub SSH key) ... -2. **attested to a specific test run** (named test node IDs, all of which passed) ... -3. **over a specific code state** (SHA-256 fingerprint of `src/` and `tests/`) ... -4. **at a specific time** (UTC timestamps in the session block) ... -5. **at a specific git commit** (commit SHA recorded in the receipt). +- names an exact selected pytest collection; +- records terminal session, test, and comparison outcomes; +- binds a Git commit and deterministic source manifest; +- records UTC run times and software/environment metadata; +- satisfies the repository's verification policy at verification time. -## What it does NOT establish +In plain language: **the signer attests that these tests passed over this code +state at this time**. -| Claim | Status | +## What it does not establish + +| Claim | Established? | |---|---| -| The local machine was uncompromised | ❌ Not proven | -| The GPU hardware ran the code faithfully | ❌ Not proven (no hardware attestation) | -| The signing key was stored in a hardware security module | ❌ Not proven | -| The tests were run exactly once and not cherry-picked | ❌ Not enforced by the receipt alone | -| The signer is who they claim to be (beyond their GitHub identity) | ❌ Depends on GitHub account security | - -## Why a plain hash is not enough - -A SHA-256 hash of the code can be recomputed by anyone without running the tests. -A signed receipt requires the private key, which only the signer holds — so the receipt -proves the signer's involvement, not just the existence of a code state. - -## Why local signing is still useful - -For team workflows, the practical threat is **accidental breakage**, not adversarial attack. -The receipt answers the question "did someone with write access to this repository actually -run these GPU tests against this exact code and confirm they passed?" That is sufficient for: - -- Avoiding GPU cloud spend on every CI run -- Auditing which commits have been GPU-validated and by whom -- Catching the common failure mode of "tests passed last time I ran them manually" - -## When to prefer GitHub GPU execution instead - -Use `--gpu-proof-mode=ci-gpu` (GitHub GPU runner) when: - -- Your team cannot trust individual developer machines. -- You need proof that runs happened in a controlled environment. -- Your compliance requirements specify where tests must run. -- You want the receipt produced by a key that is not on a developer laptop. - -## Trust hierarchy - -``` -Strongest Weakest - │ - ├── Hardware attestation (NVIDIA HOPPER TEE, Confidential Computing) - │ Establishes that GPU HW faithfully executed the code - │ - ├── GitHub Actions GPU runner + Sigstore keyless - │ Establishes that GitHub's infrastructure ran the code - │ Signer identity tied to GitHub OIDC, logged in Rekor transparency log - │ - ├── GitHub Actions GPU runner + SSH key (CI-GPU mode) - │ Establishes that a CI job ran the code - │ Key is a GitHub Actions secret, not on any developer laptop - │ - └── Local SSH key (local mode — this plugin's default) - Establishes that a developer with GitHub push access ran the code - Key security depends on the developer's machine -``` - -## Future extension: hardware attestation - -NVIDIA's Attestation SDK (Hopper and later) can produce hardware-level evidence -that a specific GPU executed a specific workload in a verified environment. -This is out of scope for v1 but the receipt format is designed to be extendable — -a `hardware_attestation` block could be added to the `environment` section in a -future version without breaking existing receipts. - -## Carried shards (schema 2) - -A carried shard is an attestation about a **prior** run: the tests passed at an -ancestor commit, and the shard's declared input paths are byte-identical at the -verified commit (the verifier recomputes this; it is not taken on faith). What -is NOT re-established: that the prior run's environment still exists, or that -paths *outside* the shard's declared fingerprint didn't change its behavior — -declaring too-narrow shard paths weakens the claim, exactly like declaring -too-narrow global fingerprint paths. That is why `allow_carried` defaults to -**false**: accepting carried shards is an explicit policy decision, bounded by -`carried_max_age_days`. +| The signing machine was uncompromised | No | +| A physical GPU executed every operation | No | +| Self-reported GPU metadata is honest | No | +| The tests are sufficient or scientifically valid | No | +| The signer ran the tests exactly once | No | +| GitHub account/key control maps to a legal identity | No | +| A merged input shard's original signature was verified by the merger | No | + +This is not remote execution proof, a trusted execution environment, or +hardware attestation. `--require-gpu` catches accidental GPU-less recording; +it does not resist a dishonest signer. + +## Why it is useful + +Many GPU projects otherwise rely on an unaudited statement that someone ran +tests locally. The receipt makes that workflow explicit and machine-checkable: + +- code drift invalidates the manifest; +- stale receipts fail freshness policy; +- incomplete collections and setup/teardown failures are visible; +- signer identity and key fingerprint are signed; +- contributor receipts can be reviewed in open mode; +- sensitive repositories can restrict users or keys. + +The practical target is accidental breakage and accountable review, not a +malicious developer who controls both the test environment and signing key. + +## Signer policy + +Open mode accepts any valid GitHub-key holder. It does not imply repository +write access; GitHub branch protection and PR review remain responsible for +authorization. This is intentional so external contributors can submit +receipts signed by themselves. + +Restricted mode adds repository-owned username and/or key-fingerprint +allowlists. If both are configured, both must match. + +GitHub key lookup uses the account's **current** `.keys` endpoint. Key removal +therefore invalidates future verification of old receipts unless another +registered key happens to match. This is useful revocation behavior, but the +project does not provide archival key transparency. + +## Source and test scope + +The default source scope is the whole tracked repository. Narrower global or +shard scopes weaken the claim because omitted files may influence builds or +tests. Policy can require exact path lists and an exact test node-ID manifest. + +Ignored and generated inputs are not included automatically; projects must +declare them through `fingerprint_extra_paths`. Missing, empty, unreadable, +escaping, or unmerged inputs fail closed. + +## Dirty trees and replay + +Schema 3 rejects dirty recording and verification trees by default. A receipt +may verify at its exact commit or a descendant only when the source manifest +still matches. Freshness limits replay duration but no server-issued nonce is +currently used. + +`allow_dirty: true`, long age limits, `--allow-skipped`, and +`--allow-unsigned` weaken guarantees. Unsigned mode authenticates no signer, +and a `signer_mode: restricted` policy therefore refuses unsigned receipts +regardless of `--allow-unsigned`. The receipt file under verification is +itself excluded from the verification-tree dirty check (its integrity is +protected by its signature, not by Git state), so the canonical +run-then-verify flow works without gitignoring the receipt. + +Legacy schema-1/2 receipts remain verifiable for migration, with weaker +identity binding: the signer username and key fingerprint are not part of the +signed payload. The verifier compensates by deriving the policy-checked +fingerprint from the key that actually verified the signature; pin +`min_schema: 3` to refuse legacy receipts entirely once migration is done. + +## Merge and carry-forward + +The merger records shard signer provenance but does not fetch keys or verify +each input signature. The merger's signature attests to the union. A review +workflow that needs independent shard authentication should verify every +input before merging. + +Carry-forward means a test passed at an ancestor commit and its declared shard +inputs are byte-identical now. It does not establish that omitted dependencies +or the old environment remain equivalent. Carried shards are rejected unless +policy opts in and bounds their original age. + +## Stronger alternatives + +Use controlled GPU CI when the local machine cannot be trusted. Hardware +attestation, confidential computing, Sigstore/OIDC provenance, and transparent +execution logs can establish stronger properties, but are outside this +plugin's current scope. diff --git a/docs/sharding.md b/docs/sharding.md index 9b4ae92..5e63942 100644 --- a/docs/sharding.md +++ b/docs/sharding.md @@ -1,109 +1,77 @@ -# Sharded runs & merging receipts +# Sharding and merge -Large suites often can't (or shouldn't) run as one pytest session: per-module -subprocesses give crash isolation (one CUDA abort no longer erases the whole -run's results), and big projects split GPU tests across invocations or -machines. Each invocation emits its own receipt; CI wants **one** artifact. +Large GPU suites often need crash isolation or shorter retry units. Run each +shard as a separate pytest process. Receipt emission deliberately rejects +xdist workers because concurrent hooks cannot safely own one artifact. -## Emit one receipt per shard - -Point each invocation at its own output path: +## Record explicit shards ```bash -pytest tests/gpu/test_a.py --gpu-proof-enable --gpu-proof-out=receipts/a.json -pytest tests/gpu/test_b.py --gpu-proof-enable --gpu-proof-out=receipts/b.json +pytest tests/gpu/l1 --gpu-proof-enable \ + --gpu-proof-shard=l1 \ + --gpu-proof-shard-fingerprint-paths=src/l1,tests/gpu/l1 \ + --gpu-proof-shard-fingerprint-extra-paths=generated/l1.cuh \ + --gpu-proof-out=receipts/l1.json ``` -Every shard receipt is a complete, individually verifiable receipt. +Each input is independently readable as a complete schema-3 receipt. The +shard block adds a unique name, narrow manifest, exact member node IDs, +environment, timestamps, and optional carry metadata. ## Merge ```bash -gpu-proof merge --out gpu-proof.json receipts/a.json receipts/b.json +gpu-proof merge receipts/l1.json receipts/solvers.json \ + --out gpu-proof.json \ + --github-user YOUR_USER ``` -`merge` unions the shards' `tests`, spans `session.started_at`/`ended_at` -across them, records per-shard provenance under `session.shards` -(`source`, `node_count`, timestamps, and each shard's recorded signer), and -**re-signs the merged payload with your local SSH key**. The result flows -through `gpu-proof verify` completely unchanged — same schema, same seven -checks. - -Options: - -- `--github-user USERNAME` — recorded signer identity for the merged receipt - (default: the first shard's `repo.github_username`). -- `--key PATH` — SSH private key (default: `git config user.signingKey`, then - `~/.ssh/id_ed25519` / `id_ecdsa` / `id_rsa`). -- `--unsigned` — write `signature: null`; verifies only with - `--allow-unsigned`, loudly. - -## What merge refuses - -A merged receipt must mean exactly what a single-session receipt means, so -`merge` hard-refuses shards that disagree on anything a receipt pins: - -- `schema_version`, `repo.commit_sha`, `fingerprint` (digest + paths), - `mode`, or the `environment` the tests ran under (python/pytest/plugin - versions, platform); -- **duplicate node IDs across shards** — two shards attesting the same test is - a sharding bug in the runner, never something to dedupe silently. +Merge refuses inputs that disagree on: -`repo.dirty` is OR-ed: one dirty shard makes the merged attestation dirty, and -your verify-time dirty policy applies honestly. `gpu_info` is taken from the -first shard that has one, so a CPU-only shard doesn't erase the GPU record. +- schema, commit, global fingerprint, or mode; +- Python, platform, pytest, or plugin version; +- duplicate test node IDs or duplicate shard names. -## Trust model +Dirty state is ORed. GPU information comes from the first input that has it. +Session times span the inputs and schema-3 session outcome fails if any input +session failed. -Consistent with the [security model](security_model.md): the merged receipt is -an **attestation by the merger**. Shard signatures are recorded as provenance -but not re-verified at merge time (merging is offline); the merged signature -is what CI verifies. If shards were signed by someone else, verification of -the merged receipt attests that *you* vouch for the union. +The merged receipt records input source names, counts, times, and recorded +signers, then signs the result with the merger's key. Input signatures are +provenance, not independently verified during offline merge. The merger +attests to the union. -## Per-shard fingerprints & carry-forward (schema 2) - -Declare each invocation as a **shard** and the receipt becomes schema `"2"`, -carrying that shard's own *narrow* fingerprint over the paths you declare: - -```bash -pytest tests/gpu/test_a.py --gpu-proof-enable \ - --gpu-proof-shard=test_a \ - --gpu-proof-shard-fingerprint-paths=tests/gpu/test_a.py,src/kernels_a \ - --gpu-proof-out=receipts/a.json -``` - -`gpu-proof merge` unions schema-2 shards exactly like schema-1 receipts (shard -names must be unique). The new capability is **carry-forward**: +## Carry forward unchanged shards ```bash -gpu-proof merge --out gpu-proof.json --carry-from last-green/gpu-proof.json \ - --repo . receipts/*.json +gpu-proof merge receipts/fresh-l1.json \ + --carry-from last-green/gpu-proof.json \ + --repo . \ + --out gpu-proof.json ``` -Shards present in the old receipt but absent from the fresh inputs are grafted -in, **marked `carried`**, iff: - -1. the old receipt's commit is an **ancestor** of the fresh one (same history), and -2. the shard's narrow fingerprint **recomputes identical** against the current - tree — the inputs that shard proved are unchanged. +An absent old shard is carried only when: -A shard whose inputs changed refuses to carry (re-run it). Freshly re-run -shards always win over old ones. +1. its receipt commit is the current commit or an ancestor; +2. its stored narrow manifest—including explicit extra paths—recomputes + identically at the current tree; +3. it does not duplicate a fresh node ID; +4. every claimed node ID exists in the old tests list. -### Verification of schema-2 receipts +Freshly rerun shards always win. Changed or malformed scopes require a rerun. -`gpu-proof verify` additionally checks, for every shard: the narrow -fingerprint recomputes clean at the verifying tree, and shard membership -exactly partitions `tests[]`. **Carried shards are rejected by default** — the -policy must opt in: +Verification rejects carried shards by default: ```yaml -allow_carried: true # default false — the trust boundary -carried_max_age_days: 30 # carried shard's ORIGINAL run must be fresher +allow_carried: true +carried_max_age_days: 14 +required_shard_fingerprints: + l1: + paths: [src/l1, tests/gpu/l1] + extra_paths: [generated/l1.cuh] + excluded_paths: [gpu-proof.json] ``` -A receipt with carried shards verified under `allow_carried: true` means: -*every test either ran at this commit, or ran at an ancestor commit on inputs -that are provably byte-identical today, within the age window* — and the -merger signed for that claim. +The required shard map pins both the full shard set and every declared scope. +Carry-forward remains weaker than a fresh run; use it only when the dependency +boundaries are reviewable and complete. diff --git a/mkdocs.yml b/mkdocs.yml index 0452f38..d82071e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,6 +32,7 @@ nav: - Home: index.md - Quickstart: quickstart.md - Local Mode: local_mode.md + - Verification Policy: policy.md - CI-GPU Mode: ci_gpu_mode.md - Sharding & Merge: sharding.md - Architecture: architecture.md diff --git a/pyproject.toml b/pyproject.toml index 94c7845..8564411 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "pytest-gpu-proof" -version = "0.3.0" +version = "0.4.0" description = "pytest plugin for GPU equivalence testing with signed receipts verified via GitHub SSH keys" readme = "README.md" requires-python = ">=3.11" @@ -37,12 +37,15 @@ Changelog = "https://github.com/A2R-Lab/pytest-gpu-proof/blob/main/CHANGELOG.md" [project.optional-dependencies] dev = [ - "pytest-cov", - "numpy", -] -sigstore = [ - "sigstore", + "build>=1.2", + "mkdocs-material>=9.5", + "numpy>=1.26", + "coverage[toml]>=7.6", + "hatchling>=1.25", + "PyYAML>=6.0", + "twine>=5.0", ] +yaml = ["PyYAML>=6.0"] [project.entry-points."pytest11"] gpu-proof = "pytest_gpu_proof.plugin" @@ -52,9 +55,23 @@ gpu-proof = "pytest_gpu_proof.cli:main" [tool.hatch.build.targets.wheel] packages = ["src/pytest_gpu_proof"] +core-metadata-version = "2.4" + +[tool.hatch.build.targets.sdist] +core-metadata-version = "2.4" [tool.pytest.ini_options] testpaths = ["tests"] +addopts = "--strict-markers" + +[tool.coverage.run] +branch = true +source = ["pytest_gpu_proof"] + +[tool.coverage.report] +fail_under = 100 +show_missing = true +skip_covered = true [tool.gpu_proof] # Default plugin/verifier configuration, read from the rootdir pyproject.toml. @@ -64,6 +81,7 @@ testpaths = ["tests"] # signing_backend = "ed25519" # required_marker = "gpu_proof" # fail_on_skip = false -# fingerprint_paths = ["src", "tests"] +# fingerprint_paths = ["."] +# fingerprint_excluded_paths = ["gpu-proof.json"] # max_age_days = 30 # verifier freshness default # require_gpu = false # verifier: require environment.gpu_info in the receipt diff --git a/requirements-dev.txt b/requirements-dev.txt index 330c3ca..32a43d8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,8 @@ -r requirements.txt -pytest-cov>=4.0 -numpy>=1.20 +coverage[toml]>=7.6 +hatchling>=1.25 +numpy>=1.26 +PyYAML>=6.0 +mkdocs-material>=9.5 +build>=1.2 +twine>=5.0 diff --git a/src/pytest_gpu_proof/__init__.py b/src/pytest_gpu_proof/__init__.py index 493f741..6a9beea 100644 --- a/src/pytest_gpu_proof/__init__.py +++ b/src/pytest_gpu_proof/__init__.py @@ -1 +1 @@ -__version__ = "0.3.0" +__version__ = "0.4.0" diff --git a/src/pytest_gpu_proof/cli.py b/src/pytest_gpu_proof/cli.py index 2817127..804a6f5 100644 --- a/src/pytest_gpu_proof/cli.py +++ b/src/pytest_gpu_proof/cli.py @@ -98,7 +98,7 @@ def main(): help="Write signature: null — the merged receipt then " "verifies only with --allow-unsigned, loudly") mp.add_argument("--carry-from", default=None, metavar="RECEIPT", - help="Graft still-valid shards from an older schema-2 " + help="Graft still-valid shards from an older sharded " "receipt: each absent-from-fresh shard is carried iff the " "old commit is an ancestor of the new one AND its narrow " "fingerprint recomputes clean at the current tree. Carried " @@ -130,7 +130,7 @@ def main(): + (" (UNSIGNED)" if args.unsigned else "")) sys.exit(0) - if args.command == "verify": + if args.command == "verify": # pragma: no branch - argparse requires a known subcommand from .verify import verify_receipt ok = verify_receipt( diff --git a/src/pytest_gpu_proof/compare.py b/src/pytest_gpu_proof/compare.py index 7b47587..159bd59 100644 --- a/src/pytest_gpu_proof/compare.py +++ b/src/pytest_gpu_proof/compare.py @@ -2,19 +2,31 @@ def default_compare(ref: Any, cand: Any) -> None: - """numpy.allclose for float arrays, exact equality otherwise. Raises AssertionError on mismatch.""" + """Shape-safe allclose for floats and exact equality for other arrays.""" try: import numpy as np - ref_arr = np.asarray(ref) - cand_arr = np.asarray(cand) - if ref_arr.dtype.kind in ("f", "c"): - if not np.allclose(ref_arr, cand_arr): - max_diff = float(np.max(np.abs(ref_arr - cand_arr))) - raise AssertionError(f"Arrays not close: max difference = {max_diff:.6e}") + except ImportError: + np = None + if np is not None: + try: + ref_arr = np.asarray(ref) + cand_arr = np.asarray(cand) + except (TypeError, ValueError): + ref_arr = cand_arr = None + if ref_arr is not None and cand_arr is not None: + if ref_arr.shape != cand_arr.shape: + raise AssertionError( + f"Shapes differ: reference={ref_arr.shape}, candidate={cand_arr.shape}" + ) + if ref_arr.dtype.kind in ("f", "c") or cand_arr.dtype.kind in ("f", "c"): + if not np.allclose(ref_arr, cand_arr, equal_nan=True): + max_diff = float(np.nanmax(np.abs(ref_arr - cand_arr))) + raise AssertionError(f"Arrays not close: max difference = {max_diff:.6e}") + return + if not np.array_equal(ref_arr, cand_arr): + raise AssertionError("Arrays are not exactly equal") return - except (ImportError, TypeError, ValueError): - pass if ref != cand: raise AssertionError(f"Values not equal: {ref!r} != {cand!r}") @@ -47,3 +59,10 @@ def run_comparison( return "passed", ref_result, cand_result, None except AssertionError as e: return "failed", ref_result, cand_result, str(e) + except Exception as e: + return ( + "error", + ref_result, + cand_result, + f"Comparator raised {type(e).__name__}: {e}", + ) diff --git a/src/pytest_gpu_proof/config.py b/src/pytest_gpu_proof/config.py index 4cc576a..c2212c7 100644 --- a/src/pytest_gpu_proof/config.py +++ b/src/pytest_gpu_proof/config.py @@ -11,17 +11,24 @@ class GpuProofConfig: output: str = "gpu-proof.json" key_path: Optional[str] = None signing_backend: str = "ed25519" - policy_path: Optional[str] = None required_marker: str = "gpu_proof" fail_on_skip: bool = False - fingerprint_paths: List[str] = field(default_factory=lambda: ["src", "tests"]) + fingerprint_paths: List[str] = field(default_factory=lambda: ["."]) + fingerprint_extra_paths: List[str] = field(default_factory=list) + fingerprint_excluded_paths: List[str] = field( + default_factory=lambda: ["gpu-proof.json"] + ) github_username: Optional[str] = None max_age_days: int = 30 require_gpu: bool = False - # Sharded emission (schema "2"): a declared shard name + its narrow + # Sharded emission: a declared shard name + its narrow # fingerprint paths (None -> the global fingerprint_paths). shard_name: Optional[str] = None shard_fingerprint_paths: Optional[List[str]] = None + shard_fingerprint_extra_paths: Optional[List[str]] = None + repo_root: str = "." + invocation_args: List[str] = field(default_factory=list) + best_effort: bool = False def load_toml_defaults(root: Union[str, Path]) -> Dict[str, Any]: @@ -64,7 +71,7 @@ def resolve(opt_name, toml_key, default): return toml_val return default - raw_paths = resolve("--gpu-proof-fingerprint-paths", "fingerprint_paths", "src,tests") + raw_paths = resolve("--gpu-proof-fingerprint-paths", "fingerprint_paths", ".") if isinstance(raw_paths, str): paths = [p.strip() for p in raw_paths.split(",") if p.strip()] else: @@ -79,6 +86,12 @@ def resolve(opt_name, toml_key, default): else: shard_paths = None + def path_list(opt_name, toml_key, default=None): + raw = resolve(opt_name, toml_key, [] if default is None else default) + if isinstance(raw, str): + return [p.strip() for p in raw.split(",") if p.strip()] + return [str(p) for p in raw] + max_age = toml_cfg.get("max_age_days") max_age_days = int(max_age) if max_age is not None else 30 @@ -88,16 +101,42 @@ def resolve(opt_name, toml_key, default): output=resolve("--gpu-proof-out", "output", "gpu-proof.json"), key_path=resolve("--gpu-proof-key", "key_path", None), signing_backend=resolve("--gpu-proof-signing-backend", "signing_backend", "ed25519"), - policy_path=resolve("--gpu-proof-policy", "policy_path", None), required_marker=resolve("--gpu-proof-required-marker", "required_marker", "gpu_proof"), # store_true flag: False just means "not passed", so OR with the toml # value rather than sentinel-resolving (a CLI flag can only turn it ON). fail_on_skip=bool(opt("--gpu-proof-fail-on-skip", False) or toml_cfg.get("fail_on_skip", False)), fingerprint_paths=paths, + fingerprint_extra_paths=path_list( + "--gpu-proof-fingerprint-extra-paths", "fingerprint_extra_paths" + ), + fingerprint_excluded_paths=path_list( + "--gpu-proof-fingerprint-excluded-paths", + "fingerprint_excluded_paths", + ["gpu-proof.json"], + ), github_username=resolve("--gpu-proof-github-user", "github_username", None), max_age_days=max_age_days, require_gpu=bool(toml_cfg.get("require_gpu", False)), shard_name=resolve("--gpu-proof-shard", "shard_name", None), shard_fingerprint_paths=shard_paths, + shard_fingerprint_extra_paths=( + path_list( + "--gpu-proof-shard-fingerprint-extra-paths", + "shard_fingerprint_extra_paths", + ) + if resolve( + "--gpu-proof-shard-fingerprint-extra-paths", + "shard_fingerprint_extra_paths", + None, + ) + is not None + else None + ), + repo_root=str(pytest_config.rootpath), + invocation_args=[str(a) for a in pytest_config.invocation_params.args], + best_effort=bool( + opt("--gpu-proof-best-effort", False) + or toml_cfg.get("best_effort", False) + ), ) diff --git a/src/pytest_gpu_proof/fingerprint.py b/src/pytest_gpu_proof/fingerprint.py index f083920..5c6b590 100644 --- a/src/pytest_gpu_proof/fingerprint.py +++ b/src/pytest_gpu_proof/fingerprint.py @@ -1,38 +1,166 @@ +"""Deterministic source manifests for receipt binding.""" + +from __future__ import annotations + import hashlib import json import os -from typing import List +from pathlib import Path +from typing import Sequence -from .gitutils import get_tracked_files +from .gitutils import GitError, TrackedEntry, get_tracked_entries, get_tracked_files -def _hash_file(path: str, root: str) -> str: - h = hashlib.sha256() - with open(os.path.join(root, path), "rb") as f: - for chunk in iter(lambda: f.read(65536), b""): - h.update(chunk) - return h.hexdigest() +FINGERPRINT_ALGORITHM = "sha256-manifest-v2" -def compute_fingerprint(paths: List[str], root: str = ".") -> dict: - tracked = get_tracked_files(paths, root) - file_hashes = {} - for p in tracked: +class FingerprintError(RuntimeError): + """Raised when the configured source scope cannot be fingerprinted safely.""" + + +def _sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _regular_entry(entry: TrackedEntry, root: Path) -> dict: + path = root / entry.path + try: + if entry.mode == "120000": + return {"kind": "symlink", "mode": entry.mode, "sha256": _sha256_bytes(os.readlink(path).encode())} + if entry.mode == "160000": + from .gitutils import get_commit_sha + + # Only trust rev-parse when the submodule is actually initialized + # there; on an empty checkout dir git would walk up and report the + # PARENT repo's HEAD. + initialized = path.is_dir() and (path / ".git").exists() + head = get_commit_sha(str(path)) if initialized else None + return {"kind": "gitlink", "mode": entry.mode, "commit": head or entry.object_id} + return {"kind": "file", "mode": entry.mode, "sha256": _sha256_bytes(path.read_bytes())} + except OSError as exc: + raise FingerprintError(f"cannot read fingerprint input {entry.path!r}: {exc}") from exc + + +def _extra_files(extra_paths: Sequence[str], root: Path) -> list[Path]: + files: list[Path] = [] + for raw in extra_paths: + candidate = Path(os.path.abspath(root / raw)) try: - file_hashes[p] = _hash_file(p, root) - except OSError: - pass + candidate.relative_to(root) + except ValueError as exc: + raise FingerprintError(f"fingerprint input escapes repository root: {raw!r}") from exc + if candidate.is_symlink() or candidate.is_file(): + files.append(candidate) + elif candidate.is_dir(): + files.extend(path for path in candidate.rglob("*") if path.is_file() or path.is_symlink()) + else: + raise FingerprintError(f"explicit fingerprint input does not exist: {raw!r}") + return sorted(set(files), key=lambda path: path.as_posix()) + + +def compute_fingerprint( + paths: Sequence[str], + root: str = ".", + *, + extra_paths: Sequence[str] = (), + exclude_paths: Sequence[str] = (), +) -> dict: + """Hash tracked inputs plus explicitly requested generated/ignored inputs. + + Directories in ``paths`` select Git-tracked entries only. Generated or + ignored artifacts must be named through ``extra_paths`` so build debris is + never swept into a receipt accidentally. + """ + root_path = Path(root).resolve() + clean_paths = sorted(dict.fromkeys(str(path) for path in paths if str(path))) + clean_extra = sorted(dict.fromkeys(str(path) for path in extra_paths if str(path))) + clean_excluded = sorted( + dict.fromkeys(str(path) for path in exclude_paths if str(path)) + ) + if not clean_paths and not clean_extra: + raise FingerprintError("fingerprint scope is empty") + try: + tracked = get_tracked_entries(clean_paths, str(root_path)) if clean_paths else [] + except GitError as exc: + raise FingerprintError(str(exc)) from exc + + manifest = { + entry.path: _regular_entry(entry, root_path) + for entry in tracked + if entry.path not in clean_excluded + } + for path in _extra_files(clean_extra, root_path): + rel = path.relative_to(root_path).as_posix() + if rel in manifest or rel in clean_excluded: + continue + try: + if path.is_symlink(): + manifest[rel] = {"kind": "extra-symlink", "sha256": _sha256_bytes(os.readlink(path).encode())} + else: + manifest[rel] = {"kind": "extra-file", "sha256": _sha256_bytes(path.read_bytes())} + except OSError as exc: + raise FingerprintError(f"cannot read fingerprint input {rel!r}: {exc}") from exc + + if not manifest: + raise FingerprintError( + "fingerprint scope matched zero files; fix fingerprint_paths or " + "fingerprint_extra_paths" + ) + canonical = json.dumps({"files": manifest}, sort_keys=True, separators=(",", ":")) + return { + "algorithm": FINGERPRINT_ALGORITHM, + "included_paths": clean_paths, + "extra_paths": clean_extra, + "excluded_paths": clean_excluded, + "file_count": len(manifest), + "digest": _sha256_bytes(canonical.encode()), + } + + +def compute_legacy_fingerprint(paths: Sequence[str], root: str = ".") -> dict: + """Reproduce the schema-1/2 fingerprint algorithm for old receipts.""" + file_hashes = {} + try: + tracked = get_tracked_files(paths, root) + except GitError as exc: + raise FingerprintError(str(exc)) from exc + for rel in tracked: + path = Path(root) / rel + if not path.is_file(): + continue + try: + file_hashes[rel] = _sha256_bytes(path.read_bytes()) + except OSError as exc: + raise FingerprintError(f"cannot read legacy fingerprint input {rel!r}: {exc}") from exc canonical = json.dumps( - {"files": {k: file_hashes[k] for k in sorted(file_hashes)}}, + {"files": {key: file_hashes[key] for key in sorted(file_hashes)}}, sort_keys=True, separators=(",", ":"), ) - digest = hashlib.sha256(canonical.encode()).hexdigest() - return { "algorithm": "sha256", "included_paths": sorted(paths), "file_count": len(file_hashes), - "digest": digest, + "digest": _sha256_bytes(canonical.encode()), } + + +def recompute_fingerprint(stored: dict, root: str = ".") -> dict: + algorithm = stored.get("algorithm") + paths = stored.get("included_paths") + if not isinstance(paths, list): + raise FingerprintError("fingerprint included_paths must be a list") + if algorithm == FINGERPRINT_ALGORITHM: + extras = stored.get("extra_paths", []) + if not isinstance(extras, list): + raise FingerprintError("fingerprint extra_paths must be a list") + excluded = stored.get("excluded_paths", []) + if not isinstance(excluded, list): + raise FingerprintError("fingerprint excluded_paths must be a list") + return compute_fingerprint( + paths, root, extra_paths=extras, exclude_paths=excluded + ) + if algorithm == "sha256": + return compute_legacy_fingerprint(paths, root) + raise FingerprintError(f"unsupported fingerprint algorithm: {algorithm!r}") diff --git a/src/pytest_gpu_proof/gitutils.py b/src/pytest_gpu_proof/gitutils.py index 41e35c2..78f56d7 100644 --- a/src/pytest_gpu_proof/gitutils.py +++ b/src/pytest_gpu_proof/gitutils.py @@ -1,51 +1,101 @@ +"""Small, strict git helpers used by receipt capture and verification.""" + +from __future__ import annotations + +import os import re import subprocess -from typing import Optional +from dataclasses import dataclass +from typing import Optional, Sequence + + +class GitError(RuntimeError): + """Raised when a trust-relevant git query cannot be completed.""" + +@dataclass(frozen=True) +class TrackedEntry: + path: str + mode: str + object_id: str -def _git(*args) -> Optional[str]: + +def _git(*args: str, root: str = ".", required: bool = False) -> Optional[str]: try: result = subprocess.run( - ["git"] + list(args), + ["git", "-C", root, *args], capture_output=True, text=True, check=True, ) - return result.stdout.strip() or None - except (subprocess.CalledProcessError, FileNotFoundError): + except (subprocess.CalledProcessError, FileNotFoundError) as exc: + if required: + detail = getattr(exc, "stderr", "") or str(exc) + raise GitError(f"git {' '.join(args)} failed in {root}: {detail.strip()}") from exc return None + return result.stdout.strip() or None + +def require_repository(root: str = ".") -> None: + if _git("rev-parse", "--show-toplevel", root=root) is None: + raise GitError(f"{root!r} is not inside a git repository") -def get_commit_sha() -> Optional[str]: - return _git("rev-parse", "HEAD") +def get_commit_sha(root: str = ".", *, required: bool = False) -> Optional[str]: + return _git("rev-parse", "HEAD", root=root, required=required) -def get_branch() -> Optional[str]: - return _git("rev-parse", "--abbrev-ref", "HEAD") +def get_branch(root: str = ".") -> Optional[str]: + return _git("rev-parse", "--abbrev-ref", "HEAD", root=root) -def is_dirty() -> bool: - # --ignore-submodules=untracked: untracked files INSIDE a submodule (build - # deps, caches) cannot change the code a receipt attests — the submodule's - # content is pinned by the parent's gitlink, and a *pin* change (a different - # or new commit checked out in the submodule) still reports as modified - # under this flag. Without it, a consumer whose submodule carries build - # artifacts can never produce a clean receipt (first hit: GATO's sqpcpu - # baseline submodule). + +def is_dirty( + root: str = ".", + *, + required: bool = False, + exclude_paths: Sequence[str] = (), +) -> bool: try: result = subprocess.run( - ["git", "status", "--porcelain", "--ignore-submodules=untracked"], + [ + "git", + "-C", + root, + "status", + "--porcelain", + "-z", + "--ignore-submodules=untracked", + ], capture_output=True, - text=True, check=True, ) - return bool(result.stdout.strip()) - except (subprocess.CalledProcessError, FileNotFoundError): + except (subprocess.CalledProcessError, FileNotFoundError) as exc: + if required: + raise GitError(f"could not inspect git status in {root}") from exc return False - - -def get_remote_url() -> Optional[str]: - return _git("remote", "get-url", "origin") + excluded = set(exclude_paths) + records = result.stdout.split(b"\0") + index = 0 + while index < len(records): + record = records[index] + index += 1 + if not record: + continue + status = record[:2] + paths = [record[3:].decode("utf-8", errors="surrogateescape")] + if b"R" in status or b"C" in status: + if index < len(records) and records[index]: + paths.append( + records[index].decode("utf-8", errors="surrogateescape") + ) + index += 1 + if any(path not in excluded for path in paths): + return True + return False + + +def get_remote_url(root: str = ".") -> Optional[str]: + return _git("remote", "get-url", "origin", root=root) def extract_github_username(remote_url: str) -> Optional[str]: @@ -53,18 +103,12 @@ def extract_github_username(remote_url: str) -> Optional[str]: return match.group(1) if match else None -def get_github_username() -> Optional[str]: - url = get_remote_url() +def get_github_username(root: str = ".") -> Optional[str]: + url = get_remote_url(root) return extract_github_username(url) if url else None def get_gh_cli_login() -> Optional[str]: - """The authenticated GitHub CLI user, if `gh` is installed and logged in. - - This is the KEYHOLDER — the account whose github.com/.keys will - verify the receipt — unlike the origin-remote owner, which for org-owned - repos is an org with no SSH keys. - """ try: result = subprocess.run( ["gh", "api", "user", "--jq", ".login"], @@ -73,40 +117,51 @@ def get_gh_cli_login() -> Optional[str]: check=True, timeout=10, ) - return result.stdout.strip() or None - except (subprocess.CalledProcessError, FileNotFoundError, - subprocess.TimeoutExpired): + except ( + subprocess.CalledProcessError, + FileNotFoundError, + subprocess.TimeoutExpired, + ): return None + return result.stdout.strip() or None -def get_git_signing_key() -> Optional[str]: - val = _git("config", "--get", "user.signingKey") - if not val: - return None - import os - return os.path.expanduser(val) +def get_git_signing_key(root: str = ".") -> Optional[str]: + value = _git("config", "--get", "user.signingKey", root=root) + return os.path.expanduser(value) if value else None -def get_tracked_files(paths, root="."): +def get_tracked_entries(paths: Sequence[str], root: str = ".") -> list[TrackedEntry]: + """Return stage-0 tracked entries under *paths*, including gitlinks.""" + if not paths: + raise GitError("fingerprint path list is empty") try: result = subprocess.run( - ["git", "ls-files"] + list(paths), + ["git", "-C", root, "ls-files", "--stage", "-z", "--", *paths], capture_output=True, - text=True, check=True, - cwd=root, ) - return sorted(line for line in result.stdout.splitlines() if line) - except (subprocess.CalledProcessError, FileNotFoundError): - import os - files = [] - for path in paths: - full = os.path.join(root, path) - if os.path.isfile(full): - files.append(os.path.relpath(full, root)) - elif os.path.isdir(full): - for dirpath, _, filenames in os.walk(full): - for fn in filenames: - fp = os.path.join(dirpath, fn) - files.append(os.path.relpath(fp, root)) - return sorted(files) + except (subprocess.CalledProcessError, FileNotFoundError) as exc: + raise GitError(f"could not enumerate tracked files in {root}") from exc + + entries: list[TrackedEntry] = [] + for raw in result.stdout.split(b"\0"): + if not raw: + continue + metadata, raw_path = raw.split(b"\t", 1) + mode, object_id, stage = metadata.decode().split() + if stage != "0": + raise GitError(f"unmerged index entry cannot be fingerprinted: {raw_path!r}") + entries.append( + TrackedEntry( + path=raw_path.decode("utf-8", errors="surrogateescape"), + mode=mode, + object_id=object_id, + ) + ) + return sorted(entries, key=lambda entry: entry.path) + + +def get_tracked_files(paths: Sequence[str], root: str = ".") -> list[str]: + """Compatibility wrapper retained for legacy fingerprint verification.""" + return [entry.path for entry in get_tracked_entries(paths, root)] diff --git a/src/pytest_gpu_proof/merge.py b/src/pytest_gpu_proof/merge.py index 9fd7c1f..7a2343f 100644 --- a/src/pytest_gpu_proof/merge.py +++ b/src/pytest_gpu_proof/merge.py @@ -4,10 +4,7 @@ Motivation: large suites run their GPU tests as several pytest invocations (per-module crash isolation, machine sharding). Each invocation emits its own receipt; CI wants ONE artifact to verify. ``gpu-proof merge`` unions the shard -receipts' ``tests`` and re-signs the result with the merger's local SSH key — -so the merged receipt flows through the existing ``gpu-proof verify`` path -completely unchanged (schema_version stays "1"; the only addition is the -OPTIONAL ``session.shards`` provenance list, which the verifier ignores). +receipts' ``tests`` and re-signs the result with the merger's local SSH key. Trust model (consistent with docs/security_model.md): the merged receipt is an attestation by the MERGER — shard signatures are recorded as provenance but are @@ -61,10 +58,10 @@ def merge_payloads(receipts: List[dict], sources: List[str]) -> dict: _require_identical(receipts, sources, lambda r: r.get("schema_version"), "schema_version") schema = receipts[0].get("schema_version") - if schema not in ("1", "2"): + if schema not in ("1", "2", "3"): raise MergeError( f"unsupported schema_version {schema!r} (this version merges " - f"schema '1' and schema '2' receipts, not mixed)" + f"schema '1', '2', and '3' receipts, not mixed)" ) _require_identical(receipts, sources, lambda r: r.get("repo", {}).get("commit_sha"), "repo.commit_sha") @@ -91,11 +88,11 @@ def merge_payloads(receipts: List[dict], sources: List[str]) -> dict: if not tests: raise MergeError("merged receipt would contain zero tests") - # schema 2: union the shards lists (shard names must be unique across + # Sharded receipts: union the shards lists (names must be unique across # inputs; a shard's node_ids stay attached to it, so the merged receipt # still partitions tests[] by shard for the verifier's membership check). merged_shards = None - if schema == "2": + if schema == "2" or (schema == "3" and any(r.get("shards") for r in receipts)): merged_shards = [] shard_names: dict = {} for src, r in zip(sources, receipts): @@ -110,8 +107,11 @@ def merge_payloads(receipts: List[dict], sources: List[str]) -> dict: merged_shards.append(shard) sessions = [r.get("session", {}) for r in receipts] - started = min(s.get("started_at") for s in sessions) - ended = max(s.get("ended_at") for s in sessions) + for src, s in zip(sources, sessions): + if not isinstance(s.get("started_at"), str) or not isinstance(s.get("ended_at"), str): + raise MergeError(f"receipt {src} has no session timestamps") + started = min(s["started_at"] for s in sessions) + ended = max(s["ended_at"] for s in sessions) merged = dict(receipts[0]) merged.pop("signature", None) @@ -140,11 +140,23 @@ def merge_payloads(receipts: List[dict], sources: List[str]) -> dict: "node_count": len(r.get("tests", [])), "started_at": s.get("started_at"), "ended_at": s.get("ended_at"), - "signer": (r.get("signature") or {}).get("signer"), + "signer": ( + (r.get("signer") or {}).get("github_user") + or (r.get("signature") or {}).get("signer") + ), } for src, r, s in zip(sources, receipts, sessions) ], } + if schema == "3": + merged["session"]["outcome"] = ( + "passed" + if all(s.get("outcome") == "passed" for s in sessions) + else "failed" + ) + merged["session"]["pytest_args"] = [ + s.get("pytest_args", []) for s in sessions + ] if merged_shards is not None: merged["shards"] = merged_shards return merged @@ -172,17 +184,18 @@ def carry_forward(payload: dict, old_receipt: dict, old_source: str, current tree — the inputs that shard proved are unchanged at HEAD. Shard signatures are provenance (merge is offline); the re-signed merged - receipt is the attestation, and the VERIFIER re-checks every carried + receipt is the attestation, and the verifier re-checks every carried shard's fingerprint and gates them on policy ``allow_carried``.""" - from .fingerprint import compute_fingerprint + from .fingerprint import FingerprintError, recompute_fingerprint - if old_receipt.get("schema_version") != "2": + if old_receipt.get("schema_version") not in ("2", "3"): raise MergeError( - f"--carry-from {old_source}: not a schema '2' (sharded) receipt") + f"--carry-from {old_source}: not a schema '2'/'3' sharded receipt") old_sha = old_receipt.get("repo", {}).get("commit_sha") new_sha = payload.get("repo", {}).get("commit_sha") - if old_sha and new_sha and old_sha != new_sha and not _git_is_ancestor( - repo_root, old_sha, new_sha): + if not old_sha or not new_sha: + raise MergeError("carry-forward receipts must contain commit SHAs") + if old_sha != new_sha and not _git_is_ancestor(repo_root, old_sha, new_sha): raise MergeError( f"--carry-from {old_source}: its commit {old_sha[:12]} is not an " f"ancestor of {new_sha[:12]} — different history, cannot carry.") @@ -196,7 +209,12 @@ def carry_forward(payload: dict, old_receipt: dict, old_source: str, if nm in fresh_names: continue # freshly re-run — the new result wins sfp = shard.get("fingerprint", {}) - snow = compute_fingerprint(sfp.get("included_paths", []), root=repo_root) + try: + snow = recompute_fingerprint(sfp, root=repo_root) + except FingerprintError as exc: + raise MergeError( + f"--carry-from {old_source}: shard {nm!r} fingerprint is invalid: {exc}" + ) from exc if snow["digest"] != sfp.get("digest"): raise MergeError( f"--carry-from {old_source}: shard {nm!r} fingerprint no longer " @@ -213,7 +231,10 @@ def carry_forward(payload: dict, old_receipt: dict, old_source: str, "from": old_source.rsplit("/", 1)[-1], "original_commit_sha": old_sha, "original_ended_at": old_receipt.get("session", {}).get("ended_at"), - "original_signer": (old_receipt.get("signature") or {}).get("signer"), + "original_signer": ( + (old_receipt.get("signer") or {}).get("github_user") + or (old_receipt.get("signature") or {}).get("signer") + ), } payload.setdefault("shards", []).append(grafted) for nid in ids: @@ -225,7 +246,7 @@ def carry_forward(payload: dict, old_receipt: dict, old_source: str, fresh_ids.add(nid) carried_count += 1 payload["session"]["node_ids"] = [t["node_id"] for t in payload["tests"]] - payload["schema_version"] = "2" + payload["schema_version"] = str(payload.get("schema_version")) if not carried_count: print(f"[gpu-proof] merge: nothing to carry from {old_source} " f"(all its shards were freshly re-run)") @@ -248,7 +269,7 @@ def merge_receipts( shard's ``repo.github_username`` — correct when the merger is also the shard runner). ``unsigned=True`` writes ``signature: null`` (verifies only with ``--allow-unsigned``, loudly, same as the plugin's 'none' backend). - ``carry_from`` grafts still-valid shards from an older schema-2 receipt — + ``carry_from`` grafts still-valid shards from an older sharded receipt — see :py:func:`carry_forward` for the soundness conditions. """ receipts = [load_receipt(p) for p in paths] @@ -265,7 +286,7 @@ def merge_receipts( receipt["signature"] = None else: from .signers.ed25519 import SSHSigner - signer = SSHSigner(key_path=key_path) + signer = SSHSigner(key_path=key_path, root=repo_root) receipt = finalize_receipt(payload, signer) write_receipt(receipt, out) return receipt diff --git a/src/pytest_gpu_proof/plugin.py b/src/pytest_gpu_proof/plugin.py index f91cc7e..7d86222 100644 --- a/src/pytest_gpu_proof/plugin.py +++ b/src/pytest_gpu_proof/plugin.py @@ -7,6 +7,7 @@ import datetime import warnings +from pathlib import Path from typing import Any, Dict, List import pytest @@ -39,6 +40,8 @@ class GpuProofPlugin: def __init__(self, pytest_config): self.gpu_proof_config: GpuProofConfig = load_config(pytest_config) self.test_results: List[Dict[str, Any]] = [] + self._results_by_node: Dict[str, Dict[str, Any]] = {} + self.collected_node_ids: List[str] = [] self.skipped_required: List[str] = [] self.started_at: str = "" @@ -48,13 +51,52 @@ def __init__(self, pytest_config): def pytest_sessionstart(self, session): self.started_at = _utcnow() + output = Path(self.gpu_proof_config.output) + if not output.is_absolute(): + output = Path(self.gpu_proof_config.repo_root) / output + self.gpu_proof_config.output = str(output) + try: + output.unlink(missing_ok=True) + except OSError as exc: + # An exitstatus write here would be overwritten by wrap_session; + # only UsageError reliably fails the run this early. + if self.gpu_proof_config.best_effort: + self._fail_or_warn(session, f"cannot clear stale receipt {output}: {exc}") + else: + raise pytest.UsageError( + f"[gpu-proof] cannot clear stale receipt {output}: {exc}" + ) from exc + + def pytest_collection_finish(self, session): + self.collected_node_ids = [ + item.nodeid + for item in session.items + if self._is_marked(item) + or "gpu_proof_check" in getattr(item, "fixturenames", ()) + ] + + def _fail_or_warn(self, session, message): + if self.gpu_proof_config.best_effort: + warnings.warn(f"[gpu-proof] {message}", stacklevel=1) + else: + print(f"\n[gpu-proof] ERROR: {message}") + session.exitstatus = pytest.ExitCode.TESTS_FAILED def pytest_sessionfinish(self, session, exitstatus): if not self.gpu_proof_config.enabled: return + if not self.collected_node_ids: + print( + "\n[gpu-proof] --gpu-proof-enable is set but no gpu_proof tests were found." + ) + self._fail_or_warn(session, "receipt requested but no marked tests were collected") + return + skipped_marked = [ - t["node_id"] for t in self.test_results if t.get("outcome") == "skipped" + node_id + for node_id, result in self._results_by_node.items() + if result.get("outcome") == "skipped" ] if self.gpu_proof_config.fail_on_skip and (skipped_marked or self.skipped_required): names = sorted(set(skipped_marked + self.skipped_required)) @@ -64,18 +106,24 @@ def pytest_sessionfinish(self, session, exitstatus): + "".join(f" - {n}\n" for n in names) + " No receipt was written; session marked as failed." ) - session.exitstatus = 1 + session.exitstatus = pytest.ExitCode.TESTS_FAILED return - if not self.test_results: - print( - "\n[gpu-proof] --gpu-proof-enable is set but no gpu_proof tests were found.\n" - " Add @pytest.mark.gpu_proof to your tests or use the gpu_proof_check fixture.\n" - " No receipt was written.\n" - " Tip: try 'pytest examples/minimal_python_only/test_minimal.py --gpu-proof-enable -v'" - ) - return - self._emit_receipt() + for node_id in self.collected_node_ids: + if node_id not in self._results_by_node: + self._results_by_node[node_id] = { + "node_id": node_id, + "outcome": "error", + "duration_s": 0.0, + "checks": [], + "phase": "missing-terminal-report", + } + self.test_results = [self._results_by_node[node] for node in self.collected_node_ids] + session_outcome = "passed" if session.exitstatus == pytest.ExitCode.OK else "failed" + try: + self._emit_receipt(session_outcome) + except Exception as exc: + self._fail_or_warn(session, f"failed to create receipt: {exc}") # ------------------------------------------------------------------ # result collection @@ -92,21 +140,36 @@ def pytest_runtest_makereport(self, item, call): outcome = yield report = outcome.get_result() + included = self._is_marked(item) or "gpu_proof_check" in getattr( + item, "fixturenames", () + ) + if not included: + if call.when == "setup" and report.skipped and item.get_closest_marker("gpu_required"): + self.skipped_required.append(item.nodeid) + return + if call.when == "setup" and report.skipped: # Skipped tests never reach the "call" phase — record them here so # they are visible in the receipt (and to --gpu-proof-fail-on-skip) # instead of being silently dropped. - if item.get_closest_marker("gpu_required") and not self._is_marked(item): - self.skipped_required.append(item.nodeid) - if self._is_marked(item): - self.test_results.append( - { - "node_id": item.nodeid, - "outcome": "skipped", - "duration_s": round(call.duration, 4), - "checks": [], - } - ) + self._results_by_node[item.nodeid] = { + "node_id": item.nodeid, + "outcome": "skipped", + "duration_s": round(call.duration, 4), + "checks": [], + "phase": "setup", + } + return + + if report.failed: + previous = self._results_by_node.get(item.nodeid, {}) + self._results_by_node[item.nodeid] = { + "node_id": item.nodeid, + "outcome": "failed", + "duration_s": round(previous.get("duration_s", 0.0) + call.duration, 4), + "checks": getattr(item, "_gpu_proof_checks", previous.get("checks", [])), + "phase": call.when, + } return if call.when == "teardown": @@ -115,19 +178,14 @@ def pytest_runtest_makereport(self, item, call): # already-recorded entry here so checks are not silently dropped. checks = getattr(item, "_gpu_proof_checks", None) if checks is not None: - for t in reversed(self.test_results): - if t["node_id"] == item.nodeid: - t["checks"] = checks - break + result = self._results_by_node.get(item.nodeid) + if result is not None: + result["checks"] = checks return if call.when != "call": return - uses_fixture = "gpu_proof_check" in getattr(item, "fixturenames", ()) - if not uses_fixture and not self._is_marked(item): - return - if report.passed: outcome_str = "passed" elif report.skipped: @@ -135,56 +193,45 @@ def pytest_runtest_makereport(self, item, call): else: outcome_str = "failed" - self.test_results.append( - { - "node_id": item.nodeid, - "outcome": outcome_str, - "duration_s": round(call.duration, 4), - "checks": [], # filled in at teardown once the fixture finalizes - } - ) + self._results_by_node[item.nodeid] = { + "node_id": item.nodeid, + "outcome": outcome_str, + "duration_s": round(call.duration, 4), + "checks": [], + "phase": "call", + } # ------------------------------------------------------------------ # receipt emission # ------------------------------------------------------------------ - def _emit_receipt(self): + def _emit_receipt(self, session_outcome): from .receipt import build_receipt_payload, finalize_receipt, write_receipt from .signers.ed25519 import SSHSigner - from .signers.base import VerifierError ended_at = _utcnow() cfg = self.gpu_proof_config + payload = build_receipt_payload( + cfg, + self.test_results, + self.started_at, + ended_at, + session_outcome=session_outcome, + collected_node_ids=self.collected_node_ids, + ) if cfg.signing_backend == "none": - try: - payload = build_receipt_payload(cfg, self.test_results, self.started_at, ended_at) - receipt = dict(payload) - receipt["signature"] = None - write_receipt(receipt, cfg.output) - print(f"\n[gpu-proof] Receipt written to {cfg.output}") - print( - "[gpu-proof] WARNING: signing backend is 'none' — the receipt is UNSIGNED\n" - " and will fail verification unless --allow-unsigned is passed." - ) - except Exception as e: - warnings.warn(f"[gpu-proof] Failed to write receipt: {e}", stacklevel=1) - return - - try: - signer = SSHSigner(key_path=cfg.key_path) - except VerifierError as e: - warnings.warn(f"[gpu-proof] Signing skipped: {e}", stacklevel=1) - return - - try: - payload = build_receipt_payload(cfg, self.test_results, self.started_at, ended_at) + receipt = dict(payload) + receipt["signature"] = None + else: + signer = SSHSigner(key_path=cfg.key_path, root=cfg.repo_root) receipt = finalize_receipt(payload, signer) - write_receipt(receipt, cfg.output) - print(f"\n[gpu-proof] Receipt written to {cfg.output}") + write_receipt(receipt, cfg.output) + print(f"\n[gpu-proof] Receipt written to {cfg.output}") + if cfg.signing_backend == "none": + print("[gpu-proof] WARNING: receipt is UNSIGNED") + else: print(f"[gpu-proof] Signed with key {signer.key_fingerprint()}") - except Exception as e: - warnings.warn(f"[gpu-proof] Failed to write receipt: {e}", stacklevel=1) # ------------------------------------------------------------------ @@ -223,12 +270,6 @@ def pytest_addoption(parser): choices=["ed25519", "none"], help="Signing backend (default: ed25519 via SSH key)", ) - group.addoption( - "--gpu-proof-policy", - default=None, - metavar="PATH", - help="Path to verification policy YAML", - ) group.addoption( "--gpu-proof-required-marker", default=None, @@ -244,14 +285,27 @@ def pytest_addoption(parser): "--gpu-proof-fingerprint-paths", default=None, metavar="PATHS", - help="Comma-separated paths to fingerprint (default: src,tests)", + help="Comma-separated tracked paths to fingerprint (default: entire repository)", + ) + group.addoption( + "--gpu-proof-fingerprint-extra-paths", + default=None, + metavar="PATHS", + help="Explicit generated/ignored files or directories to fingerprint", + ) + group.addoption( + "--gpu-proof-fingerprint-excluded-paths", + default=None, + metavar="PATHS", + help="Comma-separated receipt artifacts to exclude from the source manifest " + "(default: gpu-proof.json)", ) group.addoption( "--gpu-proof-shard", default=None, metavar="NAME", - help="Declare this run as one SHARD of a larger suite: the receipt is " - "emitted as schema '2' with a per-shard fingerprint, enabling " + help="Declare this run as one SHARD of a larger suite: the receipt has " + "a per-shard fingerprint, enabling " "verifiable carry-forward via `gpu-proof merge --carry-from`", ) group.addoption( @@ -261,12 +315,24 @@ def pytest_addoption(parser): help="Comma-separated paths for THIS shard's narrow fingerprint " "(default: the global fingerprint paths)", ) + group.addoption( + "--gpu-proof-shard-fingerprint-extra-paths", + default=None, + metavar="PATHS", + help="Explicit generated/ignored inputs for this shard", + ) group.addoption( "--gpu-proof-github-user", default=None, metavar="USERNAME", help="GitHub username of the signer (default: auto-detect from git remote)", ) + group.addoption( + "--gpu-proof-best-effort", + action="store_true", + default=False, + help="Warn instead of failing pytest when receipt creation fails", + ) def pytest_configure(config): @@ -283,16 +349,16 @@ def pytest_configure(config): enabled = False if enabled: + if hasattr(config, "workerinput"): + raise pytest.UsageError( + "pytest-gpu-proof does not support xdist workers; run receipt " + "shards as separate pytest processes and merge them instead" + ) plugin = GpuProofPlugin(config) config.pluginmanager.register(plugin, "gpu-proof-plugin") def pytest_collection_modifyitems(config, items): - try: - enabled = config.getoption("--gpu-proof-enable") - except ValueError: - enabled = False - skip_no_gpu = pytest.mark.skip(reason="No GPU available (gpu_required marker)") has_gpu = None # lazy diff --git a/src/pytest_gpu_proof/receipt.py b/src/pytest_gpu_proof/receipt.py index b7837d1..63fd7d0 100644 --- a/src/pytest_gpu_proof/receipt.py +++ b/src/pytest_gpu_proof/receipt.py @@ -1,16 +1,16 @@ -""" -Build, sign, and write the JSON receipt artifact. +"""Build, sign, and atomically write receipt artifacts.""" -Signing covers the canonical (compact, sorted-key) JSON of the receipt without -the signature field. The signature is then embedded as receipt["signature"]. -""" +from __future__ import annotations import base64 import datetime import json +import os import platform import subprocess import sys +import tempfile +from pathlib import Path from typing import Any, Dict, List, Optional from .gitutils import ( @@ -20,6 +20,7 @@ get_github_username, get_remote_url, is_dirty, + require_repository, ) @@ -32,45 +33,76 @@ def _gpu_info() -> Optional[Dict[str, Any]]: result = subprocess.run( [ "nvidia-smi", - "--query-gpu=name,driver_version,memory.total", + "--query-gpu=index,uuid,name,driver_version,memory.total,compute_cap", "--format=csv,noheader", ], capture_output=True, text=True, timeout=10, ) - if result.returncode == 0 and result.stdout.strip(): - parts = [p.strip() for p in result.stdout.strip().split(",")] - return { - "name": parts[0] if len(parts) > 0 else None, - "driver_version": parts[1] if len(parts) > 1 else None, - "memory": parts[2] if len(parts) > 2 else None, - } except (FileNotFoundError, subprocess.TimeoutExpired): - pass - return None + return None + if result.returncode != 0 or not result.stdout.strip(): + return None + devices = [] + for line in result.stdout.splitlines(): + parts = [part.strip() for part in line.split(",")] + devices.append( + { + "index": parts[0] if len(parts) > 0 else None, + "uuid": parts[1] if len(parts) > 1 else None, + "name": parts[2] if len(parts) > 2 else None, + "driver_version": parts[3] if len(parts) > 3 else None, + "memory": parts[4] if len(parts) > 4 else None, + "compute_capability": parts[5] if len(parts) > 5 else None, + } + ) + return { + "devices": devices, + "cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES"), + } def _env_info() -> Dict[str, Any]: - try: - import pytest - pytest_version = pytest.__version__ - except ImportError: - pytest_version = None + import pytest from pytest_gpu_proof import __version__ return { "python_version": sys.version.split()[0], - "platform": platform.system().lower(), - "pytest_version": pytest_version, + "platform": platform.platform(), + "pytest_version": pytest.__version__, "plugin_version": __version__, "gpu_info": _gpu_info(), } def canonicalize(receipt_dict: dict) -> bytes: - return json.dumps(receipt_dict, sort_keys=True, separators=(",", ":")).encode() + try: + return json.dumps( + receipt_dict, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + except (TypeError, ValueError) as exc: + raise ValueError(f"receipt contains a non-JSON value: {exc}") from exc + + +def _resolve_github_username(config, override: Optional[str], root: str) -> str: + username = override or config.github_username or get_gh_cli_login() + if username: + return username + username = get_github_username(root) + if username: + print( + f"[gpu-proof] WARNING: signer {username!r} was derived from the " + "origin owner; set --gpu-proof-github-user for organization repos." + ) + return username + raise ValueError( + "cannot determine signer GitHub username; pass --gpu-proof-github-user" + ) def build_receipt_payload( @@ -79,88 +111,127 @@ def build_receipt_payload( started_at: str, ended_at: str, override_github_username: Optional[str] = None, + *, + session_outcome: str = "passed", + collected_node_ids: Optional[List[str]] = None, ) -> dict: from .fingerprint import compute_fingerprint - remote_url = get_remote_url() - # Signer resolution: explicit override > [tool.gpu_proof]/flag config > - # authenticated gh CLI login (the actual keyholder) > origin-remote owner. - # The last is only a guess — for org-owned repos it yields the ORG, which - # has no SSH keys, so verification would fail; warn when we land there. - github_username = override_github_username or config.github_username - if not github_username: - github_username = get_gh_cli_login() - if not github_username: - github_username = get_github_username() - if github_username: - print( - f"[gpu-proof] WARNING: signer '{github_username}' was derived " - "from the origin remote owner, which may be an org with no SSH " - "keys. If verification fails, set --gpu-proof-github-user or " - "github_username in [tool.gpu_proof] to the keyholder." - ) - fingerprint = compute_fingerprint(config.fingerprint_paths) - - # Sharded emission (schema "2", additive): when the run declares a shard - # name, the receipt carries a `shards` list whose single entry pins THIS - # shard's own narrow fingerprint (its declared paths, defaulting to the - # global fingerprint paths) and its test membership by node id. The flat - # `tests` list remains authoritative for outcomes; the global fingerprint - # keeps its schema-1 meaning. This is what makes per-shard carry-forward - # verifiable later: a shard whose narrow fingerprint still recomputes clean - # provably ran on identical inputs. - shard_name = getattr(config, "shard_name", None) - schema_version = "2" if shard_name else "1" - shards = None - if shard_name: - shard_paths = getattr(config, "shard_fingerprint_paths", None) or config.fingerprint_paths - shards = [{ - "name": shard_name, - "fingerprint": compute_fingerprint(shard_paths), - "node_ids": [t["node_id"] for t in test_results], - "carried": None, - }] + root = str(Path(config.repo_root).resolve()) + require_repository(root) + username = _resolve_github_username(config, override_github_username, root) + excluded_paths = config.fingerprint_excluded_paths + output = Path(config.output) + if not output.is_absolute(): + output = Path(root) / output + try: + output_rel = output.resolve(strict=False).relative_to(Path(root)).as_posix() + except ValueError: + output_rel = None + dirty_exclusions = [output_rel] if output_rel in excluded_paths else [] + fingerprint = compute_fingerprint( + config.fingerprint_paths, + root, + extra_paths=config.fingerprint_extra_paths, + exclude_paths=excluded_paths, + ) + node_ids = collected_node_ids or [test["node_id"] for test in test_results] + if len(node_ids) != len(set(node_ids)): + raise ValueError("collected GPU-proof node IDs are not unique") payload = { - "schema_version": schema_version, + "schema_version": "3", "mode": config.mode, "repo": { - "remote_url": remote_url, - "github_username": github_username, - "commit_sha": get_commit_sha(), - "branch": get_branch(), - "dirty": is_dirty(), + "remote_url": get_remote_url(root), + "github_username": username, + "commit_sha": get_commit_sha(root, required=True), + "branch": get_branch(root), + "dirty": is_dirty( + root, required=True, exclude_paths=dirty_exclusions + ), }, "fingerprint": fingerprint, "session": { "started_at": started_at, "ended_at": ended_at, - "node_ids": [t["node_id"] for t in test_results], + "outcome": session_outcome, + "node_ids": list(node_ids), + "pytest_args": list(config.invocation_args), }, "tests": test_results, "environment": _env_info(), } - if shards is not None: - payload["shards"] = shards + + if config.shard_name: + shard_paths = config.shard_fingerprint_paths or config.fingerprint_paths + shard_extras = ( + config.fingerprint_extra_paths + if config.shard_fingerprint_extra_paths is None + else config.shard_fingerprint_extra_paths + ) + payload["shards"] = [ + { + "name": config.shard_name, + "fingerprint": compute_fingerprint( + shard_paths, + root, + extra_paths=shard_extras, + exclude_paths=excluded_paths, + ), + "node_ids": list(node_ids), + "environment": payload["environment"], + "started_at": started_at, + "ended_at": ended_at, + "carried": None, + } + ] return payload def finalize_receipt(payload: dict, signer) -> dict: - data = canonicalize(payload) - sig_bytes = signer.sign(data) - + """Bind signer metadata inside the signed schema-3 payload.""" + signed_payload = dict(payload) + schema = str(payload.get("schema_version", "1")) + if schema == "3": + signed_payload["signer"] = { + "github_user": payload.get("repo", {}).get("github_username"), + "algorithm": signer.algorithm(), + "backend": "ssh-local", + "key_fingerprint": signer.key_fingerprint(), + } + signature = signer.sign(canonicalize(signed_payload)) + receipt = dict(signed_payload) + receipt["signature"] = {"value": base64.b64encode(signature).decode()} + return receipt + + signature = signer.sign(canonicalize(payload)) receipt = dict(payload) receipt["signature"] = { "algorithm": signer.algorithm(), "backend": "ssh-local", - "signer": payload["repo"].get("github_username") or "unknown", + "signer": payload.get("repo", {}).get("github_username") or "unknown", "key_fingerprint": signer.key_fingerprint(), - "value": base64.b64encode(sig_bytes).decode(), + "value": base64.b64encode(signature).decode(), } return receipt def write_receipt(receipt: dict, path: str) -> None: - with open(path, "w") as f: - json.dump(receipt, f, indent=2, sort_keys=True) - f.write("\n") + """Validate JSON serialization and replace the destination atomically.""" + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + text = json.dumps(receipt, indent=2, sort_keys=True, allow_nan=False) + "\n" + temporary = None + try: + with tempfile.NamedTemporaryFile( + "w", dir=destination.parent, prefix=f".{destination.name}.", delete=False + ) as handle: + temporary = Path(handle.name) + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, destination) + finally: + if temporary is not None and temporary.exists(): + temporary.unlink() diff --git a/src/pytest_gpu_proof/signers/ed25519.py b/src/pytest_gpu_proof/signers/ed25519.py index 8363657..45efd02 100644 --- a/src/pytest_gpu_proof/signers/ed25519.py +++ b/src/pytest_gpu_proof/signers/ed25519.py @@ -9,6 +9,7 @@ import base64 import hashlib import os +import re from pathlib import Path from typing import List, Optional from urllib.request import urlopen @@ -36,10 +37,10 @@ from .base import SignerBase, VerifierError -def _discover_ssh_key() -> Optional[str]: +def _discover_ssh_key(root: str = ".") -> Optional[str]: from pytest_gpu_proof.gitutils import get_git_signing_key - signing_key = get_git_signing_key() + signing_key = get_git_signing_key(root) if signing_key and os.path.exists(signing_key): return signing_key @@ -61,6 +62,16 @@ def _public_key_fingerprint(public_key) -> str: return "SHA256:" + base64.b64encode(digest).decode().rstrip("=") +def public_key_algorithm(public_key) -> str: + if isinstance(public_key, Ed25519PublicKey): + return "ed25519" + if isinstance(public_key, EllipticCurvePublicKey): + return "ecdsa-sha256" + if isinstance(public_key, RSAPublicKey): + return "rsa-pss-sha256" + raise VerifierError(f"Unsupported public key type: {type(public_key).__name__}") + + def _sign_with_key(private_key, data: bytes) -> bytes: if isinstance(private_key, Ed25519PrivateKey): return private_key.sign(data) @@ -105,11 +116,16 @@ def _parse_pubkey_line(line: str): return None +_GITHUB_USERNAME_RE = re.compile(r"^[A-Za-z0-9](?:-?[A-Za-z0-9]){0,38}$") + + def fetch_github_public_keys(username: str) -> List: + if not _GITHUB_USERNAME_RE.match(username): + raise VerifierError(f"invalid GitHub username: {username!r}") url = f"https://github.com/{username}.keys" try: with urlopen(url, timeout=10) as resp: - content = resp.read().decode() + content = resp.read(1024 * 1024).decode() except Exception as e: raise VerifierError( f"Could not fetch public keys for GitHub user {username!r}: {e}" @@ -132,12 +148,20 @@ def verify_with_github_keys(data: bytes, signature: bytes, github_username: str) return any(_verify_with_key(k, signature, data) for k in keys) +def find_verifying_github_key(data: bytes, signature: bytes, github_username: str): + """Return the matching GitHub key, or ``None`` when none verifies.""" + for key in fetch_github_public_keys(github_username): + if _verify_with_key(key, signature, data): + return key + return None + + class SSHSigner(SignerBase): """Signs with the developer's SSH private key (same key used to push to GitHub).""" - def __init__(self, key_path: Optional[str] = None): + def __init__(self, key_path: Optional[str] = None, root: str = "."): if key_path is None: - key_path = _discover_ssh_key() + key_path = _discover_ssh_key(root) if key_path is None: raise VerifierError( "No SSH private key found. Tried git config user.signingKey and " @@ -153,10 +177,23 @@ def __init__(self, key_path: Optional[str] = None): try: self._private_key = load_ssh_private_key(key_data, password=None) - except TypeError: - import getpass - pw = getpass.getpass(f"Passphrase for {key_path}: ").encode() - self._private_key = load_ssh_private_key(key_data, password=pw) + except (TypeError, ValueError): + # cryptography signals a passphrase-protected key as TypeError or + # ValueError depending on version/format; prompt, but fail closed + # with an actionable message when no terminal is available or the + # key cannot be loaded. + try: + import getpass + + pw = getpass.getpass(f"Passphrase for {key_path}: ").encode() + self._private_key = load_ssh_private_key(key_data, password=pw) + except Exception as exc: + raise VerifierError( + f"Could not load SSH private key at {key_path}: {exc}. " + "If the key is passphrase-protected and no terminal is " + "available, use ssh-agent, an unencrypted key, or pass " + "--gpu-proof-key=PATH to a usable key." + ) from exc self._public_key = self._private_key.public_key() @@ -167,12 +204,4 @@ def key_fingerprint(self) -> str: return _public_key_fingerprint(self._public_key) def algorithm(self) -> str: - if isinstance(self._private_key, Ed25519PrivateKey): - return "ed25519" - if isinstance(self._private_key, EllipticCurvePrivateKey): - return "ecdsa-sha256" - if isinstance(self._private_key, RSAPrivateKey): - return "rsa-pss-sha256" - raise VerifierError( - f"Unsupported private key type: {type(self._private_key).__name__}" - ) + return public_key_algorithm(self._public_key) diff --git a/src/pytest_gpu_proof/verify.py b/src/pytest_gpu_proof/verify.py index 03f2159..c0f4287 100644 --- a/src/pytest_gpu_proof/verify.py +++ b/src/pytest_gpu_proof/verify.py @@ -1,18 +1,6 @@ -""" -Standalone receipt verifier. - -Checks: - 1. Signature — fetches signer's public keys from github.com/{username}.keys - (unsigned receipts are rejected unless allow_unsigned is set) - 2. Fingerprint — recomputes and compares digest - 3. Commit SHA — compares against current repo state - 4. Test outcomes — all tests in receipt must have passed; skipped marked - tests are rejected unless allow_skipped is set, or an expected-skips - baseline is given and the receipt's skip set matches it EXACTLY - 5. GPU info — optionally require environment.gpu_info (require_gpu) - 6. Freshness — receipt must not be older than max_age_days - 7. Dirty policy — reject dirty-tree receipts if policy requires clean -""" +"""Strict, CPU-only verification of signed pytest GPU receipts.""" + +from __future__ import annotations import base64 import datetime @@ -22,58 +10,148 @@ from pathlib import Path from typing import Optional -from .signers.ed25519 import verify_with_github_keys +from .fingerprint import FingerprintError, recompute_fingerprint +from .gitutils import GitError, get_commit_sha, is_dirty, require_repository from .signers.base import VerifierError as _VerifierError +from .signers.ed25519 import ( + _public_key_fingerprint, + find_verifying_github_key, + public_key_algorithm, +) class VerificationError(Exception): pass +_POLICY_KEYS = { + "allow_carried", + "allow_dirty", + "allowed_key_fingerprints", + "allowed_signers", + "carried_max_age_days", + "max_age_days", + "min_schema", + "require_mode", + "required_fingerprint_extra_paths", + "required_fingerprint_excluded_paths", + "required_fingerprint_paths", + "required_shard_fingerprints", + "required_test_manifest", + "signer_mode", +} + + def _load_policy(policy_path: Optional[str]) -> dict: if not policy_path: return {} - text = Path(policy_path).read_text() - if policy_path.endswith(".json"): - return json.loads(text) or {} + path = Path(policy_path) try: - import yaml # type: ignore - except ImportError: - raise VerificationError( - f"Policy file {policy_path!r} is YAML but PyYAML is not installed. " - "Install it with 'pip install pyyaml', or use a .json policy file." - ) - return yaml.safe_load(text) or {} + text = path.read_text() + if path.suffix.lower() == ".json": + policy = json.loads(text) + else: + try: + import yaml # type: ignore + except ImportError as exc: + raise VerificationError( + "YAML policy requires the 'yaml' extra: pip install " + "pytest-gpu-proof[yaml]" + ) from exc + policy = yaml.safe_load(text) + except VerificationError: + raise + except (OSError, json.JSONDecodeError, ValueError) as exc: + raise VerificationError(f"cannot read policy {policy_path!r}: {exc}") from exc + if policy is None: + return {} + if not isinstance(policy, dict): + raise VerificationError("policy must be a JSON/YAML object") + unknown = sorted(set(policy) - _POLICY_KEYS) + if unknown: + raise VerificationError(f"unknown policy field(s): {', '.join(unknown)}") + + def string_list(name: str) -> None: + value = policy.get(name) + if value is not None and ( + not isinstance(value, list) + or not all(isinstance(item, str) and item for item in value) + ): + raise VerificationError(f"policy field {name!r} must be a list of strings") + + for name in ( + "allowed_key_fingerprints", + "allowed_signers", + "required_fingerprint_excluded_paths", + "required_fingerprint_extra_paths", + "required_fingerprint_paths", + ): + string_list(name) + for name in ("allow_carried", "allow_dirty"): + if name in policy and type(policy[name]) is not bool: + raise VerificationError(f"policy field {name!r} must be a boolean") + for name in ("carried_max_age_days", "max_age_days"): + if name in policy and ( + type(policy[name]) is not int or policy[name] < 0 + ): + raise VerificationError( + f"policy field {name!r} must be a non-negative integer" + ) + if "min_schema" in policy and policy["min_schema"] not in {1, 2, 3}: + raise VerificationError("policy field 'min_schema' must be 1, 2, or 3") + if "require_mode" in policy and policy["require_mode"] not in { + "local", + "ci-gpu", + }: + raise VerificationError("policy field 'require_mode' must be 'local' or 'ci-gpu'") + if "required_test_manifest" in policy and not isinstance( + policy["required_test_manifest"], str + ): + raise VerificationError("policy field 'required_test_manifest' must be a path string") + shard_policy = policy.get("required_shard_fingerprints") + if shard_policy is not None: + if not isinstance(shard_policy, dict): + raise VerificationError("required_shard_fingerprints must be an object") + for name, scope in shard_policy.items(): + if not isinstance(name, str) or not isinstance(scope, dict): + raise VerificationError("each required shard scope must be an object") + unknown_scope = set(scope) - {"paths", "extra_paths", "excluded_paths"} + if unknown_scope: + raise VerificationError(f"shard {name!r} has unknown policy fields") + for field in ("paths", "extra_paths", "excluded_paths"): + value = scope.get(field, []) + if not isinstance(value, list) or not all( + isinstance(item, str) and item for item in value + ): + raise VerificationError( + f"shard {name!r} field {field!r} must be a list of strings" + ) + mode = policy.get("signer_mode", "open") + if mode not in {"open", "restricted"}: + raise VerificationError("signer_mode must be 'open' or 'restricted'") + if mode == "restricted" and not ( + policy.get("allowed_signers") or policy.get("allowed_key_fingerprints") + ): + raise VerificationError("restricted signer policy has no allowlist") + return policy -def _load_expected_skips(path: str) -> set: - """Baseline file: one node ID per line; blank lines and '#' comments ignored.""" - lines = Path(path).read_text().splitlines() - entries = set() - for line in lines: - line = line.strip() - if line and not line.startswith("#"): - entries.add(line) - return entries +def _load_node_ids(path: Path) -> set[str]: + try: + lines = path.read_text().splitlines() + except OSError as exc: + raise VerificationError(f"cannot read node-id manifest {path}: {exc}") from exc + return { + line.strip() + for line in lines + if line.strip() and not line.lstrip().startswith("#") + } def _receipt_payload_without_sig(receipt: dict) -> bytes: from .receipt import canonicalize - payload = {k: v for k, v in receipt.items() if k != "signature"} - return canonicalize(payload) - -def _git(repo_root: str, *args: str) -> Optional[str]: - try: - result = subprocess.run( - ["git", "-C", repo_root, *args], - capture_output=True, - text=True, - check=True, - ) - return result.stdout.strip() or None - except (subprocess.CalledProcessError, FileNotFoundError): - return None + return canonicalize({key: value for key, value in receipt.items() if key != "signature"}) def _is_ancestor(repo_root: str, ancestor_sha: str, descendant_sha: str) -> bool: @@ -84,9 +162,141 @@ def _is_ancestor(repo_root: str, ancestor_sha: str, descendant_sha: str) -> bool text=True, check=True, ) - return True except (subprocess.CalledProcessError, FileNotFoundError): return False + return True + + +def _require_dict(container: dict, key: str) -> dict: + value = container.get(key) + if not isinstance(value, dict): + raise VerificationError(f"receipt field {key!r} must be an object") + return value + + +def _parse_time(value, field: str) -> datetime.datetime: + if not isinstance(value, str): + raise VerificationError(f"{field} is missing or is not a UTC timestamp") + try: + return datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace( + tzinfo=datetime.UTC + ) + except ValueError as exc: + raise VerificationError(f"{field} is not a valid UTC timestamp: {value!r}") from exc + + +def _validate_structure(receipt: dict, schema: str) -> tuple[dict, dict, list]: + repo = _require_dict(receipt, "repo") + session = _require_dict(receipt, "session") + _require_dict(receipt, "fingerprint") + _require_dict(receipt, "environment") + if receipt.get("mode") not in {"local", "ci-gpu"}: + raise VerificationError("receipt mode is missing or invalid") + if not isinstance(repo.get("commit_sha"), str) or not repo["commit_sha"]: + raise VerificationError("receipt repo.commit_sha is missing") + tests = receipt.get("tests") + if not isinstance(tests, list) or not tests: + raise VerificationError("receipt contains no test results") + node_ids = [] + for index, test in enumerate(tests): + if not isinstance(test, dict) or not isinstance(test.get("node_id"), str): + raise VerificationError(f"tests[{index}] has no valid node_id") + node_ids.append(test["node_id"]) + if test.get("outcome") not in {"passed", "failed", "error", "skipped"}: + raise VerificationError(f"tests[{index}] has an invalid outcome") + checks = test.get("checks", []) + if not isinstance(checks, list): + raise VerificationError(f"tests[{index}].checks must be a list") + for check_index, check in enumerate(checks): + if not isinstance(check, dict) or check.get("outcome") not in { + "passed", + "failed", + "error", + }: + raise VerificationError( + f"tests[{index}].checks[{check_index}] is invalid" + ) + if len(node_ids) != len(set(node_ids)): + raise VerificationError("receipt contains duplicate test node IDs") + recorded = session.get("node_ids") + if not isinstance(recorded, list) or recorded != node_ids: + raise VerificationError("session.node_ids does not exactly match tests[]") + started = _parse_time(session.get("started_at"), "session.started_at") + ended = _parse_time(session.get("ended_at"), "session.ended_at") + if ended < started: + raise VerificationError("session.ended_at precedes session.started_at") + if schema == "3" and session.get("outcome") not in {"passed", "failed"}: + raise VerificationError("schema-3 session.outcome is missing or invalid") + return repo, session, tests + + +def _verify_signature(receipt: dict, schema: str, override: Optional[str], policy: dict): + sig = receipt.get("signature") + if not sig: + if policy.get("signer_mode", "open") == "restricted": + raise VerificationError( + "repository policy restricts signers; unsigned receipts are not acceptable" + ) + return None, None, None + if not isinstance(sig, dict) or not isinstance(sig.get("value"), str): + raise VerificationError("signature.value is missing") + try: + signature = base64.b64decode(sig["value"], validate=True) + except ValueError as exc: + raise VerificationError("signature.value is not valid base64") from exc + + if schema == "3": + signer = _require_dict(receipt, "signer") + username = signer.get("github_user") + fingerprint = signer.get("key_fingerprint") + algorithm = signer.get("algorithm") + if not all(isinstance(value, str) and value for value in (username, fingerprint, algorithm)): + raise VerificationError("schema-3 signer identity is incomplete") + if override and override != username: + raise VerificationError( + f"--github-user {override!r} does not match the signed identity @{username}" + ) + try: + key = find_verifying_github_key( + _receipt_payload_without_sig(receipt), signature, username + ) + except _VerifierError as exc: + raise VerificationError(str(exc)) from exc + if key is None: + raise VerificationError(f"signature does not match a current GitHub key for @{username}") + if _public_key_fingerprint(key) != fingerprint: + raise VerificationError("signed key_fingerprint does not match the verifying key") + if public_key_algorithm(key) != algorithm: + raise VerificationError("signed algorithm does not match the verifying key") + else: + username = override or sig.get("signer") or receipt.get("repo", {}).get("github_username") + if not isinstance(username, str) or not username: + raise VerificationError("cannot determine legacy receipt signer") + try: + key = find_verifying_github_key( + _receipt_payload_without_sig(receipt), signature, username + ) + except _VerifierError as exc: + raise VerificationError(str(exc)) from exc + if key is None: + raise VerificationError(f"signature does not match a current GitHub key for @{username}") + # The policy-checked fingerprint must come from the key that actually + # verified, never from the unsigned envelope (which anyone can edit). + fingerprint = _public_key_fingerprint(key) + asserted = sig.get("key_fingerprint") + if isinstance(asserted, str) and asserted and asserted != fingerprint: + raise VerificationError( + "legacy signature.key_fingerprint does not match the verifying key" + ) + + if policy.get("signer_mode", "open") == "restricted": + users = set(policy.get("allowed_signers", [])) + fingerprints = set(policy.get("allowed_key_fingerprints", [])) + if users and username not in users: + raise VerificationError(f"signer @{username} is not allowed by repository policy") + if fingerprints and fingerprint not in fingerprints: + raise VerificationError("signing key is not allowed by repository policy") + return username, fingerprint, schema def verify_receipt( @@ -112,10 +322,10 @@ def verify_receipt( require_gpu=require_gpu, expected_skips_path=expected_skips_path, ) - return True - except VerificationError as e: - print(f"[gpu-proof] FAIL: {e}", file=sys.stderr) + except VerificationError as exc: + print(f"[gpu-proof] FAIL: {exc}", file=sys.stderr) return False + return True def _verify( @@ -131,282 +341,208 @@ def _verify( ): from .config import load_toml_defaults - toml_cfg = load_toml_defaults(repo_root) - - # --- load receipt --- - receipt_text = Path(receipt_path).read_text() - receipt = json.loads(receipt_text) - - schema = receipt.get("schema_version") - if schema not in ("1", "2"): - raise VerificationError(f"Unknown schema_version: {schema!r}") + root = str(Path(repo_root).resolve()) + try: + require_repository(root) + current_sha = get_commit_sha(root, required=True) + except GitError as exc: + raise VerificationError(str(exc)) from exc + try: + receipt = json.loads(Path(receipt_path).read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise VerificationError(f"cannot read receipt {receipt_path!r}: {exc}") from exc + if not isinstance(receipt, dict): + raise VerificationError("receipt must be a JSON object") + schema = str(receipt.get("schema_version")) + if schema not in {"1", "2", "3"}: + raise VerificationError(f"unknown schema_version: {schema!r}") if schema == "1" and "shards" in receipt: + raise VerificationError("schema-1 receipts cannot contain shards") + + policy = _load_policy(policy_path) + min_schema = policy.get("min_schema") + if min_schema is not None and int(schema) < min_schema: raise VerificationError( - "schema '1' receipts must not carry a shards block (sharded receipts " - "are schema '2')" + f"receipt schema {schema} is below the policy minimum of {min_schema}" ) - - sig_block = receipt.get("signature") - if not sig_block: + toml = load_toml_defaults(root) + repo, session, tests = _validate_structure(receipt, schema) + signer = _verify_signature(receipt, schema, github_user_override, policy) + if not receipt.get("signature"): if not allow_unsigned: - raise VerificationError( - "Receipt is UNSIGNED (no signature block). Unsigned receipts prove " - "nothing about who ran the tests. Pass --allow-unsigned only if you " - "explicitly accept that." - ) - print( - "[gpu-proof] WARNING: receipt is UNSIGNED and --allow-unsigned was passed.\n" - "[gpu-proof] WARNING: signature verification SKIPPED — this receipt proves\n" - "[gpu-proof] WARNING: nothing about who ran the tests or on what machine." - ) + raise VerificationError("receipt is unsigned") + print("[gpu-proof] WARNING: accepting an unsigned receipt") else: - sig_b64 = sig_block.get("value", "") - if not sig_b64: - raise VerificationError("Signature value is missing") + print(f"[gpu-proof] Signature valid (signer: @{signer[0]})") - try: - signature = base64.b64decode(sig_b64) - except Exception: - raise VerificationError("Signature value is not valid base64") - - # --- verify signature via GitHub public keys --- - github_username = ( - github_user_override - or sig_block.get("signer") - or receipt.get("repo", {}).get("github_username") - ) - if not github_username: - raise VerificationError( - "Cannot determine GitHub username. Pass --github-user=USERNAME." - ) - - payload_bytes = _receipt_payload_without_sig(receipt) - - print(f"[gpu-proof] Fetching public keys for @{github_username} …") - try: - ok = verify_with_github_keys(payload_bytes, signature, github_username) - except _VerifierError as e: - raise VerificationError(str(e)) - - if not ok: - raise VerificationError( - f"Signature does not match any SSH key registered by @{github_username} on GitHub" - ) - print(f"[gpu-proof] Signature valid (signer: @{github_username})") - - # --- recompute fingerprint --- - repo = receipt.get("repo", {}) - stored_fp = receipt.get("fingerprint", {}) - fp_paths = stored_fp.get("included_paths", ["src", "tests"]) - - from .fingerprint import compute_fingerprint # noqa: PLC0415 (local import ok here) - - current_fp = compute_fingerprint(fp_paths, root=repo_root) - stored_digest = stored_fp.get("digest") - if not stored_digest: - raise VerificationError("Receipt fingerprint block is missing its digest") - if current_fp["digest"] != stored_digest: + required_mode = policy.get("require_mode") + if required_mode is not None and receipt.get("mode") != required_mode: raise VerificationError( - f"Fingerprint mismatch: stored={stored_digest[:12]}… " - f"current={current_fp['digest'][:12]}…\n" - "The code under src/ or tests/ has changed since the receipt was generated." + f"receipt mode {receipt.get('mode')!r} does not match required mode {required_mode!r}" ) - print(f"[gpu-proof] Fingerprint OK ({current_fp['digest'][:12]}…)") - # --- commit SHA check --- - current_sha = _git(repo_root, "rev-parse", "HEAD") - stored_sha = repo.get("commit_sha") + stored_fp = receipt["fingerprint"] if ( - current_sha - and stored_sha - and current_sha != stored_sha - and not _is_ancestor(repo_root, stored_sha, current_sha) + not isinstance(stored_fp.get("digest"), str) + or type(stored_fp.get("file_count")) is not int + or stored_fp["file_count"] <= 0 ): - raise VerificationError( - f"Commit SHA mismatch: receipt={stored_sha[:12]} current={current_sha[:12]}" - ) - if current_sha and stored_sha and current_sha != stored_sha: - print( - f"[gpu-proof] Commit SHA OK ({stored_sha[:12]}… ancestor of {current_sha[:12]}…)" - ) - elif stored_sha: - print(f"[gpu-proof] Commit SHA OK ({stored_sha[:12]}…)") - - # --- dirty repo policy --- - policy = _load_policy(policy_path) - allow_dirty = policy.get("allow_dirty", True) - if repo.get("dirty") and not allow_dirty: - raise VerificationError( - "Receipt was generated from a dirty repository and policy requires a clean tree" - ) - - # --- schema 2: per-shard fingerprints + carried-shard policy --- - if schema == "2": - shards = receipt.get("shards") - if not shards or not isinstance(shards, list): - raise VerificationError("schema '2' receipt has no shards block") - test_ids = {t.get("node_id") for t in receipt.get("tests", [])} - claimed: set = set() - for shard in shards: - name = shard.get("name") or "" - ids = set(shard.get("node_ids", [])) - overlap = claimed & ids - if overlap: - raise VerificationError( - f"shard {name!r} re-claims node id(s) already claimed by an " - f"earlier shard (e.g. {sorted(overlap)[0]!r})" - ) - claimed |= ids - # Each shard's NARROW fingerprint must recompute clean at the - # current tree — for carried shards this is exactly the soundness - # condition: the inputs that shard proved are unchanged. - sfp = shard.get("fingerprint", {}) - sdigest = sfp.get("digest") - spaths = sfp.get("included_paths") - if not sdigest or not spaths: - raise VerificationError(f"shard {name!r} has no fingerprint") - snow = compute_fingerprint(spaths, root=repo_root) - if snow["digest"] != sdigest: - raise VerificationError( - f"shard {name!r} fingerprint mismatch: stored={sdigest[:12]}… " - f"current={snow['digest'][:12]}… — its inputs changed; " - f"re-run that shard." - ) - carried = shard.get("carried") - if carried: - if not policy.get("allow_carried", False): - raise VerificationError( - f"shard {name!r} is CARRIED from an earlier receipt and " - f"the policy does not set allow_carried: true. Carried " - f"shards attest a PRIOR run whose inputs are unchanged — " - f"opt in explicitly or re-run the shard." - ) - carried_max = int(policy.get("carried_max_age_days", 30)) - orig_end = carried.get("original_ended_at") - if not orig_end: - raise VerificationError( - f"carried shard {name!r} has no original_ended_at") - ended = datetime.datetime.strptime( - orig_end, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=datetime.UTC) - age = (datetime.datetime.now(datetime.UTC) - ended).days - if age > carried_max: - raise VerificationError( - f"carried shard {name!r} is {age} day(s) old " - f"(carried_max_age_days: {carried_max}) — re-run it." - ) - print(f"[gpu-proof] shard {name!r}: CARRIED " - f"(from {str(carried.get('original_commit_sha'))[:12]}, " - f"{age}d old, fingerprint clean) — policy allows") - else: - print(f"[gpu-proof] shard {name!r}: fingerprint OK " - f"({sdigest[:12]}…, {len(ids)} test(s))") - if claimed != test_ids: - orphans = sorted(test_ids - claimed)[:3] - unmatched = sorted(claimed - test_ids)[:3] - raise VerificationError( - f"shard membership does not partition tests[]: " - f"unclaimed={orphans} claimed-but-absent={unmatched}" - ) + raise VerificationError("receipt fingerprint is empty or missing its digest") + required_paths = policy.get("required_fingerprint_paths") + if required_paths is not None and sorted(stored_fp.get("included_paths", [])) != sorted(required_paths): + raise VerificationError("receipt fingerprint paths do not match repository policy") + required_extras = policy.get("required_fingerprint_extra_paths") + if required_extras is not None and sorted(stored_fp.get("extra_paths", [])) != sorted(required_extras): + raise VerificationError("receipt fingerprint extra paths do not match repository policy") + required_excluded = policy.get("required_fingerprint_excluded_paths") + if required_excluded is not None and sorted( + stored_fp.get("excluded_paths", []) + ) != sorted(required_excluded): + raise VerificationError("receipt fingerprint exclusions do not match repository policy") + try: + current_fp = recompute_fingerprint(stored_fp, root) + except FingerprintError as exc: + raise VerificationError(str(exc)) from exc + if current_fp["digest"] != stored_fp["digest"] or current_fp["file_count"] != stored_fp["file_count"]: + raise VerificationError("source fingerprint does not match the checked-out tree") + print(f"[gpu-proof] Fingerprint OK ({current_fp['digest'][:12]}…)") - # --- test outcomes --- - tests = receipt.get("tests", []) - if not tests: - raise VerificationError("Receipt contains no test results") + stored_sha = repo["commit_sha"] + if stored_sha != current_sha and not _is_ancestor(root, stored_sha, current_sha): + raise VerificationError("receipt commit is not the current commit or an ancestor") + print(f"[gpu-proof] Commit ancestry OK ({stored_sha[:12]}…)") - failed = [ - t["node_id"] for t in tests if t.get("outcome") not in ("passed", "skipped") - ] - if failed: - raise VerificationError( - f"{len(failed)} test(s) did not pass: {', '.join(failed)}" + allow_dirty = bool(policy.get("allow_dirty", schema in {"1", "2"})) + # The receipt under verification is expected to sit in the tree (untracked + # right after a run, or tracked-and-committed later); it must not count as + # dirt, mirroring the recording-side exclusion. + try: + receipt_rel = ( + Path(receipt_path).resolve(strict=False).relative_to(Path(root).resolve()).as_posix() ) - skipped = [t["node_id"] for t in tests if t.get("outcome") == "skipped"] - - # Expected-skips baseline: the receipt's skip set must match EXACTLY. - # Stricter than --allow-skipped (which accepts ANY skips): new skips fail, - # and a baselined test that now runs flags the baseline as stale. - expected_skips = None + except ValueError: + receipt_rel = None + dirty_exclusions = [receipt_rel] if receipt_rel else [] + try: + current_dirty = is_dirty(root, required=True, exclude_paths=dirty_exclusions) + except GitError as exc: + raise VerificationError(str(exc)) from exc + if not allow_dirty and (repo.get("dirty") or current_dirty): + raise VerificationError("repository policy requires clean recording and verification trees") + + if schema in {"2", "3"} and "shards" in receipt: + _verify_shards(receipt, root, policy) + + failed = [test["node_id"] for test in tests if test.get("outcome") not in {"passed", "skipped"}] + if schema == "3" and session.get("outcome") != "passed": + raise VerificationError("recorded pytest session did not pass") + if failed: + raise VerificationError(f"{len(failed)} recorded test(s) did not pass") + for test in tests: + bad_checks = [check for check in test.get("checks", []) if check.get("outcome") != "passed"] + if bad_checks: + raise VerificationError(f"test {test['node_id']!r} contains a failed comparison check") + + skipped = {test["node_id"] for test in tests if test.get("outcome") == "skipped"} + if expected_skips_path is not None and allow_skipped: + raise VerificationError("--expected-skips and --allow-skipped are mutually exclusive") if expected_skips_path is not None: - if allow_skipped: - raise VerificationError( - "--expected-skips and --allow-skipped are mutually exclusive: " - "the baseline already defines exactly which skips are acceptable." - ) - expected_skips = _load_expected_skips(expected_skips_path) - elif not allow_skipped and toml_cfg.get("expected_skips"): - expected_skips = set(toml_cfg["expected_skips"]) - - if expected_skips is not None: - got = set(skipped) - unexpected = sorted(got - expected_skips) - stale = sorted(expected_skips - got) - problems = [] - if unexpected: - problems.append( - f"{len(unexpected)} skip(s) NOT in the baseline: {', '.join(unexpected)}" - ) - if stale: - problems.append( - f"{len(stale)} baseline entr(y/ies) that did NOT skip (stale " - f"baseline — update it): {', '.join(stale)}" - ) - if problems: - raise VerificationError( - "Skipped tests do not match the expected-skips baseline. " - + " | ".join(problems) - ) - print( - f"[gpu-proof] Skipped tests match the expected baseline " - f"({len(expected_skips)} pinned skip(s))" - ) - elif skipped and not allow_skipped: - raise VerificationError( - f"{len(skipped)} marked test(s) were skipped: {', '.join(skipped)}. " - "Skipped tests prove nothing; pass --allow-skipped to accept them, " - "or pin them with --expected-skips BASELINE_FILE." - ) - elif skipped: - print( - f"[gpu-proof] WARNING: {len(skipped)} skipped test(s) accepted " - "(--allow-skipped)" - ) - print(f"[gpu-proof] All {len(tests) - len(skipped)} executed test(s) passed") + expected = _load_node_ids(Path(expected_skips_path)) + elif not allow_skipped and toml.get("expected_skips"): + expected = set(toml["expected_skips"]) + else: + expected = None + if expected is not None and skipped != expected: + raise VerificationError("recorded skip set does not exactly match the expected baseline") + if skipped and expected is None and not allow_skipped: + raise VerificationError(f"{len(skipped)} marked test(s) were skipped") + + manifest = policy.get("required_test_manifest") + if manifest: + required_tests = _load_node_ids(Path(root) / manifest) + if set(session["node_ids"]) != required_tests: + raise VerificationError("recorded tests do not match the repository test manifest") - # --- gpu_info policy (modest hardening, not proof) --- if require_gpu is None: - require_gpu = bool(toml_cfg.get("require_gpu", False)) - if require_gpu: - gpu_info = (receipt.get("environment") or {}).get("gpu_info") - if not gpu_info: - raise VerificationError( - "Receipt's environment.gpu_info is missing/null but the policy " - "requires GPU info (--require-gpu). The recording machine had no " - "visible GPU (or nvidia-smi failed)." - ) - print(f"[gpu-proof] GPU info present ({gpu_info.get('name')})") - - # --- freshness --- - if max_age_days_override is not None: - max_days = max_age_days_override - elif policy.get("max_age_days") is not None: - max_days = policy["max_age_days"] - elif toml_cfg.get("max_age_days") is not None: - max_days = toml_cfg["max_age_days"] - else: - max_days = 30 - signed_at_str = receipt.get("session", {}).get("ended_at") - if signed_at_str: + require_gpu = bool(toml.get("require_gpu", False)) + if require_gpu and not (receipt.get("environment") or {}).get("gpu_info"): + raise VerificationError("repository policy requires recorded GPU information") + + max_days = ( + max_age_days_override + if max_age_days_override is not None + else policy.get("max_age_days", toml.get("max_age_days", 30)) + ) + if type(max_days) is not int or max_days < 0: + raise VerificationError("max_age_days must be a non-negative integer") + ended = _parse_time(session.get("ended_at"), "session.ended_at") + age = datetime.datetime.now(datetime.UTC) - ended + if age < datetime.timedelta(minutes=-5): + raise VerificationError("receipt timestamp is in the future") + if age > datetime.timedelta(days=max_days): + raise VerificationError(f"receipt is older than the {max_days}-day policy") + print(f"[gpu-proof] All {len(tests) - len(skipped)} executed test(s) passed") + print("[gpu-proof] Receipt verified successfully.") + + +def _verify_shards(receipt: dict, root: str, policy: dict) -> None: + shards = receipt.get("shards") + if not isinstance(shards, list) or not shards: + raise VerificationError("sharded receipt has no shards") + test_ids = [test["node_id"] for test in receipt["tests"]] + claimed: list[str] = [] + names: set[str] = set() + required = policy.get("required_shard_fingerprints", {}) + for shard in shards: + if not isinstance(shard, dict) or not isinstance(shard.get("name"), str): + raise VerificationError("shard entry has no valid name") + name = shard["name"] + if name in names: + raise VerificationError(f"duplicate shard name: {name!r}") + names.add(name) + ids = shard.get("node_ids") + if not isinstance(ids, list) or len(ids) != len(set(ids)): + raise VerificationError(f"shard {name!r} has invalid node_ids") + if set(claimed) & set(ids): + raise VerificationError(f"shard {name!r} overlaps another shard") + claimed.extend(ids) + fingerprint = shard.get("fingerprint") + if not isinstance(fingerprint, dict): + raise VerificationError(f"shard {name!r} has no fingerprint") + if name in required: + expected = required[name] + if "paths" in expected and sorted( + fingerprint.get("included_paths", []) + ) != sorted(expected["paths"]): + raise VerificationError(f"shard {name!r} paths do not match policy") + if "extra_paths" in expected and sorted( + fingerprint.get("extra_paths", []) + ) != sorted(expected["extra_paths"]): + raise VerificationError(f"shard {name!r} extra paths do not match policy") + if "excluded_paths" in expected and sorted( + fingerprint.get("excluded_paths", []) + ) != sorted(expected["excluded_paths"]): + raise VerificationError(f"shard {name!r} exclusions do not match policy") try: - signed_at = datetime.datetime.strptime( - signed_at_str, "%Y-%m-%dT%H:%M:%SZ" - ).replace(tzinfo=datetime.UTC) - age = (datetime.datetime.now(datetime.UTC) - signed_at).days - if age > max_days: - raise VerificationError( - f"Receipt is {age} days old; policy allows max {max_days} days" - ) - print(f"[gpu-proof] Freshness OK (age: {age} day(s), limit: {max_days})") - except ValueError: - pass - - print(f"[gpu-proof] Receipt verified successfully.") + current = recompute_fingerprint(fingerprint, root) + except FingerprintError as exc: + raise VerificationError(f"shard {name!r}: {exc}") from exc + if current["digest"] != fingerprint.get("digest"): + raise VerificationError(f"shard {name!r} fingerprint does not match") + carried = shard.get("carried") + if carried: + if not policy.get("allow_carried", False): + raise VerificationError(f"shard {name!r} is carried but policy rejects carry-forward") + if not isinstance(carried, dict): + raise VerificationError(f"shard {name!r} carried metadata is invalid") + original = _parse_time(carried.get("original_ended_at"), f"shard {name}.original_ended_at") + age = datetime.datetime.now(datetime.UTC) - original + limit = policy.get("carried_max_age_days", 30) + if type(limit) is not int or limit < 0: + raise VerificationError("carried_max_age_days must be a non-negative integer") + if age < datetime.timedelta(minutes=-5) or age > datetime.timedelta(days=limit): + raise VerificationError(f"carried shard {name!r} is outside its age policy") + if claimed != test_ids: + raise VerificationError("shard membership does not exactly partition tests[]") + if required and set(required) != names: + raise VerificationError("receipt shard set does not match repository policy") diff --git a/tests/conftest.py b/tests/conftest.py index 5ced2aa..9a5c427 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import json import os import tempfile +import subprocess import pytest @@ -35,6 +36,25 @@ def ed25519_keypair(): return private_key, public_key +@pytest.fixture(autouse=True) +def _pytester_git_repo(request): + """Receipt-emission integration tests run in a real, committed git tree.""" + if "pytester" not in request.fixturenames: + return + pytester = request.getfixturevalue("pytester") + subprocess.run(["git", "init", "-q"], cwd=pytester.path, check=True) + subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=pytester.path, check=True) + subprocess.run(["git", "config", "user.name", "Test"], cwd=pytester.path, check=True) + subprocess.run( + ["git", "remote", "add", "origin", "git@github.com:testuser/example.git"], + cwd=pytester.path, + check=True, + ) + (pytester.path / ".gitkeep").write_text("") + subprocess.run(["git", "add", ".gitkeep"], cwd=pytester.path, check=True) + subprocess.run(["git", "commit", "-q", "-m", "init"], cwd=pytester.path, check=True) + + @pytest.fixture def tmp_git_repo(tmp_path): """Minimal git repo with a couple of tracked files.""" @@ -61,4 +81,11 @@ def tmp_git_repo(tmp_path): ["git", "commit", "-m", "init"], cwd=tmp_path, check=True, capture_output=True, ) + subprocess.run( + ["git", "remote", "add", "origin", "git@github.com:testuser/example.git"], + cwd=tmp_path, check=True, capture_output=True, + ) + (tmp_path / ".git" / "info" / "exclude").write_text( + "*.json\nid_*\nexpected_skips.txt\npyproject.toml\n" + ) return tmp_path diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..8f22468 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,82 @@ +import runpy +import sys +from types import SimpleNamespace + +import pytest + +from pytest_gpu_proof import cli +from pytest_gpu_proof.merge import MergeError + + +def test_verify_cli_forwards_options(monkeypatch, tmp_path): + seen = {} + + def verify_receipt(**kwargs): + seen.update(kwargs) + return True + + monkeypatch.setattr("pytest_gpu_proof.verify.verify_receipt", verify_receipt) + monkeypatch.setattr( + sys, + "argv", + [ + "gpu-proof", "verify", "--receipt", "r.json", "--policy", "p.json", + "--repo", str(tmp_path), "--github-user", "alice", "--max-age-days", "4", + "--allow-unsigned", "--allow-skipped", "--require-gpu", + ], + ) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 0 + assert seen["github_user_override"] == "alice" + assert seen["max_age_days"] == 4 + assert seen["allow_unsigned"] and seen["allow_skipped"] and seen["require_gpu"] + + +def test_verify_cli_failure(monkeypatch): + monkeypatch.setattr("pytest_gpu_proof.verify.verify_receipt", lambda **_: False) + monkeypatch.setattr(sys, "argv", ["gpu-proof", "verify", "--receipt", "r.json"]) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 1 + + +def test_merge_cli_success_and_error(monkeypatch, capsys): + monkeypatch.setattr( + "pytest_gpu_proof.merge.merge_receipts", + lambda *args, **kwargs: {"tests": [{}, {}], "session": {"shards": [{}, {}]}}, + ) + monkeypatch.setattr( + sys, + "argv", + ["gpu-proof", "merge", "a.json", "b.json", "--out", "out.json", "--unsigned"], + ) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 0 + assert "2 shard(s), 2 tests" in capsys.readouterr().out + + def fail(*args, **kwargs): + raise MergeError("bad shards") + + monkeypatch.setattr("pytest_gpu_proof.merge.merge_receipts", fail) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 1 + assert "bad shards" in capsys.readouterr().err + + +def test_module_entrypoint(monkeypatch): + monkeypatch.setattr("pytest_gpu_proof.cli.main", lambda: (_ for _ in ()).throw(SystemExit(7))) + with pytest.raises(SystemExit) as exc: + runpy.run_module("pytest_gpu_proof.__main__", run_name="__main__") + assert exc.value.code == 7 + + +@pytest.mark.filterwarnings("ignore:.*found in sys.modules.*:RuntimeWarning") +def test_cli_file_entrypoint(monkeypatch): + monkeypatch.setattr("pytest_gpu_proof.verify.verify_receipt", lambda **kwargs: True) + monkeypatch.setattr(sys, "argv", ["gpu-proof", "verify", "--receipt", "r.json"]) + with pytest.raises(SystemExit) as exc: + runpy.run_module("pytest_gpu_proof.cli", run_name="__main__") + assert exc.value.code == 0 diff --git a/tests/test_compare.py b/tests/test_compare.py index c21a3a1..8ae9523 100644 --- a/tests/test_compare.py +++ b/tests/test_compare.py @@ -1,5 +1,9 @@ +import builtins + import pytest +np = pytest.importorskip("numpy") + from pytest_gpu_proof.compare import default_compare, run_comparison @@ -16,6 +20,45 @@ def test_default_compare_unequal_scalars(): default_compare(1, 2) +def test_default_compare_numpy_integer_arrays(): + default_compare(np.array([1, 2]), np.array([1, 2])) + with pytest.raises(AssertionError, match="not exactly equal"): + default_compare(np.array([1, 2]), np.array([1, 3])) + + +def test_default_compare_numpy_float_arrays_and_nan(): + default_compare(np.array([1.0, np.nan]), np.array([1.0, np.nan])) + with pytest.raises(AssertionError, match="max diff"): + default_compare(np.array([1.0, 2.0]), np.array([1.0, 3.0])) + + +def test_default_compare_numpy_shape_mismatch(): + with pytest.raises(AssertionError, match="Shapes differ"): + default_compare(np.array([1.0, 2.0]), np.array([[1.0, 2.0]])) + + +def test_default_compare_falls_back_without_numpy(monkeypatch): + real_import = builtins.__import__ + + def without_numpy(name, *args, **kwargs): + if name == "numpy": + raise ImportError("not installed") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", without_numpy) + default_compare("same", "same") + with pytest.raises(AssertionError, match="Values not equal"): + default_compare("left", "right") + + +def test_default_compare_falls_back_for_non_array_values(monkeypatch): + def reject(_value): + raise TypeError("cannot convert") + + monkeypatch.setattr(np, "asarray", reject) + default_compare("same", "same") + + def test_run_comparison_passed(): outcome, ref, cand, err = run_comparison( lambda x: x * 2, @@ -46,6 +89,17 @@ def strict(a, b): assert outcome == "passed" +def test_run_comparison_custom_compare_error(): + def broken_compare(_reference, _candidate): + raise ValueError("comparison broke") + + outcome, _, _, error = run_comparison( + lambda: 1, lambda: 1, (), {}, compare_fn=broken_compare + ) + assert outcome == "error" + assert "Comparator raised ValueError" in error + + def test_run_comparison_reference_raises(): def boom(*args): raise RuntimeError("boom") diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..a0a3579 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,106 @@ +from types import SimpleNamespace + +from pytest_gpu_proof.config import GpuProofConfig, load_config, load_toml_defaults + + +class FakeConfig: + def __init__(self, rootpath, options=None, args=("-q",)): + self.rootpath = rootpath + self.options = options or {} + self.invocation_params = SimpleNamespace(args=args) + + def getoption(self, name): + if name not in self.options: + raise ValueError(name) + return self.options[name] + + +def test_dataclass_defaults_are_safe(): + first = GpuProofConfig() + second = GpuProofConfig() + first.fingerprint_paths.append("x") + assert second.fingerprint_paths == ["."] + assert second.best_effort is False + + +def test_load_toml_defaults_absent_invalid_and_non_table(tmp_path): + assert load_toml_defaults(tmp_path) == {} + (tmp_path / "pyproject.toml").write_text("not = [valid") + assert load_toml_defaults(tmp_path) == {} + (tmp_path / "pyproject.toml").write_text('[tool]\ngpu_proof = "bad"\n') + assert load_toml_defaults(tmp_path) == {} + + +def test_load_config_toml_lists_and_cli_precedence(tmp_path): + (tmp_path / "pyproject.toml").write_text( + """ +[tool.gpu_proof] +mode = "ci" +output = "from-toml.json" +fingerprint_paths = ["src", "tests"] +fingerprint_extra_paths = "generated/a,generated/b" +shard_name = "core" +shard_fingerprint_paths = ["src"] +shard_fingerprint_extra_paths = ["generated/a"] +fail_on_skip = true +best_effort = true +max_age_days = 7 +require_gpu = true +""" + ) + config = load_config( + FakeConfig( + tmp_path, + { + "--gpu-proof-enable": True, + "--gpu-proof-mode": "local", + "--gpu-proof-out": None, + "--gpu-proof-fingerprint-paths": "src, docs,", + "--gpu-proof-shard-fingerprint-paths": None, + }, + args=("tests", "-q"), + ) + ) + assert config.enabled is True + assert config.mode == "local" + assert config.output == "from-toml.json" + assert config.fingerprint_paths == ["src", "docs"] + assert config.fingerprint_extra_paths == ["generated/a", "generated/b"] + assert config.fingerprint_excluded_paths == ["gpu-proof.json"] + assert config.shard_fingerprint_paths == ["src"] + assert config.shard_fingerprint_extra_paths == ["generated/a"] + assert config.fail_on_skip and config.best_effort and config.require_gpu + assert config.max_age_days == 7 + assert config.invocation_args == ["tests", "-q"] + + +def test_load_config_defaults_and_string_shard_paths(tmp_path): + config = load_config( + FakeConfig( + tmp_path, + { + "--gpu-proof-fingerprint-paths": None, + "--gpu-proof-shard-fingerprint-paths": "src,tests", + "--gpu-proof-fail-on-skip": False, + "--gpu-proof-best-effort": False, + }, + ) + ) + assert config.fingerprint_paths == ["."] + assert config.shard_fingerprint_paths == ["src", "tests"] + assert config.shard_name is None + + +def test_load_config_list_fingerprint_paths(tmp_path): + (tmp_path / "pyproject.toml").write_text( + '[tool.gpu_proof]\nfingerprint_paths = ["src", 7]\n' + ) + config = load_config(FakeConfig(tmp_path)) + assert config.fingerprint_paths == ["src", "7"] + + +def test_explicit_empty_fingerprint_exclusions(tmp_path): + (tmp_path / "pyproject.toml").write_text( + "[tool.gpu_proof]\nfingerprint_excluded_paths = []\n" + ) + assert load_config(FakeConfig(tmp_path)).fingerprint_excluded_paths == [] diff --git a/tests/test_fingerprint.py b/tests/test_fingerprint.py index f2e9080..e22ef3e 100644 --- a/tests/test_fingerprint.py +++ b/tests/test_fingerprint.py @@ -1,8 +1,16 @@ import os +from pathlib import Path import pytest -from pytest_gpu_proof.fingerprint import compute_fingerprint +from pytest_gpu_proof import fingerprint as fingerprint_module +from pytest_gpu_proof.fingerprint import ( + FingerprintError, + compute_fingerprint, + compute_legacy_fingerprint, + recompute_fingerprint, +) +from pytest_gpu_proof.gitutils import GitError, TrackedEntry def test_fingerprint_deterministic(tmp_git_repo): @@ -27,7 +35,7 @@ def test_fingerprint_changes_on_file_edit(tmp_git_repo): def test_fingerprint_structure(tmp_git_repo): fp = compute_fingerprint(["src", "tests"], root=str(tmp_git_repo)) - assert fp["algorithm"] == "sha256" + assert fp["algorithm"] == "sha256-manifest-v2" assert isinstance(fp["digest"], str) assert len(fp["digest"]) == 64 assert fp["file_count"] > 0 @@ -35,6 +43,167 @@ def test_fingerprint_structure(tmp_git_repo): def test_fingerprint_empty_paths(tmp_path): - fp = compute_fingerprint(["nonexistent"], root=str(tmp_path)) + with pytest.raises(FingerprintError): + compute_fingerprint(["nonexistent"], root=str(tmp_path)) + + +def test_extra_files_directories_and_symlinks(tmp_git_repo): + generated = tmp_git_repo / "generated" + generated.mkdir() + (generated / "data.bin").write_bytes(b"one") + (generated / "link").symlink_to("data.bin") + before = compute_fingerprint([], str(tmp_git_repo), extra_paths=["generated"]) + assert before["file_count"] == 2 + (generated / "data.bin").write_bytes(b"two") + after = compute_fingerprint([], str(tmp_git_repo), extra_paths=["generated"]) + assert after["digest"] != before["digest"] + + +def test_tracked_symlink_and_duplicate_scope(tmp_git_repo): + import subprocess + + (tmp_git_repo / "src" / "alias.py").symlink_to("mymodule.py") + subprocess.run(["git", "add", "src/alias.py"], cwd=tmp_git_repo, check=True) + fp = compute_fingerprint(["src", "src", ""], str(tmp_git_repo)) + assert fp["included_paths"] == ["src"] + assert fp["file_count"] == 2 + + +@pytest.mark.parametrize("path", ["missing.bin", "../escape"]) +def test_extra_path_must_exist_inside_repo(tmp_git_repo, path): + with pytest.raises(FingerprintError): + compute_fingerprint([], str(tmp_git_repo), extra_paths=[path]) + + +def test_legacy_and_recompute_dispatch(tmp_git_repo): + legacy = compute_legacy_fingerprint(["src"], str(tmp_git_repo)) + assert legacy["algorithm"] == "sha256" + assert recompute_fingerprint(legacy, str(tmp_git_repo))["digest"] == legacy["digest"] + current = compute_fingerprint(["src"], str(tmp_git_repo)) + assert recompute_fingerprint(current, str(tmp_git_repo))["digest"] == current["digest"] + + +def test_recompute_rejects_invalid_metadata(tmp_git_repo): + with pytest.raises(FingerprintError, match="included_paths"): + recompute_fingerprint({"algorithm": "sha256", "included_paths": "src"}, str(tmp_git_repo)) + with pytest.raises(FingerprintError, match="extra_paths"): + recompute_fingerprint( + {"algorithm": "sha256-manifest-v2", "included_paths": ["src"], "extra_paths": "x"}, + str(tmp_git_repo), + ) + with pytest.raises(FingerprintError, match="unsupported"): + recompute_fingerprint({"algorithm": "md5", "included_paths": ["src"]}, str(tmp_git_repo)) + + +def test_empty_scope_and_overlapping_extra(tmp_git_repo): + with pytest.raises(FingerprintError, match="scope is empty"): + compute_fingerprint([], str(tmp_git_repo)) + fp = compute_fingerprint( + ["src"], str(tmp_git_repo), extra_paths=["src/mymodule.py"] + ) + assert fp["file_count"] == 1 + + +def test_gitlink_entries_use_checked_out_or_index_commit(tmp_git_repo, monkeypatch): + sub = tmp_git_repo / "sub" + sub.mkdir() + entry = TrackedEntry("sub", "160000", "index-commit") + monkeypatch.setattr("pytest_gpu_proof.gitutils.get_commit_sha", lambda root: "head-commit") + # An empty (uninitialized) submodule dir must NOT be rev-parsed: git would + # walk up and report the parent repo's HEAD. The index commit is truth. + assert fingerprint_module._regular_entry(entry, tmp_git_repo)["commit"] == "index-commit" + # An initialized submodule (".git" present) is read from its checkout. + (sub / ".git").write_text("gitdir: ../.git/modules/sub\n") + assert fingerprint_module._regular_entry(entry, tmp_git_repo)["commit"] == "head-commit" + (sub / ".git").unlink() + sub.rmdir() + assert fingerprint_module._regular_entry(entry, tmp_git_repo)["commit"] == "index-commit" + + +def test_fingerprint_wraps_git_and_io_errors(tmp_git_repo, monkeypatch): + monkeypatch.setattr( + fingerprint_module, + "get_tracked_entries", + lambda *a, **k: (_ for _ in ()).throw(GitError("index broke")), + ) + with pytest.raises(FingerprintError, match="index broke"): + compute_fingerprint(["src"], str(tmp_git_repo)) + + monkeypatch.undo() + extra = tmp_git_repo / "generated.bin" + extra.write_bytes(b"x") + monkeypatch.setattr(Path, "read_bytes", lambda path: (_ for _ in ()).throw(OSError("read broke"))) + with pytest.raises(FingerprintError, match="cannot read"): + compute_fingerprint([], str(tmp_git_repo), extra_paths=["generated.bin"]) + + +def test_legacy_skips_non_files_and_wraps_errors(tmp_git_repo, monkeypatch): + monkeypatch.setattr(fingerprint_module, "get_tracked_files", lambda *a: ["src"]) + fp = compute_legacy_fingerprint(["src"], str(tmp_git_repo)) assert fp["file_count"] == 0 - assert isinstance(fp["digest"], str) + monkeypatch.setattr( + fingerprint_module, + "get_tracked_files", + lambda *a: (_ for _ in ()).throw(GitError("legacy index broke")), + ) + with pytest.raises(FingerprintError, match="legacy index broke"): + compute_legacy_fingerprint(["src"], str(tmp_git_repo)) + + +def test_tracked_and_legacy_read_errors(tmp_git_repo, monkeypatch): + missing = TrackedEntry("missing.py", "100644", "object") + with pytest.raises(FingerprintError, match="cannot read"): + fingerprint_module._regular_entry(missing, tmp_git_repo) + + monkeypatch.setattr(fingerprint_module, "get_tracked_files", lambda *a: ["src/mymodule.py"]) + monkeypatch.setattr(Path, "read_bytes", lambda path: (_ for _ in ()).throw(OSError("read broke"))) + with pytest.raises(FingerprintError, match="legacy fingerprint"): + compute_legacy_fingerprint(["src"], str(tmp_git_repo)) + + +def test_tracked_scope_matching_zero_files(tmp_git_repo): + with pytest.raises(FingerprintError, match="matched zero files"): + compute_fingerprint(["does-not-exist"], str(tmp_git_repo)) + + +def test_excluded_receipt_is_not_self_referential(tmp_git_repo): + import subprocess + + receipt = tmp_git_repo / "gpu-proof.json" + receipt.write_text('{"old": true}\n') + subprocess.run(["git", "add", "-f", "gpu-proof.json"], cwd=tmp_git_repo, check=True) + subprocess.run(["git", "commit", "-m", "receipt"], cwd=tmp_git_repo, check=True, capture_output=True) + before = compute_fingerprint( + ["."], str(tmp_git_repo), exclude_paths=["gpu-proof.json"] + ) + receipt.write_text('{"new": true}\n') + after = compute_fingerprint( + ["."], str(tmp_git_repo), exclude_paths=["gpu-proof.json"] + ) + assert after["digest"] == before["digest"] + assert after["excluded_paths"] == ["gpu-proof.json"] + + +def test_direct_extra_symlink_binds_link_target(tmp_git_repo): + (tmp_git_repo / "one").write_text("same") + (tmp_git_repo / "two").write_text("same") + link = tmp_git_repo / "generated-link" + link.symlink_to("one") + before = compute_fingerprint([], str(tmp_git_repo), extra_paths=["generated-link"]) + link.unlink() + link.symlink_to("two") + after = compute_fingerprint([], str(tmp_git_repo), extra_paths=["generated-link"]) + assert after["digest"] != before["digest"] + + +def test_recompute_rejects_invalid_excluded_paths(tmp_git_repo): + with pytest.raises(FingerprintError, match="excluded_paths"): + recompute_fingerprint( + { + "algorithm": "sha256-manifest-v2", + "included_paths": ["src"], + "extra_paths": [], + "excluded_paths": "gpu-proof.json", + }, + str(tmp_git_repo), + ) diff --git a/tests/test_gitutils.py b/tests/test_gitutils.py index c8469f5..5641ba6 100644 --- a/tests/test_gitutils.py +++ b/tests/test_gitutils.py @@ -9,7 +9,10 @@ import subprocess -from pytest_gpu_proof.gitutils import is_dirty +import pytest + +from pytest_gpu_proof import gitutils +from pytest_gpu_proof.gitutils import GitError, is_dirty def _git(*args, cwd): @@ -67,3 +70,92 @@ def test_is_dirty_submodule_semantics(tmp_path, monkeypatch): (parent / "sub" / "f.txt").write_text("changed") _commit_all(parent / "sub", "advance pin") assert is_dirty() is True + + +def test_git_metadata_helpers(tmp_git_repo): + assert gitutils.get_commit_sha(str(tmp_git_repo)) + assert gitutils.get_branch(str(tmp_git_repo)) == "master" + assert gitutils.get_remote_url(str(tmp_git_repo)).endswith("testuser/example.git") + assert gitutils.get_github_username(str(tmp_git_repo)) == "testuser" + assert gitutils.extract_github_username("https://example.com/nope") is None + assert gitutils.get_tracked_files(["src"], str(tmp_git_repo)) == ["src/mymodule.py"] + gitutils.require_repository(str(tmp_git_repo)) + + +def test_git_failures_are_fail_closed(tmp_path, monkeypatch): + assert gitutils.get_commit_sha(str(tmp_path)) is None + assert gitutils.is_dirty(str(tmp_path)) is False + with pytest.raises(GitError, match="not inside"): + gitutils.require_repository(str(tmp_path)) + with pytest.raises(GitError, match="git rev-parse"): + gitutils.get_commit_sha(str(tmp_path), required=True) + with pytest.raises(GitError, match="could not inspect"): + gitutils.is_dirty(str(tmp_path), required=True) + with pytest.raises(GitError, match="empty"): + gitutils.get_tracked_entries([], str(tmp_path)) + + +def test_gh_login_and_signing_key(monkeypatch, tmp_git_repo): + monkeypatch.setattr( + gitutils.subprocess, + "run", + lambda *a, **k: subprocess.CompletedProcess(a, 0, stdout="alice\n", stderr=""), + ) + assert gitutils.get_gh_cli_login() == "alice" + + def missing(*args, **kwargs): + raise FileNotFoundError + + monkeypatch.setattr(gitutils.subprocess, "run", missing) + assert gitutils.get_gh_cli_login() is None + + +def test_unmerged_index_entry_rejected(tmp_git_repo, monkeypatch): + output = b"100644 deadbeef 1\tconflicted.py\0" + monkeypatch.setattr( + gitutils.subprocess, + "run", + lambda *a, **k: subprocess.CompletedProcess(a, 0, stdout=output, stderr=b""), + ) + with pytest.raises(GitError, match="unmerged"): + gitutils.get_tracked_entries(["."], str(tmp_git_repo)) + + +def test_git_signing_key(tmp_git_repo): + _git("config", "user.signingKey", "~/keys/id_ed25519", cwd=tmp_git_repo) + assert gitutils.get_git_signing_key(str(tmp_git_repo)).endswith("keys/id_ed25519") + + +def test_dirty_check_can_ignore_only_the_receipt_artifact(tmp_git_repo): + receipt = tmp_git_repo / "gpu-proof.json" + receipt.write_text("old\n") + _git("add", "-f", "gpu-proof.json", cwd=tmp_git_repo) + _git("commit", "-m", "receipt", cwd=tmp_git_repo) + receipt.write_text("new\n") + assert is_dirty(str(tmp_git_repo)) is True + assert is_dirty( + str(tmp_git_repo), exclude_paths=["gpu-proof.json"] + ) is False + (tmp_git_repo / "src" / "mymodule.py").write_text("changed\n") + assert is_dirty( + str(tmp_git_repo), exclude_paths=["gpu-proof.json"] + ) is True + + +def test_dirty_rename_parsing(tmp_git_repo): + _git("mv", "src/mymodule.py", "src/renamed.py", cwd=tmp_git_repo) + assert is_dirty( + str(tmp_git_repo), exclude_paths=["src/mymodule.py", "src/renamed.py"] + ) is False + assert is_dirty( + str(tmp_git_repo), exclude_paths=["src/renamed.py"] + ) is True + + +def test_dirty_parser_handles_truncated_rename(monkeypatch): + monkeypatch.setattr( + gitutils.subprocess, + "run", + lambda *a, **k: subprocess.CompletedProcess(a, 0, stdout=b"R new.py\0", stderr=b""), + ) + assert is_dirty(".", exclude_paths=["new.py"]) is False diff --git a/tests/test_merge.py b/tests/test_merge.py index cf8fd5d..77f8d56 100644 --- a/tests/test_merge.py +++ b/tests/test_merge.py @@ -14,7 +14,7 @@ ) from pytest_gpu_proof.config import GpuProofConfig -from pytest_gpu_proof.merge import MergeError, merge_payloads, merge_receipts +from pytest_gpu_proof.merge import MergeError, load_receipt, merge_payloads, merge_receipts from pytest_gpu_proof.receipt import build_receipt_payload, finalize_receipt, write_receipt from pytest_gpu_proof.signers.ed25519 import SSHSigner, _verify_with_key from pytest_gpu_proof.verify import verify_receipt @@ -58,8 +58,8 @@ def _shard(tmp_path, tmp_git_repo, signer, name, results, *, def _mock_github_keys(public_key): def _fake_verify(data, signature, username): - return _verify_with_key(public_key, signature, data) - return patch("pytest_gpu_proof.verify.verify_with_github_keys", + return public_key if _verify_with_key(public_key, signature, data) else None + return patch("pytest_gpu_proof.verify.find_verifying_github_key", side_effect=_fake_verify) @@ -161,3 +161,83 @@ def test_gpu_info_survives_cpu_only_shard(tmp_path, tmp_git_repo, signer_with_ke merged = merge_payloads([json.loads(a.read_text()), json.loads(b.read_text())], ["a.json", "b.json"]) assert merged["environment"]["gpu_info"] == gpu + + +def test_load_receipt_rejects_bad_inputs(tmp_path): + with pytest.raises(MergeError, match="readable"): + load_receipt(str(tmp_path / "missing.json")) + bad = tmp_path / "bad.json" + bad.write_text("[]") + with pytest.raises(MergeError, match="no 'tests'"): + load_receipt(str(bad)) + + +def test_merge_rejects_empty_unsupported_and_zero_tests(): + with pytest.raises(MergeError, match="nothing"): + merge_payloads([], []) + with pytest.raises(MergeError, match="unsupported"): + merge_payloads([{"schema_version": "9", "tests": [{}]}], ["x"]) + base = { + "schema_version": "3", + "repo": {"commit_sha": "a", "dirty": False}, + "fingerprint": {"digest": "d"}, + "mode": "local", + "environment": {}, + "session": {"started_at": "a", "ended_at": "b", "outcome": "passed"}, + "tests": [], + } + with pytest.raises(MergeError, match="zero tests"): + merge_payloads([base], ["x"]) + + missing_stamp = dict(base) + missing_stamp["session"] = {"outcome": "passed"} + missing_stamp["tests"] = [{"node_id": "t::a", "outcome": "passed", "checks": []}] + with pytest.raises(MergeError, match="no session timestamps"): + merge_payloads([missing_stamp], ["x"]) + + +def test_merge_schema_and_mode_mismatch(tmp_path, tmp_git_repo, signer_with_key): + signer, _, _ = signer_with_key + a = json.loads(_shard(tmp_path, tmp_git_repo, signer, "a.json", [_result("t::a")]).read_text()) + b = json.loads(_shard(tmp_path, tmp_git_repo, signer, "b.json", [_result("t::b")]).read_text()) + b["schema_version"] = "2" + with pytest.raises(MergeError, match="schema_version"): + merge_payloads([a, b], ["a", "b"]) + b = dict(a) + b["tests"] = [_result("t::b")] + b["mode"] = "ci-gpu" + with pytest.raises(MergeError, match="mode"): + merge_payloads([a, b], ["a", "b"]) + + +def test_merge_failed_session_and_github_override( + tmp_path, tmp_git_repo, signer_with_key +): + signer, _, key_path = signer_with_key + a = _shard(tmp_path, tmp_git_repo, signer, "a.json", [_result("t::a")]) + b = _shard( + tmp_path, tmp_git_repo, signer, "b.json", [_result("t::b")], + mutate=lambda p: p["session"].__setitem__("outcome", "failed"), + ) + out = tmp_path / "merged.json" + merged = merge_receipts( + [str(a), str(b)], str(out), key_path=key_path, github_user="merger" + ) + assert merged["session"]["outcome"] == "failed" + assert merged["signer"]["github_user"] == "merger" + + +def test_schema1_merge_has_no_schema3_session_fields(): + base = { + "schema_version": "1", + "repo": {"commit_sha": "a", "dirty": False}, + "fingerprint": {"digest": "d"}, + "mode": "local", + "environment": {}, + "session": {"started_at": "a", "ended_at": "b"}, + "tests": [_result("t::a")], + "signature": None, + } + merged = merge_payloads([base], ["one.json"]) + assert "outcome" not in merged["session"] + assert "shards" not in merged diff --git a/tests/test_plugin_capture.py b/tests/test_plugin_capture.py index 9c85144..6dc9f07 100644 --- a/tests/test_plugin_capture.py +++ b/tests/test_plugin_capture.py @@ -5,6 +5,8 @@ import pytest +from pytest_gpu_proof import plugin as plugin_module + @pytest.fixture def plugin_testdir(pytester): @@ -305,3 +307,347 @@ def test_ok(): result.assert_outcomes(passed=1) assert (pytester.path / "gpu-proof.json").exists() assert not (pytester.path / "toml-receipt.json").exists() + + +def test_no_marked_tests_fails_closed_and_best_effort_can_opt_out(pytester): + pytester.makepyfile("def test_plain(): assert True") + result = pytester.runpytest( + "--gpu-proof-enable", "--gpu-proof-signing-backend=none" + ) + result.assert_outcomes(passed=1) + assert result.ret != 0 + assert not (pytester.path / "gpu-proof.json").exists() + + result = pytester.runpytest( + "--gpu-proof-enable", "--gpu-proof-signing-backend=none", + "--gpu-proof-best-effort", + ) + assert result.ret == 0 + + +def test_setup_failure_is_recorded_and_session_fails(pytester): + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def broken(): + raise RuntimeError("setup broke") + + @pytest.mark.gpu_proof + def test_setup(broken): + pass + """ + ) + result = pytester.runpytest( + "--gpu-proof-enable", "--gpu-proof-signing-backend=none" + ) + result.assert_outcomes(errors=1) + receipt = _read_receipt(pytester) + assert receipt["session"]["outcome"] == "failed" + assert receipt["tests"][0]["outcome"] == "failed" + assert receipt["tests"][0]["phase"] == "setup" + + +def test_teardown_failure_overrides_passed_call(pytester): + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def broken_teardown(): + yield + raise RuntimeError("teardown broke") + + @pytest.mark.gpu_proof + def test_teardown(broken_teardown): + assert True + """ + ) + result = pytester.runpytest( + "--gpu-proof-enable", "--gpu-proof-signing-backend=none" + ) + result.assert_outcomes(passed=1, errors=1) + receipt = _read_receipt(pytester) + assert receipt["tests"][0]["outcome"] == "failed" + assert receipt["tests"][0]["phase"] == "teardown" + + +def test_stale_receipt_removed_before_failed_emission(pytester): + stale = pytester.path / "gpu-proof.json" + stale.write_text('{"stale": true}') + pytester.makepyfile( + """ + import pytest + @pytest.mark.gpu_proof + def test_ok(): assert True + """ + ) + result = pytester.runpytest( + "--gpu-proof-enable", "--gpu-proof-fingerprint-extra-paths=missing.bin", + "--gpu-proof-signing-backend=none", + ) + result.assert_outcomes(passed=1) + assert result.ret != 0 + assert not stale.exists() + + +def test_emission_failure_best_effort_warns(pytester): + pytester.makepyfile( + """ + import pytest + @pytest.mark.gpu_proof + def test_ok(): assert True + """ + ) + result = pytester.runpytest( + "--gpu-proof-enable", "--gpu-proof-fingerprint-extra-paths=missing.bin", + "--gpu-proof-signing-backend=none", "--gpu-proof-best-effort", + ) + result.assert_outcomes(passed=1, warnings=1) + assert result.ret == 0 + + +def test_signed_receipt_path(pytester, tmp_path): + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat + + key = tmp_path / "id_ed25519" + key.write_bytes( + Ed25519PrivateKey.generate().private_bytes( + Encoding.PEM, PrivateFormat.OpenSSH, NoEncryption() + ) + ) + pytester.makepyfile( + """ + import pytest + @pytest.mark.gpu_proof + def test_ok(): assert True + """ + ) + result = pytester.runpytest( + "--gpu-proof-enable", f"--gpu-proof-key={key}", + "--gpu-proof-github-user=testuser", + ) + result.assert_outcomes(passed=1) + assert _read_receipt(pytester)["signature"]["value"] + + +def test_has_gpu_fallbacks(monkeypatch): + import builtins + import subprocess + from types import SimpleNamespace + + monkeypatch.setattr( + plugin_module.subprocess if hasattr(plugin_module, "subprocess") else subprocess, + "run", + lambda *a, **k: subprocess.CompletedProcess([], 0), + ) + assert plugin_module._has_gpu() is True + + monkeypatch.setattr(subprocess, "run", lambda *a, **k: subprocess.CompletedProcess([], 1)) + monkeypatch.setitem(__import__("sys").modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: True))) + assert plugin_module._has_gpu() is True + + real_import = builtins.__import__ + def no_torch(name, *args, **kwargs): + if name == "torch": + raise ImportError + return real_import(name, *args, **kwargs) + monkeypatch.setattr(builtins, "__import__", no_torch) + assert plugin_module._has_gpu() is False + + +def test_xdist_worker_is_rejected(): + class Config: + workerinput = {} + + def addinivalue_line(self, *args): + pass + + def getoption(self, name): + return True + + with pytest.raises(pytest.UsageError, match="xdist"): + plugin_module.pytest_configure(Config()) + + +def test_runtime_skip_is_recorded(pytester): + pytester.makepyfile( + """ + import pytest + @pytest.mark.gpu_proof + def test_runtime_skip(): pytest.skip("later") + """ + ) + result = pytester.runpytest( + "--gpu-proof-enable", "--gpu-proof-signing-backend=none" + ) + result.assert_outcomes(skipped=1) + assert _read_receipt(pytester)["tests"][0]["outcome"] == "skipped" + + +def test_absolute_output_path(pytester): + out = pytester.path / "nested" / "receipt.json" + pytester.makepyfile( + """ + import pytest + @pytest.mark.gpu_proof + def test_ok(): pass + """ + ) + result = pytester.runpytest( + "--gpu-proof-enable", "--gpu-proof-signing-backend=none", + f"--gpu-proof-out={out}", + ) + result.assert_outcomes(passed=1) + assert out.exists() + + +def test_has_gpu_handles_nvidia_error(monkeypatch): + import builtins + import subprocess + + monkeypatch.setattr( + subprocess, "run", lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError()) + ) + real_import = builtins.__import__ + def no_torch(name, *args, **kwargs): + if name == "torch": + raise ImportError + return real_import(name, *args, **kwargs) + monkeypatch.setattr(builtins, "__import__", no_torch) + assert plugin_module._has_gpu() is False + + +def test_plugin_disabled_and_missing_terminal_result(monkeypatch): + from pytest_gpu_proof.config import GpuProofConfig + + plugin = plugin_module.GpuProofPlugin.__new__(plugin_module.GpuProofPlugin) + plugin.gpu_proof_config = GpuProofConfig(enabled=False) + plugin.pytest_sessionfinish(type("S", (), {"exitstatus": pytest.ExitCode.OK})(), 0) + + plugin.gpu_proof_config = GpuProofConfig(enabled=True) + plugin.collected_node_ids = ["t::missing"] + plugin._results_by_node = {} + plugin.test_results = [] + seen = {} + monkeypatch.setattr(plugin, "_emit_receipt", lambda outcome: seen.setdefault("outcome", outcome)) + session = type("S", (), {"exitstatus": pytest.ExitCode.OK})() + plugin.pytest_sessionfinish(session, 0) + assert plugin.test_results[0]["phase"] == "missing-terminal-report" + assert seen["outcome"] == "passed" + + +def test_configure_and_collection_tolerate_missing_options(): + class PluginManager: + def register(self, *args): + raise AssertionError("must not register") + + class Config: + pluginmanager = PluginManager() + + def addinivalue_line(self, *args): + pass + + def getoption(self, name): + raise ValueError(name) + + plugin_module.pytest_configure(Config()) + plugin_module.pytest_collection_modifyitems(Config(), []) + + +def test_sessionstart_unlink_failure_fails_closed(monkeypatch, tmp_path): + from pytest_gpu_proof.config import GpuProofConfig + + plugin = plugin_module.GpuProofPlugin.__new__(plugin_module.GpuProofPlugin) + plugin.gpu_proof_config = GpuProofConfig( + enabled=True, output=str(tmp_path / "receipt.json"), repo_root=str(tmp_path) + ) + plugin.started_at = "" + session = type("S", (), {"exitstatus": pytest.ExitCode.OK})() + monkeypatch.setattr( + plugin_module.Path, + "unlink", + lambda *a, **k: (_ for _ in ()).throw(OSError("permission denied")), + ) + # An exitstatus write at sessionstart would be overwritten by wrap_session, + # so failing closed this early must raise UsageError instead. + with pytest.raises(pytest.UsageError, match="cannot clear stale receipt"): + plugin.pytest_sessionstart(session) + + plugin.gpu_proof_config = GpuProofConfig( + enabled=True, + output=str(tmp_path / "receipt.json"), + repo_root=str(tmp_path), + best_effort=True, + ) + with pytest.warns(UserWarning, match="cannot clear stale receipt"): + plugin.pytest_sessionstart(session) + assert session.exitstatus == pytest.ExitCode.OK + + +def test_report_defensive_paths(): + from types import SimpleNamespace + from pytest_gpu_proof.config import GpuProofConfig + + plugin = plugin_module.GpuProofPlugin.__new__(plugin_module.GpuProofPlugin) + plugin.gpu_proof_config = GpuProofConfig(enabled=True) + plugin._results_by_node = {} + plugin.skipped_required = [] + + class Item: + nodeid = "t::x" + fixturenames = () + _gpu_proof_checks = [] + + def get_closest_marker(self, name): + return object() if name == "gpu_proof" else None + + def drive(call, report): + hook = plugin.pytest_runtest_makereport(Item(), call) + next(hook) + with pytest.raises(StopIteration): + hook.send(SimpleNamespace(get_result=lambda: report)) + + # Teardown may have no prior call result after an interrupted protocol. + drive( + SimpleNamespace(when="teardown", duration=0.0), + SimpleNamespace(failed=False, passed=True, skipped=False), + ) + # A nonstandard report plugin may provide no terminal boolean. + drive( + SimpleNamespace(when="call", duration=0.0), + SimpleNamespace(failed=False, passed=False, skipped=False), + ) + assert plugin._results_by_node["t::x"]["outcome"] == "failed" + + +def test_collection_gpu_probe_is_lazy_and_reused(monkeypatch): + from types import SimpleNamespace + + class Config: + def getoption(self, name): + return False + + class Item: + def __init__(self, marked): + self.marked = marked + self.added = [] + + def get_closest_marker(self, name): + return self.marked if name == "gpu_required" else None + + def add_marker(self, marker): + self.added.append(marker) + + calls = [] + monkeypatch.setattr(plugin_module, "_has_gpu", lambda: calls.append(1) or False) + plain, first, second = Item(False), Item(True), Item(True) + plugin_module.pytest_collection_modifyitems(Config(), [plain, first, second]) + assert not plain.added and first.added and second.added + assert calls == [1] + available = Item(True) + monkeypatch.setattr(plugin_module, "_has_gpu", lambda: True) + plugin_module.pytest_collection_modifyitems(Config(), [available]) + assert not available.added diff --git a/tests/test_receipt.py b/tests/test_receipt.py index 8bb502a..1864787 100644 --- a/tests/test_receipt.py +++ b/tests/test_receipt.py @@ -9,6 +9,10 @@ from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat from pytest_gpu_proof.receipt import ( + _env_info, + _gpu_info, + _resolve_github_username, + _utcnow, build_receipt_payload, canonicalize, finalize_receipt, @@ -63,12 +67,56 @@ def test_canonicalize_sorts_keys(): assert result.index('"a"') < result.index('"z"') +@pytest.mark.parametrize("value", [{"bad": float("nan")}, {"bad": object()}]) +def test_canonicalize_rejects_non_json_values(value): + with pytest.raises(ValueError, match="non-JSON"): + canonicalize(value) + + +def test_environment_and_gpu_capture(monkeypatch): + import subprocess + + result = subprocess.CompletedProcess( + [], + 0, + stdout=( + "0, GPU-1, Ada, 555.1, 24564 MiB, 8.9\n" + "1, GPU-2, Hopper\n" + ), + stderr="", + ) + monkeypatch.setattr("pytest_gpu_proof.receipt.subprocess.run", lambda *a, **k: result) + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1") + info = _gpu_info() + assert len(info["devices"]) == 2 + assert info["devices"][1]["driver_version"] is None + assert info["cuda_visible_devices"] == "1" + assert _env_info()["gpu_info"] == info + assert _utcnow().endswith("Z") + + +@pytest.mark.parametrize( + "behavior", + [ + lambda: __import__("subprocess").CompletedProcess([], 1, stdout="", stderr="bad"), + lambda: (_ for _ in ()).throw(FileNotFoundError()), + lambda: (_ for _ in ()).throw(__import__("subprocess").TimeoutExpired("nvidia-smi", 10)), + ], +) +def test_gpu_capture_unavailable(monkeypatch, behavior): + monkeypatch.setattr( + "pytest_gpu_proof.receipt.subprocess.run", lambda *a, **k: behavior() + ) + assert _gpu_info() is None + + def test_finalize_receipt_has_signature(mock_config, sample_test_results, signer, tmp_git_repo): os.chdir(tmp_git_repo) payload = build_receipt_payload(mock_config, sample_test_results, "2026-04-28T00:00:00Z", "2026-04-28T00:01:00Z") receipt = finalize_receipt(payload, signer) assert "signature" in receipt - assert receipt["signature"]["algorithm"] == "ed25519" + assert receipt["signer"]["algorithm"] == "ed25519" + assert set(receipt["signature"]) == {"value"} assert receipt["signature"]["value"] @@ -77,7 +125,7 @@ def test_receipt_structure(mock_config, sample_test_results, signer, tmp_git_rep payload = build_receipt_payload(mock_config, sample_test_results, "2026-04-28T00:00:00Z", "2026-04-28T00:01:00Z") receipt = finalize_receipt(payload, signer) - assert receipt["schema_version"] == "1" + assert receipt["schema_version"] == "3" assert receipt["mode"] == "local" assert "repo" in receipt assert "fingerprint" in receipt @@ -94,10 +142,113 @@ def test_write_receipt(tmp_path, mock_config, sample_test_results, signer, tmp_g write_receipt(receipt, str(out)) loaded = json.loads(out.read_text()) - assert loaded["schema_version"] == "1" + assert loaded["schema_version"] == "3" assert "signature" in loaded +def test_write_receipt_cleans_temporary_after_replace_error(tmp_path, monkeypatch): + monkeypatch.setattr("pytest_gpu_proof.receipt.os.replace", lambda *a: (_ for _ in ()).throw(OSError("no"))) + with pytest.raises(OSError, match="no"): + write_receipt({"ok": True}, str(tmp_path / "receipt.json")) + assert list(tmp_path.iterdir()) == [] + + +def test_legacy_finalize_shape(signer): + receipt = finalize_receipt( + {"schema_version": "2", "repo": {"github_username": "alice"}}, signer + ) + assert receipt["signature"]["signer"] == "alice" + assert receipt["signature"]["algorithm"] == "ed25519" + + +def test_duplicate_collected_node_ids_rejected( + mock_config, sample_test_results, tmp_git_repo +): + mock_config.repo_root = str(tmp_git_repo) + with pytest.raises(ValueError, match="not unique"): + build_receipt_payload( + mock_config, + sample_test_results, + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:01Z", + collected_node_ids=["same", "same"], + ) + + +def test_shard_extra_paths_fall_back_to_global( + mock_config, sample_test_results, tmp_git_repo +): + extra = tmp_git_repo / "generated.bin" + extra.write_bytes(b"generated") + mock_config.repo_root = str(tmp_git_repo) + mock_config.fingerprint_extra_paths = ["generated.bin"] + mock_config.shard_name = "core" + payload = build_receipt_payload( + mock_config, + sample_test_results, + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:01Z", + override_github_username="alice", + ) + assert payload["shards"][0]["fingerprint"]["extra_paths"] == ["generated.bin"] + + +def test_explicit_empty_shard_extras_override_global( + mock_config, sample_test_results, tmp_git_repo +): + extra = tmp_git_repo / "generated.bin" + extra.write_bytes(b"generated") + mock_config.repo_root = str(tmp_git_repo) + mock_config.fingerprint_extra_paths = ["generated.bin"] + mock_config.shard_name = "core" + mock_config.shard_fingerprint_extra_paths = [] + payload = build_receipt_payload( + mock_config, + sample_test_results, + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:01Z", + override_github_username="alice", + ) + assert payload["shards"][0]["fingerprint"]["extra_paths"] == [] + + +def test_recording_ignores_only_tracked_output_dirtiness( + mock_config, sample_test_results, tmp_git_repo +): + import subprocess + + receipt = tmp_git_repo / "gpu-proof.json" + receipt.write_text("old\n") + subprocess.run(["git", "add", "-f", "gpu-proof.json"], cwd=tmp_git_repo, check=True) + subprocess.run(["git", "commit", "-m", "receipt"], cwd=tmp_git_repo, check=True, capture_output=True) + receipt.unlink() + mock_config.repo_root = str(tmp_git_repo) + mock_config.output = str(receipt) + payload = build_receipt_payload( + mock_config, + sample_test_results, + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:01Z", + override_github_username="alice", + ) + assert payload["repo"]["dirty"] is False + + +def test_output_outside_repo_is_not_a_dirty_exclusion( + mock_config, sample_test_results, tmp_git_repo +): + mock_config.repo_root = str(tmp_git_repo) + mock_config.output = str(tmp_git_repo.parent / "outside" / "receipt.json") + payload = build_receipt_payload( + mock_config, + sample_test_results, + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:01Z", + override_github_username="alice", + ) + assert payload["repo"]["dirty"] is False + + # ─── signer (github_username) resolution ──────────────────────────────────── def _build(mock_config, sample_test_results, tmp_git_repo): @@ -134,9 +285,16 @@ def test_signer_remote_owner_fallback_warns( # gh CLI unavailable (autouse fixture) and the origin remote is org-owned import subprocess subprocess.run( - ["git", "remote", "add", "origin", "git@github.com:Some-Org/repo.git"], + ["git", "remote", "set-url", "origin", "git@github.com:Some-Org/repo.git"], cwd=tmp_git_repo, check=True, capture_output=True, ) payload = _build(mock_config, sample_test_results, tmp_git_repo) assert payload["repo"]["github_username"] == "Some-Org" - assert "origin remote owner" in capsys.readouterr().out + assert "origin owner" in capsys.readouterr().out + + +def test_signer_resolution_failure(monkeypatch, mock_config): + monkeypatch.setattr("pytest_gpu_proof.receipt.get_gh_cli_login", lambda: None) + monkeypatch.setattr("pytest_gpu_proof.receipt.get_github_username", lambda root: None) + with pytest.raises(ValueError, match="cannot determine"): + _resolve_github_username(mock_config, None, ".") diff --git a/tests/test_sharding.py b/tests/test_sharding.py index d29cb05..9a592fa 100644 --- a/tests/test_sharding.py +++ b/tests/test_sharding.py @@ -46,12 +46,13 @@ def _result(node_id): def _shard_receipt(tmp_path, tmp_git_repo, signer, fname, shard_name, shard_paths, - results, *, ended_days_ago=0.0, mutate=None): + results, *, ended_days_ago=0.0, mutate=None, shard_extra_paths=None): """A schema-2 single-shard receipt built at tmp_git_repo's current HEAD.""" os.chdir(tmp_git_repo) config = GpuProofConfig(enabled=True, fingerprint_paths=["src", "tests"], shard_name=shard_name, - shard_fingerprint_paths=shard_paths) + shard_fingerprint_paths=shard_paths, + shard_fingerprint_extra_paths=shard_extra_paths or []) payload = build_receipt_payload( config, results, _utcstamp(ended_days_ago), _utcstamp(ended_days_ago)) if mutate is not None: @@ -64,8 +65,8 @@ def _shard_receipt(tmp_path, tmp_git_repo, signer, fname, shard_name, shard_path def _mock_github_keys(public_key): def _fake(data, signature, username): - return _verify_with_key(public_key, signature, data) - return patch("pytest_gpu_proof.verify.verify_with_github_keys", side_effect=_fake) + return public_key if _verify_with_key(public_key, signature, data) else None + return patch("pytest_gpu_proof.verify.find_verifying_github_key", side_effect=_fake) def _git(repo, *args): @@ -79,7 +80,7 @@ def test_shard_emission_schema2(tmp_path, tmp_git_repo, signer_with_key): p = _shard_receipt(tmp_path, tmp_git_repo, signer, "a.json", "modA", ["src"], [_result("tests/test_add.py::test_add")]) r = json.loads(p.read_text()) - assert r["schema_version"] == "2" + assert r["schema_version"] == "3" (shard,) = r["shards"] assert shard["name"] == "modA" assert shard["fingerprint"]["included_paths"] == ["src"] @@ -102,7 +103,7 @@ def test_ok(): "--gpu-proof-shard-fingerprint-paths=.") result.assert_outcomes(passed=1) r = json.loads((pytester.path / "gpu-proof.json").read_text()) - assert r["schema_version"] == "2" + assert r["schema_version"] == "3" assert r["shards"][0]["name"] == "mymod" @@ -121,7 +122,7 @@ def test_v2_merge_verifies(tmp_path, tmp_git_repo, signer_with_key): a, b = _two_shards(tmp_path, tmp_git_repo, signer) out = tmp_path / "merged.json" merged = merge_receipts([str(a), str(b)], str(out), key_path=key_path) - assert merged["schema_version"] == "2" + assert merged["schema_version"] == "3" assert [s["name"] for s in merged["shards"]] == ["modA", "modB"] os.chdir(tmp_git_repo) with _mock_github_keys(public_key): @@ -155,7 +156,7 @@ def test_verify_rejects_shard_fingerprint_drift(tmp_path, tmp_git_repo, signer_w _git(tmp_git_repo, "commit", "-m", "drift extra only") os.chdir(tmp_git_repo) with _mock_github_keys(public_key): - with pytest.raises(VerificationError, match="shard 'modA' fingerprint mismatch"): + with pytest.raises(VerificationError, match="shard 'modA' fingerprint does not match"): _verify(str(p), None, str(tmp_git_repo), "testuser", None) @@ -166,7 +167,7 @@ def test_verify_rejects_membership_hole(tmp_path, tmp_git_repo, signer_with_key) mutate=lambda pl: pl["shards"][0]["node_ids"].remove("t::orphan")) os.chdir(tmp_git_repo) with _mock_github_keys(public_key): - with pytest.raises(VerificationError, match="does not partition"): + with pytest.raises(VerificationError, match="does not exactly partition"): _verify(str(p), None, str(tmp_git_repo), "testuser", None) @@ -221,7 +222,7 @@ def test_carry_forward_happy_and_policy_gate(tmp_path, tmp_git_repo, signer_with os.chdir(tmp_git_repo) with _mock_github_keys(public_key): # no policy -> carried shard REJECTED (the trust boundary) - with pytest.raises(VerificationError, match="allow_carried"): + with pytest.raises(VerificationError, match="policy rejects carry-forward"): _verify(str(out), None, str(tmp_git_repo), "testuser", None) # opt-in policy -> verifies _verify(str(out), _policy(tmp_path, allow_carried=True), @@ -255,7 +256,7 @@ def test_verify_rejects_stale_carried_shard(tmp_path, tmp_git_repo, signer_with_ carry_from=str(old), repo_root=str(tmp_git_repo)) os.chdir(tmp_git_repo) with _mock_github_keys(public_key): - with pytest.raises(VerificationError, match="day\\(s\\) old"): + with pytest.raises(VerificationError, match="outside its age policy"): _verify(str(out), _policy(tmp_path, allow_carried=True, carried_max_age_days=30), str(tmp_git_repo), "testuser", None) @@ -272,11 +273,196 @@ def test_schema1_with_shards_block_rejected(tmp_path, tmp_git_repo, signer_with_ os.chdir(tmp_git_repo) config = GpuProofConfig(enabled=True, fingerprint_paths=["src", "tests"]) payload = build_receipt_payload(config, [_result("t::a")], _utcstamp(), _utcstamp()) - assert payload["schema_version"] == "1" + assert payload["schema_version"] == "3" + payload["schema_version"] = "1" payload["shards"] = [{"name": "smuggled", "fingerprint": {}, "node_ids": []}] receipt = finalize_receipt(payload, signer) path = tmp_path / "smuggled.json" write_receipt(receipt, str(path)) with _mock_github_keys(public_key): - with pytest.raises(VerificationError, match="must not carry a shards block"): + with pytest.raises(VerificationError, match="cannot contain shards"): _verify(str(path), None, str(tmp_git_repo), "testuser", None) + + +def test_carry_preserves_and_checks_extra_paths(tmp_path, tmp_git_repo, signer_with_key): + signer, _, _ = signer_with_key + generated = tmp_git_repo / "generated.bin" + generated.write_bytes(b"stable") + old = _shard_receipt( + tmp_path, tmp_git_repo, signer, "old-extra.json", "extra", [], + [_result("t::extra")], shard_extra_paths=["generated.bin"], + ) + old_r = json.loads(old.read_text()) + fresh = _shard_receipt( + tmp_path, tmp_git_repo, signer, "fresh.json", "fresh", ["src"], + [_result("t::fresh")], + ) + payload = merge_payloads([json.loads(fresh.read_text())], ["fresh.json"]) + carried = carry_forward(payload, old_r, "old-extra.json", str(tmp_git_repo)) + assert {s["name"] for s in carried["shards"]} == {"fresh", "extra"} + + +def test_carry_refusal_matrix(tmp_path, tmp_git_repo, signer_with_key, capsys): + signer, _, _ = signer_with_key + fresh = _shard_receipt( + tmp_path, tmp_git_repo, signer, "fresh.json", "same", ["src"], [_result("t::same")] + ) + payload = merge_payloads([json.loads(fresh.read_text())], ["fresh.json"]) + old = json.loads(fresh.read_text()) + + invalid_schema = dict(old) + invalid_schema["schema_version"] = "1" + with pytest.raises(MergeError, match="not a schema"): + carry_forward(dict(payload), invalid_schema, "old", str(tmp_git_repo)) + + missing_sha = json.loads(fresh.read_text()) + missing_sha["repo"]["commit_sha"] = None + with pytest.raises(MergeError, match="commit SHAs"): + carry_forward(dict(payload), missing_sha, "old", str(tmp_git_repo)) + + unchanged = carry_forward(payload, old, "old", str(tmp_git_repo)) + assert unchanged + assert "nothing to carry" in capsys.readouterr().out + + +def test_carry_rejects_missing_claimed_test(tmp_path, tmp_git_repo, signer_with_key): + signer, _, _ = signer_with_key + old = _shard_receipt( + tmp_path, tmp_git_repo, signer, "old.json", "old", ["src"], [_result("t::old")] + ) + old_r = json.loads(old.read_text()) + old_r["tests"] = [] + fresh = _shard_receipt( + tmp_path, tmp_git_repo, signer, "fresh.json", "fresh", ["tests"], [_result("t::fresh")] + ) + payload = merge_payloads([json.loads(fresh.read_text())], ["fresh.json"]) + with pytest.raises(MergeError, match="not in its receipt"): + carry_forward(payload, old_r, "old", str(tmp_git_repo)) + + +def test_carry_rejects_invalid_fingerprint_and_duplicate_node( + tmp_path, tmp_git_repo, signer_with_key +): + signer, _, _ = signer_with_key + old = _shard_receipt( + tmp_path, tmp_git_repo, signer, "old.json", "old", ["src"], [_result("t::old")] + ) + fresh = _shard_receipt( + tmp_path, tmp_git_repo, signer, "fresh.json", "fresh", ["tests"], [_result("t::fresh")] + ) + old_r = json.loads(old.read_text()) + payload = merge_payloads([json.loads(fresh.read_text())], ["fresh.json"]) + old_r["shards"][0]["fingerprint"]["algorithm"] = "bad" + with pytest.raises(MergeError, match="fingerprint is invalid"): + carry_forward(payload, old_r, "old", str(tmp_git_repo)) + + old_r = json.loads(old.read_text()) + old_r["shards"][0]["node_ids"] = ["t::fresh"] + with pytest.raises(MergeError, match="re-introduce"): + carry_forward(payload, old_r, "old", str(tmp_git_repo)) + + +@pytest.mark.parametrize( + ("mutate", "policy", "message"), + [ + (lambda r: r.__setitem__("shards", []), {}, "has no shards"), + (lambda r: r["shards"].__setitem__(0, "bad"), {}, "valid name"), + (lambda r: r["shards"].append(dict(r["shards"][0])), {}, "duplicate shard"), + (lambda r: r["shards"][0].__setitem__("node_ids", "bad"), {}, "invalid node_ids"), + (lambda r: r["shards"][0].__setitem__("fingerprint", None), {}, "no fingerprint"), + ( + lambda r: r["shards"][0]["fingerprint"].__setitem__("algorithm", "bad"), + {}, "unsupported fingerprint", + ), + (lambda r: r["shards"][0].__setitem__("carried", "bad"), {"allow_carried": True}, "metadata is invalid"), + ( + lambda r: r["shards"][0].__setitem__( + "carried", {"original_ended_at": _utcstamp()} + ), + {"allow_carried": True, "carried_max_age_days": "bad"}, + "carried_max_age_days", + ), + ], +) +def test_shard_structure_refusal_matrix( + tmp_path, tmp_git_repo, signer_with_key, mutate, policy, message +): + signer, public_key, _ = signer_with_key + path = _shard_receipt( + tmp_path, tmp_git_repo, signer, "shard.json", "core", ["src"], [_result("t::a")] + ) + receipt = json.loads(path.read_text()) + mutate(receipt) + path.write_text(json.dumps(receipt)) + policy_path = _policy(tmp_path, **policy) if policy else None + with patch("pytest_gpu_proof.verify.find_verifying_github_key", return_value=public_key): + with pytest.raises(VerificationError, match=message): + _verify(str(path), policy_path, str(tmp_git_repo), "testuser", None) + + +def test_shard_overlap_and_required_policy(tmp_path, tmp_git_repo, signer_with_key): + signer, public_key, _ = signer_with_key + path = _shard_receipt( + tmp_path, tmp_git_repo, signer, "shard.json", "core", ["src"], [_result("t::a")] + ) + receipt = json.loads(path.read_text()) + second = dict(receipt["shards"][0]) + second["name"] = "other" + receipt["shards"].append(second) + path.write_text(json.dumps(receipt)) + with patch("pytest_gpu_proof.verify.find_verifying_github_key", return_value=public_key): + with pytest.raises(VerificationError, match="overlaps"): + _verify(str(path), None, str(tmp_git_repo), "testuser", None) + + receipt["shards"] = receipt["shards"][:1] + path.write_text(json.dumps(receipt)) + with patch("pytest_gpu_proof.verify.find_verifying_github_key", return_value=public_key): + with pytest.raises(VerificationError, match="paths do not match"): + _verify( + str(path), + _policy(tmp_path, required_shard_fingerprints={"core": {"paths": ["tests"]}}), + str(tmp_git_repo), "testuser", None, + ) + with pytest.raises(VerificationError, match="extra paths"): + _verify( + str(path), + _policy(tmp_path, required_shard_fingerprints={"core": {"paths": ["src"], "extra_paths": ["x"]}}), + str(tmp_git_repo), "testuser", None, + ) + with pytest.raises(VerificationError, match="exclusions"): + _verify( + str(path), + _policy(tmp_path, required_shard_fingerprints={"core": {"excluded_paths": []}}), + str(tmp_git_repo), "testuser", None, + ) + with pytest.raises(VerificationError, match="shard set"): + _verify( + str(path), + _policy( + tmp_path, + required_shard_fingerprints={ + "core": {"paths": ["src"]}, + "missing": {"paths": ["tests"]}, + }, + ), + str(tmp_git_repo), "testuser", None, + ) + + +def test_direct_shard_validation_rejects_bad_carried_limit( + tmp_path, tmp_git_repo, signer_with_key +): + from pytest_gpu_proof.verify import _verify_shards + + signer, _, _ = signer_with_key + path = _shard_receipt( + tmp_path, tmp_git_repo, signer, "shard.json", "core", ["src"], [_result("t::a")] + ) + receipt = json.loads(path.read_text()) + receipt["shards"][0]["carried"] = {"original_ended_at": _utcstamp()} + with pytest.raises(VerificationError, match="non-negative integer"): + _verify_shards( + receipt, + str(tmp_git_repo), + {"allow_carried": True, "carried_max_age_days": "bad"}, + ) diff --git a/tests/test_signing.py b/tests/test_signing.py index 3681ba3..510225c 100644 --- a/tests/test_signing.py +++ b/tests/test_signing.py @@ -5,6 +5,7 @@ import pytest from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from cryptography.hazmat.primitives.serialization import ( + BestAvailableEncryption, Encoding, NoEncryption, PrivateFormat, @@ -13,9 +14,16 @@ from pytest_gpu_proof.signers.ed25519 import ( SSHSigner, + _discover_ssh_key, + _parse_pubkey_line, + fetch_github_public_keys, + find_verifying_github_key, + public_key_algorithm, + verify_with_github_keys, _verify_with_key, _sign_with_key, ) +from pytest_gpu_proof.signers.base import VerifierError @pytest.fixture @@ -81,16 +89,184 @@ def test_algorithm_derived_from_key_type(ssh_key_file, tmp_path): assert SSHSigner(key_path=str(ec_path)).algorithm() == "ecdsa-sha256" +def test_ecdsa_and_rsa_roundtrips(tmp_path): + from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1, generate_private_key + from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key as rsa_key + + for key in (generate_private_key(SECP256R1()), rsa_key(public_exponent=65537, key_size=2048)): + path = tmp_path / f"key-{type(key).__name__}" + path.write_bytes(key.private_bytes(Encoding.PEM, PrivateFormat.OpenSSH, NoEncryption())) + signer = SSHSigner(str(path)) + signature = signer.sign(b"payload") + assert _verify_with_key(key.public_key(), signature, b"payload") + assert signer.algorithm() in {"ecdsa-sha256", "rsa-pss-sha256"} + + +def test_unsupported_key_types_are_rejected(): + with pytest.raises(VerifierError, match="Unsupported public"): + public_key_algorithm(object()) + with pytest.raises(VerifierError, match="Unsupported private"): + _sign_with_key(object(), b"data") + assert _verify_with_key(object(), b"sig", b"data") is False + + def test_receipt_algorithm_matches_key(ssh_key_file): from pytest_gpu_proof.receipt import finalize_receipt key_path, _ = ssh_key_file signer = SSHSigner(key_path=str(key_path)) - receipt = finalize_receipt({"repo": {}, "tests": []}, signer) - assert receipt["signature"]["algorithm"] == "ed25519" + receipt = finalize_receipt({"schema_version": "3", "repo": {}, "tests": []}, signer) + assert receipt["signer"]["algorithm"] == "ed25519" def test_missing_key_raises(tmp_path): - from pytest_gpu_proof.signers.base import VerifierError with pytest.raises(VerifierError, match="No SSH private key found"): SSHSigner(key_path=str(tmp_path / "nonexistent")) + + +def test_key_discovery_prefers_git_signing_key(monkeypatch, ssh_key_file): + key_path, _ = ssh_key_file + monkeypatch.setattr( + "pytest_gpu_proof.gitutils.get_git_signing_key", lambda root: str(key_path) + ) + assert _discover_ssh_key("repo") == str(key_path) + + +def test_key_discovery_home_candidates_and_none(monkeypatch, tmp_path): + ssh = tmp_path / ".ssh" + ssh.mkdir() + candidate = ssh / "id_rsa" + candidate.write_text("placeholder") + monkeypatch.setattr("pytest_gpu_proof.gitutils.get_git_signing_key", lambda root: None) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + assert _discover_ssh_key() == str(candidate) + candidate.unlink() + assert _discover_ssh_key() is None + + +def test_signer_automatic_discovery_and_failure(monkeypatch, ssh_key_file): + key_path, _ = ssh_key_file + monkeypatch.setattr( + "pytest_gpu_proof.signers.ed25519._discover_ssh_key", lambda root: str(key_path) + ) + assert SSHSigner(root="repo").sign(b"x") + monkeypatch.setattr( + "pytest_gpu_proof.signers.ed25519._discover_ssh_key", lambda root: None + ) + with pytest.raises(VerifierError, match="No SSH private key found"): + SSHSigner(root="repo") + + +def test_encrypted_private_key_prompts(monkeypatch, tmp_path): + private_key = Ed25519PrivateKey.generate() + path = tmp_path / "encrypted" + path.write_bytes(b"encrypted-placeholder") + calls = [] + + def load(data, password): + calls.append(password) + if password is None: + raise TypeError("encrypted") + assert password == b"secret" + return private_key + + monkeypatch.setattr("pytest_gpu_proof.signers.ed25519.load_ssh_private_key", load) + monkeypatch.setattr("getpass.getpass", lambda prompt: "secret") + assert SSHSigner(str(path)).sign(b"x") + assert calls == [None, b"secret"] + + +def test_encrypted_key_valueerror_also_prompts(monkeypatch, tmp_path): + """cryptography >= 41 raises ValueError (not TypeError) for + passphrase-protected OpenSSH keys; the prompt fallback must cover both. + (Real encrypted-key round-trips need bcrypt, so this stays mocked.)""" + private_key = Ed25519PrivateKey.generate() + path = tmp_path / "encrypted" + path.write_bytes(b"encrypted-placeholder") + + def load(data, password): + if password is None: + raise ValueError("Key is password-protected.") + assert password == b"secret" + return private_key + + monkeypatch.setattr("pytest_gpu_proof.signers.ed25519.load_ssh_private_key", load) + monkeypatch.setattr("getpass.getpass", lambda prompt: "secret") + assert SSHSigner(str(path)).sign(b"x") + + +def test_encrypted_key_without_terminal_fails_closed(monkeypatch, tmp_path): + path = tmp_path / "encrypted" + path.write_bytes(b"encrypted-placeholder") + + def load(data, password): + raise ValueError("Key is password-protected.") + + monkeypatch.setattr("pytest_gpu_proof.signers.ed25519.load_ssh_private_key", load) + + def no_tty(prompt): + raise EOFError("no terminal") + + monkeypatch.setattr("getpass.getpass", no_tty) + with pytest.raises(VerifierError, match="Could not load SSH private key"): + SSHSigner(str(path)) + + # Wrong passphrase also surfaces as the actionable signer error. + monkeypatch.setattr("getpass.getpass", lambda prompt: "wrong") + with pytest.raises(VerifierError, match="Could not load SSH private key"): + SSHSigner(str(path)) + + +def test_fetch_rejects_invalid_github_username(): + with pytest.raises(VerifierError, match="invalid GitHub username"): + fetch_github_public_keys("../evil?path") + + +def test_parse_and_fetch_github_keys(monkeypatch, ssh_key_file): + _, private_key = ssh_key_file + public_line = private_key.public_key().public_bytes( + Encoding.OpenSSH, PublicFormat.OpenSSH + ).decode() + assert _parse_pubkey_line("bad") is None + assert _parse_pubkey_line("ssh-bad !!!") is None + assert _parse_pubkey_line(public_line) + + class Response: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self, limit=None): + return f"bad\n{public_line} comment\n".encode() + + monkeypatch.setattr("pytest_gpu_proof.signers.ed25519.urlopen", lambda *a, **k: Response()) + keys = fetch_github_public_keys("alice") + signature = _sign_with_key(private_key, b"payload") + assert verify_with_github_keys(b"payload", signature, "alice") + assert find_verifying_github_key(b"payload", signature, "alice") is not None + assert find_verifying_github_key(b"wrong", signature, "alice") is None + + +def test_fetch_github_keys_errors(monkeypatch): + def network_error(*args, **kwargs): + raise OSError("offline") + + monkeypatch.setattr("pytest_gpu_proof.signers.ed25519.urlopen", network_error) + with pytest.raises(VerifierError, match="Could not fetch"): + fetch_github_public_keys("alice") + + class Empty: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self, limit=None): + return b"not-a-key\n" + + monkeypatch.setattr("pytest_gpu_proof.signers.ed25519.urlopen", lambda *a, **k: Empty()) + with pytest.raises(VerifierError, match="No usable"): + fetch_github_public_keys("alice") diff --git a/tests/test_verifier.py b/tests/test_verifier.py index e10106c..6d9614c 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -18,7 +18,7 @@ write_receipt, ) from pytest_gpu_proof.signers.ed25519 import SSHSigner, _verify_with_key -from pytest_gpu_proof.verify import VerificationError, _verify +from pytest_gpu_proof.verify import VerificationError, _load_policy, _verify, verify_receipt @pytest.fixture @@ -66,7 +66,8 @@ def _make_receipt( } ] now = _utcstamp() - payload = build_receipt_payload(config, results, now, ended_at or now) + stamp = ended_at or now + payload = build_receipt_payload(config, results, stamp, stamp) if mutate is not None: mutate(payload) if sign: @@ -87,12 +88,12 @@ def good_receipt(tmp_path, tmp_git_repo, signer_with_key): def _mock_github_keys(public_key): - """Return a patcher that makes verify_with_github_keys use our local public key.""" + """Return a patcher that makes schema-3 verification use our local key.""" def _fake_verify(data, signature, username): - return _verify_with_key(public_key, signature, data) + return public_key if _verify_with_key(public_key, signature, data) else None return patch( - "pytest_gpu_proof.verify.verify_with_github_keys", + "pytest_gpu_proof.verify.find_verifying_github_key", side_effect=_fake_verify, ) @@ -110,7 +111,7 @@ def test_verify_passes_when_receipt_is_committed_after_code(good_receipt, tmp_gi import subprocess - subprocess.run(["git", "add", "gpu-proof.json"], cwd=tmp_git_repo, check=True) + subprocess.run(["git", "add", "-f", "gpu-proof.json"], cwd=tmp_git_repo, check=True) subprocess.run( ["git", "commit", "-m", "add GPU proof receipt"], cwd=tmp_git_repo, @@ -126,7 +127,7 @@ def test_verify_fails_on_bad_signature(good_receipt, tmp_git_repo): path, _ = good_receipt other_key = Ed25519PrivateKey.generate().public_key() with _mock_github_keys(other_key): - with pytest.raises(VerificationError, match="Signature does not match"): + with pytest.raises(VerificationError, match="signature does not match"): _verify(str(path), None, str(tmp_git_repo), "testuser", None) @@ -137,7 +138,7 @@ def test_verify_fails_on_modified_receipt(good_receipt, tmp_git_repo): path.write_text(json.dumps(receipt)) with _mock_github_keys(public_key): - with pytest.raises(VerificationError, match="Signature does not match"): + with pytest.raises(VerificationError, match="signature does not match"): _verify(str(path), None, str(tmp_git_repo), "testuser", None) @@ -149,7 +150,7 @@ def test_verify_fails_on_stale_receipt(tmp_path, tmp_git_repo, signer_with_key): tmp_path, tmp_git_repo, signer, ended_at="2020-01-01T00:00:00Z" ) with _mock_github_keys(public_key): - with pytest.raises(VerificationError, match=r"days old; policy allows max 30"): + with pytest.raises(VerificationError, match=r"older than the 30-day policy"): _verify(str(path), None, str(tmp_git_repo), "testuser", 30) @@ -158,14 +159,14 @@ def test_verify_max_age_zero_is_respected(tmp_path, tmp_git_repo, signer_with_ke path = _make_receipt(tmp_path, tmp_git_repo, signer, ended_at=_utcstamp(days_ago=5)) with _mock_github_keys(public_key): # An explicit override of 0 must not silently fall back to 30. - with pytest.raises(VerificationError, match=r"policy allows max 0"): + with pytest.raises(VerificationError, match=r"older than the 0-day policy"): _verify(str(path), None, str(tmp_git_repo), "testuser", 0) def test_verify_rejects_unsigned_receipt(tmp_path, tmp_git_repo, signer_with_key): signer, _ = signer_with_key path = _make_receipt(tmp_path, tmp_git_repo, signer, sign=False) - with pytest.raises(VerificationError, match="UNSIGNED"): + with pytest.raises(VerificationError, match="unsigned"): _verify(str(path), None, str(tmp_git_repo), "testuser", None) @@ -203,7 +204,7 @@ def drop_digest(payload): path = _make_receipt(tmp_path, tmp_git_repo, signer, mutate=drop_digest) with _mock_github_keys(public_key): - with pytest.raises(VerificationError, match="missing its digest"): + with pytest.raises(VerificationError, match="empty or missing"): _verify(str(path), None, str(tmp_git_repo), "testuser", None) @@ -213,7 +214,7 @@ def test_yaml_policy_without_pyyaml_raises(tmp_path, monkeypatch): policy = tmp_path / "policy.yaml" policy.write_text("max_age_days: 7\n") monkeypatch.setitem(sys.modules, "yaml", None) # force ImportError - with pytest.raises(VerificationError, match="PyYAML"): + with pytest.raises(VerificationError, match="yaml.*extra|YAML policy"): _load_policy(str(policy)) @@ -234,7 +235,7 @@ def null_gpu(payload): path = _make_receipt(tmp_path, tmp_git_repo, signer, mutate=null_gpu) with _mock_github_keys(public_key): - with pytest.raises(VerificationError, match="gpu_info"): + with pytest.raises(VerificationError, match="GPU information"): _verify( str(path), None, str(tmp_git_repo), "testuser", None, require_gpu=True ) @@ -316,7 +317,7 @@ def test_expected_skips_rejects_unexpected_skip(tmp_path, tmp_git_repo, signer_w baseline = tmp_path / "expected_skips.txt" baseline.write_text("tests/test_add.py::test_other_skip\n") with _mock_github_keys(public_key): - with pytest.raises(VerificationError, match="NOT in the baseline"): + with pytest.raises(VerificationError, match="skip set"): _verify( str(path), None, str(tmp_git_repo), "testuser", None, expected_skips_path=str(baseline), @@ -330,7 +331,7 @@ def test_expected_skips_rejects_stale_baseline(tmp_path, tmp_git_repo, signer_wi baseline = tmp_path / "expected_skips.txt" baseline.write_text("tests/test_add.py::test_skipped\n") with _mock_github_keys(public_key): - with pytest.raises(VerificationError, match="stale"): + with pytest.raises(VerificationError, match="skip set"): _verify( str(path), None, str(tmp_git_repo), "testuser", None, expected_skips_path=str(baseline), @@ -361,3 +362,504 @@ def test_expected_skips_from_toml(tmp_path, tmp_git_repo, signer_with_key): ) with _mock_github_keys(public_key): _verify(str(path), None, str(tmp_git_repo), "testuser", None) + + +# ─── strict schema and policy validation ─────────────────────────────────── + +def _policy(tmp_path, content, suffix=".json"): + path = tmp_path / f"policy{suffix}" + path.write_text(content if isinstance(content, str) else json.dumps(content)) + return str(path) + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda p: p.__setitem__("repo", []), "repo.*object"), + (lambda p: p.__setitem__("environment", []), "environment.*object"), + (lambda p: p.__setitem__("mode", "unknown"), "mode is missing"), + (lambda p: p["repo"].__setitem__("commit_sha", ""), "commit_sha"), + (lambda p: p.__setitem__("tests", []), "no test results"), + (lambda p: p["tests"].__setitem__(0, "bad"), "valid node_id"), + (lambda p: p["tests"][0].__setitem__("outcome", "maybe"), "invalid outcome"), + (lambda p: p["tests"][0].__setitem__("checks", {}), "checks must be a list"), + (lambda p: p["tests"][0].__setitem__("checks", ["bad"]), r"checks\[0\] is invalid"), + (lambda p: p["tests"].append(dict(p["tests"][0])), "duplicate test"), + (lambda p: p["session"].__setitem__("node_ids", []), "does not exactly match"), + (lambda p: p["session"].__setitem__("started_at", None), "not a UTC timestamp"), + (lambda p: p["session"].__setitem__("ended_at", "bad"), "not a valid UTC"), + ( + lambda p: p["session"].update( + started_at="2026-01-02T00:00:00Z", ended_at="2026-01-01T00:00:00Z" + ), + "precedes", + ), + (lambda p: p["session"].__setitem__("outcome", "unknown"), "session.outcome"), + ], +) +def test_structure_validation_errors( + tmp_path, tmp_git_repo, signer_with_key, mutate, message +): + signer, _ = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer, mutate=mutate, sign=False) + with pytest.raises(VerificationError, match=message): + _verify(str(path), None, str(tmp_git_repo), "testuser", None, allow_unsigned=True) + + +@pytest.mark.parametrize( + ("content", "message"), + [ + ("{bad", "cannot read policy"), + ("[]", "policy must"), + ('{"unknown": true}', "unknown policy"), + ('{"signer_mode": "closed"}', "signer_mode"), + ('{"signer_mode": "restricted"}', "no allowlist"), + ], +) +def test_policy_validation_errors(tmp_path, content, message): + with pytest.raises(VerificationError, match=message): + _load_policy(_policy(tmp_path, content)) + + +@pytest.mark.parametrize( + ("policy", "message"), + [ + ({"allowed_signers": "alice"}, "list of strings"), + ({"allowed_signers": [""]}, "list of strings"), + ({"allow_dirty": "yes"}, "boolean"), + ({"allow_carried": 1}, "boolean"), + ({"max_age_days": True}, "non-negative integer"), + ({"carried_max_age_days": -1}, "non-negative integer"), + ({"require_mode": "remote"}, "local.*ci-gpu"), + ({"required_test_manifest": []}, "path string"), + ({"required_shard_fingerprints": []}, "must be an object"), + ({"required_shard_fingerprints": {"a": []}}, "scope must be an object"), + ({"required_shard_fingerprints": {"a": {"bad": []}}}, "unknown policy"), + ({"required_shard_fingerprints": {"a": {"paths": "src"}}}, "list of strings"), + ], +) +def test_policy_field_types_are_strict(tmp_path, policy, message): + with pytest.raises(VerificationError, match=message): + _load_policy(_policy(tmp_path, policy)) + + +def test_policy_empty_and_yaml(tmp_path): + assert _load_policy(None) == {} + assert _load_policy(_policy(tmp_path, "null")) == {} + assert _load_policy(_policy(tmp_path, "max_age_days: 5\n", ".yaml")) == { + "max_age_days": 5 + } + with pytest.raises(VerificationError, match="cannot read policy"): + _load_policy(str(tmp_path / "missing.json")) + + +def test_open_and_restricted_signer_modes( + tmp_path, tmp_git_repo, signer_with_key +): + signer, public_key = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer) + key_fp = json.loads(path.read_text())["signer"]["key_fingerprint"] + with _mock_github_keys(public_key): + _verify( + str(path), + _policy(tmp_path, {"signer_mode": "restricted", "allowed_signers": ["testuser"]}), + str(tmp_git_repo), "testuser", None, + ) + _verify( + str(path), + _policy(tmp_path, {"signer_mode": "restricted", "allowed_key_fingerprints": [key_fp]}), + str(tmp_git_repo), "testuser", None, + ) + with pytest.raises(VerificationError, match="signer @testuser"): + _verify( + str(path), + _policy(tmp_path, {"signer_mode": "restricted", "allowed_signers": ["other"]}), + str(tmp_git_repo), "testuser", None, + ) + with pytest.raises(VerificationError, match="signing key"): + _verify( + str(path), + _policy(tmp_path, {"signer_mode": "restricted", "allowed_key_fingerprints": ["SHA256:no"]}), + str(tmp_git_repo), "testuser", None, + ) + + +def test_policy_mode_and_fingerprint_scope( + tmp_path, tmp_git_repo, signer_with_key +): + signer, public_key = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer) + with _mock_github_keys(public_key): + with pytest.raises(VerificationError, match="required mode"): + _verify(str(path), _policy(tmp_path, {"require_mode": "ci-gpu"}), str(tmp_git_repo), "testuser", None) + with pytest.raises(VerificationError, match="fingerprint paths"): + _verify(str(path), _policy(tmp_path, {"required_fingerprint_paths": ["src"]}), str(tmp_git_repo), "testuser", None) + with pytest.raises(VerificationError, match="extra paths"): + _verify(str(path), _policy(tmp_path, {"required_fingerprint_extra_paths": ["generated"]}), str(tmp_git_repo), "testuser", None) + with pytest.raises(VerificationError, match="exclusions"): + _verify(str(path), _policy(tmp_path, {"required_fingerprint_excluded_paths": []}), str(tmp_git_repo), "testuser", None) + + +def test_test_manifest_policy(tmp_path, tmp_git_repo, signer_with_key): + import subprocess + + manifest = tmp_git_repo / "gpu-tests.txt" + manifest.write_text("tests/test_add.py::test_add\n") + subprocess.run(["git", "add", "gpu-tests.txt"], cwd=tmp_git_repo, check=True) + subprocess.run(["git", "commit", "-m", "manifest"], cwd=tmp_git_repo, check=True, capture_output=True) + signer, public_key = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer) + with _mock_github_keys(public_key): + _verify(str(path), _policy(tmp_path, {"required_test_manifest": "gpu-tests.txt"}), str(tmp_git_repo), "testuser", None) + manifest.write_text("tests/test_add.py::other\n") + with pytest.raises(VerificationError, match="test manifest"): + _verify(str(path), _policy(tmp_path, {"required_test_manifest": "gpu-tests.txt", "allow_dirty": True}), str(tmp_git_repo), "testuser", None) + + +def test_failed_session_and_comparison_check_rejected( + tmp_path, tmp_git_repo, signer_with_key +): + signer, public_key = signer_with_key + path = _make_receipt( + tmp_path, tmp_git_repo, signer, + mutate=lambda p: p["session"].__setitem__("outcome", "failed"), + ) + with _mock_github_keys(public_key), pytest.raises(VerificationError, match="session did not pass"): + _verify(str(path), None, str(tmp_git_repo), "testuser", None) + path = _make_receipt( + tmp_path, tmp_git_repo, signer, + mutate=lambda p: p["tests"][0].__setitem__( + "checks", [{"name": "x", "outcome": "failed"}] + ), + ) + with _mock_github_keys(public_key), pytest.raises(VerificationError, match="failed comparison"): + _verify(str(path), None, str(tmp_git_repo), "testuser", None) + + +def test_bad_receipt_files_and_schema(tmp_path, tmp_git_repo): + with pytest.raises(VerificationError, match="cannot read receipt"): + _verify(str(tmp_path / "missing"), None, str(tmp_git_repo), None, None) + bad = tmp_path / "bad.json" + bad.write_text("[]") + with pytest.raises(VerificationError, match="JSON object"): + _verify(str(bad), None, str(tmp_git_repo), None, None) + bad.write_text('{"schema_version": 99}') + with pytest.raises(VerificationError, match="unknown schema"): + _verify(str(bad), None, str(tmp_git_repo), None, None) + + +def test_signature_metadata_and_encoding_errors( + tmp_path, tmp_git_repo, signer_with_key +): + signer, public_key = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer) + receipt = json.loads(path.read_text()) + receipt["signature"] = "bad" + path.write_text(json.dumps(receipt)) + with pytest.raises(VerificationError, match="signature.value"): + _verify(str(path), None, str(tmp_git_repo), None, None) + receipt["signature"] = {"value": "%%%"} + path.write_text(json.dumps(receipt)) + with pytest.raises(VerificationError, match="valid base64"): + _verify(str(path), None, str(tmp_git_repo), None, None) + + +def test_verifier_wrapper_reports_failure(tmp_path, tmp_git_repo, capsys): + assert not verify_receipt(str(tmp_path / "missing"), repo_root=str(tmp_git_repo)) + assert "FAIL" in capsys.readouterr().err + + +def test_future_and_invalid_age_policy(tmp_path, tmp_git_repo, signer_with_key): + signer, public_key = signer_with_key + future = (datetime.datetime.now(datetime.UTC) + datetime.timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + path = _make_receipt(tmp_path, tmp_git_repo, signer, ended_at=future) + with _mock_github_keys(public_key), pytest.raises(VerificationError, match="future"): + _verify(str(path), None, str(tmp_git_repo), "testuser", None) + path = _make_receipt(tmp_path, tmp_git_repo, signer) + with _mock_github_keys(public_key), pytest.raises(VerificationError, match="non-negative"): + _verify(str(path), _policy(tmp_path, {"max_age_days": "bad"}), str(tmp_git_repo), "testuser", None) + with _mock_github_keys(public_key), pytest.raises(VerificationError, match="non-negative"): + _verify(str(path), None, str(tmp_git_repo), "testuser", True) + + +def test_missing_node_manifest_is_reported(tmp_path): + from pytest_gpu_proof.verify import _load_node_ids + + with pytest.raises(VerificationError, match="cannot read node-id"): + _load_node_ids(tmp_path / "missing") + + +def test_signer_metadata_must_match_verifying_key( + tmp_path, tmp_git_repo, signer_with_key +): + signer, public_key = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer) + receipt = json.loads(path.read_text()) + + receipt["signer"]["github_user"] = "" + path.write_text(json.dumps(receipt)) + with pytest.raises(VerificationError, match="identity is incomplete"): + _verify(str(path), None, str(tmp_git_repo), None, None) + + receipt["signer"]["github_user"] = "testuser" + receipt["signer"]["key_fingerprint"] = "SHA256:wrong" + path.write_text(json.dumps(receipt)) + with patch("pytest_gpu_proof.verify.find_verifying_github_key", return_value=public_key): + with pytest.raises(VerificationError, match="key_fingerprint"): + _verify(str(path), None, str(tmp_git_repo), None, None) + + receipt["signer"]["key_fingerprint"] = signer.key_fingerprint() + receipt["signer"]["algorithm"] = "rsa-pss-sha256" + path.write_text(json.dumps(receipt)) + with patch("pytest_gpu_proof.verify.find_verifying_github_key", return_value=public_key): + with pytest.raises(VerificationError, match="algorithm"): + _verify(str(path), None, str(tmp_git_repo), None, None) + + +def test_signer_key_lookup_error_is_clean( + tmp_path, tmp_git_repo, signer_with_key +): + from pytest_gpu_proof.signers.base import VerifierError + + signer, _ = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer) + with patch( + "pytest_gpu_proof.verify.find_verifying_github_key", + side_effect=VerifierError("offline"), + ), pytest.raises(VerificationError, match="offline"): + _verify(str(path), None, str(tmp_git_repo), "testuser", None) + + +def test_legacy_signature_paths(tmp_path, tmp_git_repo, signer_with_key): + from pytest_gpu_proof.config import GpuProofConfig + from pytest_gpu_proof.signers.base import VerifierError + + signer, _ = signer_with_key + os.chdir(tmp_git_repo) + config = GpuProofConfig(enabled=True, fingerprint_paths=["src", "tests"]) + payload = build_receipt_payload( + config, + [{"node_id": "t::x", "outcome": "passed", "checks": []}], + _utcstamp(), _utcstamp(), + ) + payload["schema_version"] = "2" + receipt = finalize_receipt(payload, signer) + path = tmp_path / "legacy.json" + write_receipt(receipt, str(path)) + public_key = signer._public_key + with _mock_github_keys(public_key): + _verify(str(path), None, str(tmp_git_repo), None, None) + other_key = Ed25519PrivateKey.generate().public_key() + with _mock_github_keys(other_key): + with pytest.raises(VerificationError, match="signature does not match"): + _verify(str(path), None, str(tmp_git_repo), None, None) + with patch( + "pytest_gpu_proof.verify.find_verifying_github_key", + side_effect=VerifierError("network"), + ), pytest.raises(VerificationError, match="network"): + _verify(str(path), None, str(tmp_git_repo), None, None) + receipt["signature"].pop("signer") + receipt["repo"]["github_username"] = None + path.write_text(json.dumps(receipt)) + with pytest.raises(VerificationError, match="determine legacy"): + _verify(str(path), None, str(tmp_git_repo), None, None) + + +def _legacy_receipt(tmp_path, tmp_git_repo, signer): + from pytest_gpu_proof.config import GpuProofConfig + + os.chdir(tmp_git_repo) + config = GpuProofConfig(enabled=True, fingerprint_paths=["src", "tests"]) + payload = build_receipt_payload( + config, + [ + { + "node_id": "tests/test_add.py::test_add", + "outcome": "passed", + "duration_s": 0.01, + "checks": [], + } + ], + _utcstamp(), _utcstamp(), + ) + payload["schema_version"] = "2" + receipt = finalize_receipt(payload, signer) + path = tmp_path / "legacy.json" + write_receipt(receipt, str(path)) + return path, receipt + + +def test_legacy_fingerprint_is_bound_to_the_verifying_key( + tmp_path, tmp_git_repo, signer_with_key +): + """A spoofed signature.key_fingerprint (unsigned envelope) must not be + able to satisfy a restricted key allowlist — the policy check uses the + fingerprint of the key that actually verified.""" + from pytest_gpu_proof.signers.ed25519 import _public_key_fingerprint + + signer, public_key = signer_with_key + path, receipt = _legacy_receipt(tmp_path, tmp_git_repo, signer) + real_fp = _public_key_fingerprint(public_key) + + # Honest receipt against a policy allowing the real key: passes. + policy = tmp_path / "policy.json" + policy.write_text(json.dumps({ + "signer_mode": "restricted", + "allowed_key_fingerprints": [real_fp], + "allow_dirty": True, + })) + with _mock_github_keys(public_key): + _verify(str(path), str(policy), str(tmp_git_repo), None, None) + + # Attacker pastes an allowed fingerprint into the unsigned envelope while + # signing with a different (non-allowlisted) key: rejected. + receipt["signature"]["key_fingerprint"] = "SHA256:allowed-but-spoofed" + path.write_text(json.dumps(receipt)) + policy.write_text(json.dumps({ + "signer_mode": "restricted", + "allowed_key_fingerprints": ["SHA256:allowed-but-spoofed"], + "allow_dirty": True, + })) + with _mock_github_keys(public_key): + with pytest.raises(VerificationError, match="does not match the verifying key"): + _verify(str(path), str(policy), str(tmp_git_repo), None, None) + + +def test_restricted_policy_rejects_key_outside_allowlist( + tmp_path, tmp_git_repo, signer_with_key +): + signer, public_key = signer_with_key + path, _ = _legacy_receipt(tmp_path, tmp_git_repo, signer) + policy = tmp_path / "policy.json" + policy.write_text(json.dumps({ + "signer_mode": "restricted", + "allowed_key_fingerprints": ["SHA256:someone-else"], + "allow_dirty": True, + })) + with _mock_github_keys(public_key): + with pytest.raises(VerificationError, match="not allowed by repository policy"): + _verify(str(path), str(policy), str(tmp_git_repo), None, None) + + +def test_restricted_policy_rejects_unsigned_receipt( + tmp_path, tmp_git_repo, signer_with_key +): + signer, _ = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer, sign=False) + policy = tmp_path / "policy.json" + policy.write_text(json.dumps({"signer_mode": "restricted", "allowed_signers": ["testuser"]})) + with pytest.raises(VerificationError, match="unsigned receipts are not acceptable"): + _verify( + str(path), str(policy), str(tmp_git_repo), None, None, allow_unsigned=True + ) + + +def test_min_schema_policy(tmp_path, tmp_git_repo, signer_with_key): + signer, public_key = signer_with_key + path, _ = _legacy_receipt(tmp_path, tmp_git_repo, signer) + policy = tmp_path / "policy.json" + policy.write_text(json.dumps({"min_schema": 3})) + with pytest.raises(VerificationError, match="below the policy minimum"): + _verify(str(path), str(policy), str(tmp_git_repo), None, None) + + schema3 = _make_receipt(tmp_path, tmp_git_repo, signer) + with _mock_github_keys(public_key): + _verify(str(schema3), str(policy), str(tmp_git_repo), None, None) + + policy.write_text(json.dumps({"min_schema": 4})) + with pytest.raises(VerificationError, match="must be 1, 2, or 3"): + _verify(str(schema3), str(policy), str(tmp_git_repo), None, None) + + +def test_schema3_override_must_match_signed_identity(good_receipt, tmp_git_repo): + path, public_key = good_receipt + with _mock_github_keys(public_key): + with pytest.raises(VerificationError, match="does not match the signed identity"): + _verify(str(path), None, str(tmp_git_repo), "someone-else", None) + + +def test_untracked_receipt_in_repo_does_not_fail_clean_tree_check( + tmp_path, tmp_git_repo, signer_with_key +): + """The canonical flow — receipt lands untracked at the repo root right + after a run — must pass the schema-3 clean-tree verification gate.""" + signer, public_key = signer_with_key + built = _make_receipt(tmp_path, tmp_git_repo, signer) + # The name must dodge the fixture's .git/info/exclude (*.json) so the + # receipt is genuinely visible to `git status` — as in real repos. + in_repo = tmp_git_repo / "gpu-proof.receipt" + in_repo.write_text(built.read_text()) + with _mock_github_keys(public_key): + _verify(str(in_repo), None, str(tmp_git_repo), "testuser", None) + # Any OTHER untracked file still counts as dirt. + stray = tmp_git_repo / "stray.txt" + stray.write_text("dirt") + with _mock_github_keys(public_key): + with pytest.raises(VerificationError, match="clean recording"): + _verify(str(in_repo), None, str(tmp_git_repo), "testuser", None) + + +def test_receipt_outside_repo_verifies_clean_tree( + tmp_path_factory, tmp_path, tmp_git_repo, signer_with_key +): + """A receipt held outside the repo (e.g. a downloaded CI artifact) has no + dirty-scan exclusion to compute and the clean tree still verifies.""" + signer, public_key = signer_with_key + built = _make_receipt(tmp_path, tmp_git_repo, signer) + outside = tmp_path_factory.mktemp("artifacts") / "gpu-proof.json" + outside.write_text(built.read_text()) + with _mock_github_keys(public_key): + _verify(str(outside), None, str(tmp_git_repo), "testuser", None) + + +def test_tree_and_git_failures_are_rejected( + tmp_path, tmp_git_repo, signer_with_key, monkeypatch +): + from pytest_gpu_proof.gitutils import GitError + + signer, public_key = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer) + receipt = json.loads(path.read_text()) + + receipt["fingerprint"]["algorithm"] = "unknown" + receipt["signature"] = None + path.write_text(json.dumps(receipt)) + with pytest.raises(VerificationError, match="unsupported fingerprint"): + _verify(str(path), None, str(tmp_git_repo), None, None, allow_unsigned=True) + + path = _make_receipt(tmp_path, tmp_git_repo, signer, sign=False) + receipt = json.loads(path.read_text()) + receipt["fingerprint"]["digest"] = "0" * 64 + path.write_text(json.dumps(receipt)) + with pytest.raises(VerificationError, match="source fingerprint"): + _verify(str(path), None, str(tmp_git_repo), None, None, allow_unsigned=True) + + path = _make_receipt(tmp_path, tmp_git_repo, signer, sign=False) + receipt = json.loads(path.read_text()) + receipt["repo"]["commit_sha"] = "1" * 40 + path.write_text(json.dumps(receipt)) + with pytest.raises(VerificationError, match="not the current commit"): + _verify(str(path), None, str(tmp_git_repo), None, None, allow_unsigned=True) + + path = _make_receipt(tmp_path, tmp_git_repo, signer, sign=False) + receipt = json.loads(path.read_text()) + receipt["repo"]["dirty"] = True + path.write_text(json.dumps(receipt)) + with pytest.raises(VerificationError, match="clean recording"): + _verify(str(path), None, str(tmp_git_repo), None, None, allow_unsigned=True) + + monkeypatch.setattr("pytest_gpu_proof.verify.require_repository", lambda root: (_ for _ in ()).throw(GitError("git broke"))) + with pytest.raises(VerificationError, match="git broke"): + _verify(str(path), None, str(tmp_git_repo), None, None, allow_unsigned=True) + + +def test_dirty_query_error_is_rejected( + tmp_path, tmp_git_repo, signer_with_key, monkeypatch +): + from pytest_gpu_proof.gitutils import GitError + + signer, public_key = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer) + monkeypatch.setattr("pytest_gpu_proof.verify.is_dirty", lambda *a, **k: (_ for _ in ()).throw(GitError("status broke"))) + with _mock_github_keys(public_key), pytest.raises(VerificationError, match="status broke"): + _verify(str(path), None, str(tmp_git_repo), "testuser", None)