diff --git a/.castiron.stats.yml b/.castiron.stats.yml index 5ffde321fa..1fa8b6c928 100644 --- a/.castiron.stats.yml +++ b/.castiron.stats.yml @@ -4,3 +4,5 @@ openapi_spec_hash: dd725fb7d43ceec7fb2de6f8713d14b6 openapi_transformed_spec_hash: 10930179c5f116288e24e0c6fda46559 config_hash: 85382dd94c503b5d225adc7636a77c9f codegen_sha: 6e990f52e3cbdeaae602710a1b0f2a2c944a5c35 +codegen_hash: f041a41892c7d3be7fb000e512a610e43ef265e8422e3030d36861f17b9fe515 +public_codegen_sha: 98c2f501bcbf0ad04cdf54f11aafff44dc275d6f diff --git a/.github/workflows/castiron-custom-code.yml b/.github/workflows/castiron-custom-code.yml new file mode 100644 index 0000000000..5357fe535c --- /dev/null +++ b/.github/workflows/castiron-custom-code.yml @@ -0,0 +1,105 @@ +name: Castiron custom code + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +# These experimental refs are restricted to trusted SDK maintainers. +# The workflow definition is trusted because its branch writers are trusted. +concurrency: + group: castiron-custom-code-${{ github.event.pull_request.number }} + cancel-in-progress: false + +env: + REPORTER_SHA256: 03959d3f381df9459075ecd72004cb034d132866664a2154e3ed760dabb4e4cc + +jobs: + report: + name: Castiron / baseline consistency + if: github.event.pull_request.head.repo.full_name == github.repository && contains(fromJSON(vars.CASTIRON_CUSTOM_CODE_BRANCHES || '[]'), github.event.pull_request.head.ref) + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out the protected test branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Verify the reviewed reporter + run: printf '%s %s\n' "$REPORTER_SHA256" scripts/castiron/custom_code_report.py | sha256sum --check --strict + + - name: Test hash mismatch and snapshot isolation + run: python3 scripts/castiron/test_custom_code_report.py + + - name: Validate the codegen hash and report custom code + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PUBLIC_REPOSITORY: ${{ !github.event.repository.private }} + run: | + git fetch --quiet --no-tags origin "$BASE_SHA" "$HEAD_SHA" + mode=() + if [[ "$PUBLIC_REPOSITORY" == true ]]; then mode=(--public); fi + python3 -I scripts/castiron/custom_code_report.py report \ + --base "$BASE_SHA" --head "$HEAD_SHA" \ + --fetch --require-head-hash "${mode[@]}" \ + --out "$RUNNER_TEMP/castiron-custom-code" + + - name: Add the report to the run summary + if: always() + run: | + if test -f "$RUNNER_TEMP/castiron-custom-code/summary.md"; then + cat "$RUNNER_TEMP/castiron-custom-code/summary.md" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload report and current custom-code patch + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: castiron-custom-code-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/castiron-custom-code/ + if-no-files-found: error + retention-days: 7 + + comment: + name: Update custom-code comment + needs: report + if: always() && !cancelled() && needs.report.result != 'skipped' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + actions: read + pull-requests: write + steps: + - name: Check out the protected test branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Verify the reviewed publisher before execution + run: printf '%s %s\n' "$REPORTER_SHA256" scripts/castiron/custom_code_report.py | sha256sum --check --strict + + - name: Download this run's report + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: castiron-custom-code-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/castiron-custom-code + + - name: Create or update the single report comment + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + run: | + python3 -I scripts/castiron/custom_code_report.py comment \ + --report "$RUNNER_TEMP/castiron-custom-code/report.json" \ + --repository "$REPOSITORY" --pr "$PR_NUMBER" --run-id "$RUN_ID" \ + --run-attempt "$RUN_ATTEMPT" diff --git a/scripts/castiron/README.md b/scripts/castiron/README.md new file mode 100644 index 0000000000..f17c632c8a --- /dev/null +++ b/scripts/castiron/README.md @@ -0,0 +1,22 @@ +# Castiron custom-code reporting + +DO NOT MERGE: draft workflow experiment. This manually vendored reporter uses +Python, Git, and `gh`; it does not execute SDK code. + +Run `python3 scripts/castiron/test_custom_code_report.py` for focused tests. +Use the report comment’s expander to download the exact custom patch or reproduce +it from pinned revisions. A public checkout uses `report --public` and resolves +only public codegen snapshots. It never needs access to another repository. + +The hash format is specified in the reporter’s module docstring. It covers Git +paths, modes, and blob bytes, excluding `.github/actions/` and +`.github/workflows/`. The head must record a matching `codegen_hash`. Older base +revisions can use the verified snapshot lineage with a legacy notice. + +The experimental workflow is limited to explicitly protected maintainer-owned +branches listed in the repository variable `CASTIRON_CUSTOM_CODE_BRANCHES` +(a JSON array). Its workflow definition is trusted because those branch writers and +ruleset administrators are trusted. Before execution, both jobs verify the +reporter’s reviewed SHA-256. This content pin is defense in depth, not a +substitute for the branch restrictions. General contributor/fork support needs +a publisher defined on a trusted default branch. diff --git a/scripts/castiron/custom_code_report.py b/scripts/castiron/custom_code_report.py new file mode 100644 index 0000000000..6e2c4cd348 --- /dev/null +++ b/scripts/castiron/custom_code_report.py @@ -0,0 +1,837 @@ +#!/usr/bin/env python3 +# ruff: noqa: I001 +# Vendored verbatim into SDK repositories with different import-order policies. +"""Standalone, vendorable Castiron generated-content hashing and reporting. + +Hash v1: SHA256(domain || u64(entry count) || entries), where each entry is +u64(path length) || path bytes || six ASCII Git mode bytes || u64(blob length) +|| blob bytes. Integers are unsigned big-endian. Entries sort by raw Git path. +Only .github/actions/ and .github/workflows/ are excluded. No SDK imports. +""" + +from __future__ import annotations + +import argparse +import base64 +import collections +import hashlib +import html +import json +import os +import re +import struct +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +DOMAIN = b"castiron-codegen-v1\0" +MARKER = "" +RUN_MARKER = re.compile(r"") +SHA = re.compile(r"[0-9a-f]{40}\Z") +HASH = re.compile(r"[0-9a-f]{64}\Z") +REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z") +EXCLUDED = (b".github/actions/", b".github/workflows/") +PUBLIC_BLOCKED_PATHS = { + b".castiron", + b".github", + b".gitmodules", + b"CODEOWNERS", + b"SECURITY.md", + b"docs/CODEOWNERS", + b"docs/SECURITY.md", +} +LABELS = { + "newly_customized": "Newly customized", + "removed": "Customization removed", + "existing_changed": "Existing customization changed", + "baseline_changed": "Generated baseline changed", + "unchanged": "Existing customization unchanged", + "no_longer_generated": "No longer generated", + "newly_generation_owned": "Now generated; already existed", +} + + +class ReportError(ValueError): + """The report cannot establish a trustworthy comparison.""" + + +@dataclass(frozen=True) +class Entry: + mode: bytes + oid: str + + +def git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: + env = os.environ.copy() + # Preserve ordinary Git authentication/configuration, but do not inherit + # repository routing from the caller (for example, a Git hook). + for name in ( + "GIT_DIR", + "GIT_COMMON_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_PREFIX", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + ): + env.pop(name, None) + env["GIT_NO_REPLACE_OBJECTS"] = "1" + result = subprocess.run( + ["git", "--no-replace-objects", "-C", str(repo), *args], + input=input_bytes, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + check=False, + ) + if result.returncode: + raise ReportError(f"git {args[0]} failed: {result.stderr.decode(errors='replace').strip()}") + return result.stdout + + +def require_sha(value: str) -> str: + if not SHA.fullmatch(value): + raise ReportError("expected a full Git commit SHA") + return value + + +def included(path: bytes) -> bool: + return not path.startswith(EXCLUDED) + + +def tree_entries(repo: Path, commit: str) -> dict[bytes, Entry]: + result: dict[bytes, Entry] = {} + for record in git(repo, "ls-tree", "-r", "-z", require_sha(commit)).split(b"\0"): + if not record: + continue + metadata, path = record.split(b"\t", 1) + mode, kind, oid = metadata.split(b" ") + if not included(path): + continue + if kind != b"blob" or mode not in {b"100644", b"100755", b"120000"}: + raise ReportError(f"unsupported Git entry: {path!r}") + result[path] = Entry(mode, oid.decode("ascii")) + return result + + +def read_blobs(repo: Path, entries: dict[bytes, Entry]) -> dict[str, bytes]: + oids = sorted({entry.oid for entry in entries.values()}) + output = git( + repo, "cat-file", "--batch", input_bytes="".join(f"{oid}\n" for oid in oids).encode() + ) + result: dict[str, bytes] = {} + offset = 0 + for oid in oids: + end = output.index(b"\n", offset) + actual, kind, size = output[offset:end].split(b" ") + if actual.decode() != oid or kind != b"blob": + raise ReportError("unexpected Git object response") + offset = end + 1 + length = int(size) + result[oid] = output[offset : offset + length] + offset += length + if output[offset : offset + 1] != b"\n": + raise ReportError("truncated Git object response") + offset += 1 + if offset != len(output): + raise ReportError("unexpected trailing Git object data") + return result + + +def hash_codegen_commit(repo: Path, commit: str) -> str: + entries = tree_entries(repo, commit) + blobs = read_blobs(repo, entries) + digest = hashlib.sha256(DOMAIN + struct.pack(">Q", len(entries))) + for path, entry in sorted(entries.items()): + content = blobs[entry.oid] + digest.update(struct.pack(">Q", len(path))) + digest.update(path) + digest.update(entry.mode) + digest.update(struct.pack(">Q", len(content))) + digest.update(content) + return digest.hexdigest() + + +def read_stats(repo: Path, commit: str) -> dict[str, str]: + entry = git(repo, "ls-tree", require_sha(commit), "--", ".castiron.stats.yml") + if not entry.startswith((b"100644 blob ", b"100755 blob ")): + raise ReportError("Castiron stats must be a regular Git file") + text = git(repo, "show", f"{require_sha(commit)}:.castiron.stats.yml").decode("utf-8") + values: dict[str, str] = {} + for line in text.splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + key, sep, value = line.partition(":") + if not sep or key.strip() in values: + raise ReportError("invalid or duplicate stats field") + values[key.strip()] = value.strip().strip("\"'") + if values.get("schema_version") != "1": + raise ReportError("unsupported Castiron stats schema") + require_sha(values.get("codegen_sha", "")) + if not re.fullmatch( + r"[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}", + values.get("generation_id", ""), + ): + raise ReportError("invalid generation ID") + if "codegen_hash" in values and not HASH.fullmatch(values["codegen_hash"]): + raise ReportError("invalid codegen_hash") + if "public_codegen_sha" in values: + require_sha(values["public_codegen_sha"]) + return values + + +def public_metadata(repo: Path, commit: str) -> dict[str, str]: + message = git(repo, "show", "-s", "--format=%B", require_sha(commit)).decode("utf-8") + fields: dict[str, str] = {} + for key in ("Version", "Generation-ID", "Hash", "Branch"): + matches = re.findall(rf"^Castiron-Public-Codegen-{key}: (.+)$", message, re.MULTILINE) + if len(matches) != 1: + raise ReportError("invalid public codegen metadata") + fields[key] = matches[0] + if fields["Version"] != "1" or not HASH.fullmatch(fields["Hash"]): + raise ReportError("unsupported public codegen metadata") + if not fields["Branch"].startswith("codegen/"): + raise ReportError("invalid public codegen branch") + git(repo, "check-ref-format", f"refs/heads/{fields['Branch']}") + return fields + + +def resolve_public_baseline( + repo: Path, revision: str, *, fetch: bool, require_hash: bool, lineage: str | None = None +) -> dict[str, Any]: + stats = read_stats(repo, revision) + commit = stats.get("public_codegen_sha") + if commit is None: + # Bootstrap old public main without changing its schema-v1 stats. Search + # only the already-verified public snapshot lineage supplied by PR head. + if lineage is None: + raise ReportError("stats do not contain public_codegen_sha") + for candidate in git(repo, "rev-list", require_sha(lineage)).decode().splitlines(): + if public_metadata(repo, candidate)["Generation-ID"] == stats["generation_id"]: + commit = candidate + break + if commit is None: + raise ReportError("public snapshot lineage does not contain the base generation") + elif fetch: + git(repo, "fetch", "--quiet", "--no-tags", "origin", commit) + metadata = public_metadata(repo, commit) + if metadata["Generation-ID"] != stats["generation_id"]: + raise ReportError("public codegen generation ID does not match stats") + ref = f"refs/heads/{metadata['Branch']}" + tracking = f"refs/castiron/report/{hashlib.sha256(ref.encode()).hexdigest()}" + if fetch: + git(repo, "fetch", "--quiet", "--no-tags", "origin", f"+{ref}:{tracking}") + else: + tracking = f"refs/remotes/origin/{metadata['Branch']}" + git(repo, "merge-base", "--is-ancestor", commit, tracking) + actual = hash_codegen_commit(repo, commit) + if actual != metadata["Hash"]: + raise ReportError("public codegen content does not match snapshot metadata") + expected = stats.get("codegen_hash") + if expected is None and require_hash: + raise ReportError("head stats do not contain codegen_hash") + if expected is not None and expected != actual: + raise ReportError(f"codegen_hash mismatch: expected {expected}, got {actual}") + return { + "commit": commit, + "generation_id": stats["generation_id"], + "hash": actual, + "hash_recorded": expected is not None, + } + + +def public_snapshot_entries( + source: Path, commit: str, destination: Path, public_base: str +) -> dict[bytes, Entry]: + entries = tree_entries(source, commit) + public_entries = tree_entries(destination, public_base) + for path in entries: + if path in PUBLIC_BLOCKED_PATHS or path.startswith((b".castiron/", b".github/")): + raise ReportError(f"refusing to publish private or governance snapshot path: {path!r}") + if path not in public_entries: + raise ReportError(f"pilot snapshot path is not present on public main: {path!r}") + return entries + + +def copy_generated_tree(source: Path, commit: str, destination: Path, public_base: str) -> str: + """Transfer approved blobs/trees only, never source commits or ancestors.""" + entries = public_snapshot_entries(source, commit, destination, public_base) + for oid, blob in read_blobs(source, entries).items(): + actual = git(destination, "hash-object", "-w", "--stdin", input_bytes=blob).decode().strip() + if actual != oid: + raise ReportError("source and destination Git object formats differ") + directories: dict[bytes, list[bytes]] = {b"": []} + for path, entry in entries.items(): + parent, _, name = path.rpartition(b"/") + directories.setdefault(parent, []).append( + entry.mode + b" blob " + entry.oid.encode() + b"\t" + name + b"\0" + ) + while parent: + parent = parent.rpartition(b"/")[0] + directories.setdefault(parent, []) + for directory in sorted(directories, key=lambda p: (p.count(b"/"), len(p)), reverse=True): + tree = git( + destination, "mktree", "-z", input_bytes=b"".join(directories[directory]) + ).strip() + if not directory: + return tree.decode() + parent, _, name = directory.rpartition(b"/") + directories[parent].append(b"040000 tree " + tree + b"\t" + name + b"\0") + raise ReportError("could not construct public generated tree") + + +def prepare_public_snapshot( + source: Path, + source_base: str, + source_head: str, + destination: Path, + public_base: str, + branch: str, +) -> dict[str, str]: + """Prepare the unchanged-generation promotion pilot; never push anything.""" + before, after = read_stats(source, source_base), read_stats(source, source_head) + public = read_stats(destination, public_base) + for key in ("generation_id", "codegen_sha"): + if before[key] != after[key] or before[key] != public[key]: + raise ReportError( + "custom-code promotion pilot requires the unchanged public generation" + ) + baseline = resolve_baseline(source, source_head, fetch=True, require_hash=True) + public_snapshot_entries(source, baseline["commit"], destination, public_base) + codegen_branch = f"codegen/{branch}" + ref = f"refs/heads/{codegen_branch}" + git(destination, "check-ref-format", ref) + remote = git(destination, "ls-remote", "--refs", "origin", ref).decode().splitlines() + previous = require_sha(remote[0].split()[0]) if remote else "" + if previous: + git(destination, "fetch", "--quiet", "--no-tags", "origin", previous) + metadata = public_metadata(destination, previous) + if ( + metadata["Generation-ID"] != after["generation_id"] + or metadata["Branch"] != codegen_branch + or metadata["Hash"] != baseline["hash"] + or hash_codegen_commit(destination, previous) != baseline["hash"] + or git(destination, "rev-list", "--count", previous).strip() != b"1" + ): + raise ReportError("existing public codegen branch does not match the pilot generation") + snapshot = previous + else: + tree = copy_generated_tree(source, baseline["commit"], destination, public_base) + message = ( + "Castiron generated snapshot\n\n" + "Castiron-Public-Codegen-Version: 1\n" + f"Castiron-Public-Codegen-Generation-ID: {after['generation_id']}\n" + f"Castiron-Public-Codegen-Hash: {baseline['hash']}\n" + f"Castiron-Public-Codegen-Branch: {codegen_branch}\n" + ) + snapshot = ( + git( + destination, + "-c", + "user.name=Castiron", + "-c", + "user.email=noreply@openai.com", + "commit-tree", + tree, + "-m", + message, + ) + .decode() + .strip() + ) + require_sha(snapshot) + if git(destination, "rev-list", "--count", snapshot).strip() != b"1": + raise ReportError("initial public snapshot unexpectedly has ancestors") + if hash_codegen_commit(destination, snapshot) != baseline["hash"]: + raise ReportError("public snapshot content hash differs from private checkpoint") + stats_path = destination / ".castiron.stats.yml" + current = git(source, "show", f"{require_sha(source_head)}:.castiron.stats.yml").decode("utf-8") + lines = [line for line in current.splitlines() if not line.startswith("public_codegen_sha:")] + stats_path.write_text("\n".join(lines) + f"\npublic_codegen_sha: {snapshot}\n") + git(destination, "add", "--", ".castiron.stats.yml") + return { + "commit": snapshot, + "branch": codegen_branch, + "previous": previous, + "hash": baseline["hash"], + } + + +def resolve_baseline( + repo: Path, revision: str, *, fetch: bool, require_hash: bool +) -> dict[str, Any]: + stats = read_stats(repo, revision) + commit = stats["codegen_sha"] + if fetch: + git(repo, "fetch", "--quiet", "--no-tags", "origin", commit) + message = git(repo, "show", "-s", "--format=%B", commit).decode("utf-8") + match = re.search(r"^Generation metadata: ([A-Za-z0-9+/=]+)$", message, re.MULTILINE) + if not match: + raise ReportError("codegen checkpoint has no generation metadata") + try: + decoded: object = json.loads(base64.b64decode(match[1], validate=True)) + except (ValueError, TypeError) as exc: + raise ReportError("invalid codegen checkpoint metadata") from exc + if not isinstance(decoded, dict): + raise ReportError("invalid codegen checkpoint metadata") + metadata = cast(dict[str, Any], decoded) + if metadata.get("generation_id") != stats["generation_id"]: + raise ReportError("codegen checkpoint generation ID does not match stats") + branch = metadata.get("source_branch") + if not isinstance(branch, str) or not branch: + raise ReportError("codegen checkpoint has no source branch") + ref = f"refs/heads/codegen/{branch}" + git(repo, "check-ref-format", ref) + tracking = f"refs/castiron/report/{hashlib.sha256(ref.encode()).hexdigest()}" + if fetch: + git(repo, "fetch", "--quiet", "--no-tags", "origin", f"+{ref}:{tracking}") + else: + tracking = f"refs/remotes/origin/codegen/{branch}" + git(repo, "merge-base", "--is-ancestor", commit, tracking) + actual = hash_codegen_commit(repo, commit) + expected = stats.get("codegen_hash") + if expected is None and require_hash: + raise ReportError("head stats do not contain codegen_hash") + if expected is not None and actual != expected: + raise ReportError(f"codegen_hash mismatch: expected {expected}, got {actual}") + return { + "commit": commit, + "generation_id": stats["generation_id"], + "hash": actual, + "hash_recorded": expected is not None, + } + + +def numstats(repo: Path, generated: str, integrated: str) -> dict[bytes, tuple[str, str]]: + result: dict[bytes, tuple[str, str]] = {} + output = git( + repo, + "diff", + "--numstat", + "-z", + "--no-renames", + "--no-ext-diff", + "--no-textconv", + generated, + integrated, + "--", + ) + for record in output.split(b"\0"): + if record: + added, removed, path = record.split(b"\t", 2) + result[path] = (added.decode("ascii"), removed.decode("ascii")) + return result + + +def classify(g0: Entry | None, i0: Entry | None, g1: Entry | None, i1: Entry | None) -> str | None: + before = g0 is not None and g0 != i0 + after = g1 is not None and g1 != i1 + if g0 is not None and g1 is None and (before or i1 is not None): + return "no_longer_generated" + if g0 is None and g1 is not None and i0 is not None: + return "newly_generation_owned" + if not before and not after: + return None + if g0 is not None and g1 is not None and g0 != g1: + return "baseline_changed" + if not before: + return "newly_customized" + if not after: + return "removed" + return "unchanged" if i0 == i1 else "existing_changed" + + +def build_report( + repo: Path, + base: str, + head: str, + *, + fetch: bool = False, + require_head_hash: bool = False, + public: bool = False, +) -> tuple[dict[str, Any], bytes]: + base, head = require_sha(base), require_sha(head) + comparison_base = git(repo, "merge-base", base, head).decode().strip() + if public: + after = resolve_public_baseline(repo, head, fetch=fetch, require_hash=require_head_hash) + before = resolve_public_baseline( + repo, comparison_base, fetch=fetch, require_hash=False, lineage=after["commit"] + ) + else: + before = resolve_baseline(repo, comparison_base, fetch=fetch, require_hash=False) + after = resolve_baseline(repo, head, fetch=fetch, require_hash=require_head_hash) + g0, i0 = tree_entries(repo, before["commit"]), tree_entries(repo, comparison_base) + g1, i1 = tree_entries(repo, after["commit"]), tree_entries(repo, head) + stats = numstats(repo, after["commit"], head) + files: list[dict[str, Any]] = [] + current_paths: list[str] = [] + for path in sorted(g0.keys() | g1.keys()): + category = classify(g0.get(path), i0.get(path), g1.get(path), i1.get(path)) + if category is None: + continue + custom_before = path in g0 and g0[path] != i0.get(path) + custom_after = path in g1 and g1[path] != i1.get(path) + added, removed = stats.get(path, ("0", "0")) if custom_after else ("0", "0") + files.append( + { + "path": path.decode("utf-8"), + "category": category, + "custom_before": custom_before, + "custom_after": custom_after, + "added": added, + "removed": removed, + } + ) + if custom_after: + current_paths.append(path.decode("utf-8")) + patch = b"" + if current_paths: + patch = git( + repo, + "--literal-pathspecs", + "diff", + "--binary", + "--full-index", + "--no-renames", + "--no-ext-diff", + "--no-textconv", + "--no-color", + after["commit"], + head, + "--", + *current_paths, + ) + return { + "schema_version": 1, + "status": "ok", + "public": public, + "target_base_sha": base, + "base_sha": comparison_base, + "head_sha": head, + "before": before, + "after": after, + "files": files, + "counts": { + "before": sum(f["custom_before"] for f in files), + "after": sum(f["custom_after"] for f in files), + **dict(collections.Counter(f["category"] for f in files)), + }, + }, patch + + +def render_report( + report: dict[str, Any], + run_url: str = "", + *, + repository: str = "", + run_id: int = 0, + run_attempt: int = 0, +) -> str: + head = require_sha(report["head_sha"]) + lines = [MARKER, "", "## Castiron custom code", ""] + if report.get("status") != "ok": + reason = html.escape(str(report.get("error", "Report could not be computed"))) + lines.extend([f"⚠️ Report unavailable for `{head[:12]}`.", "", reason]) + else: + files = report["files"] + counts: collections.Counter[str] = collections.Counter() + for file in files: + if file["category"] not in LABELS: + raise ReportError("unknown report category") + counts[file["category"]] += 1 + before = sum(bool(file["custom_before"]) for file in files) + after = sum(bool(file["custom_after"]) for file in files) + base = require_sha(report["base_sha"]) + if counts["newly_customized"] == 0: + remaining = ( + f"{after} mixed file{'s' if after != 1 else ''} remain{'s' if after == 1 else ''}" + ) + changed = counts["existing_changed"] + details = [f"{changed} existing customization{'s' if changed != 1 else ''} changed"] + if counts["removed"]: + details.append(f"{counts['removed']} customizations removed") + if counts["baseline_changed"]: + details.append(f"{counts['baseline_changed']} generated baselines changed") + lines.extend( + [ + "✅ No new custom-code files detected.", + "", + remaining + "; " + "; ".join(details) + ".", + ] + ) + else: + lines.extend( + [ + f"**Mixed files: {before} → {after}**", + "", + ( + f"{counts['newly_customized']} newly customized · {counts['removed']} customizations removed · " + f"{counts['existing_changed']} existing customizations changed · {counts['baseline_changed']} generated baselines changed" + ), + ] + ) + lines.extend(["", f"Compared `{base[:12]}` → `{head[:12]}`. Generated baselines verified."]) + if not report["before"]["hash_recorded"] or not report["after"]["hash_recorded"]: + lines.extend( + [ + "", + "Legacy baseline: content hash computed from the pinned checkpoint; not yet recorded in that revision’s stats.", + ] + ) + interesting = [file for file in files if file["category"] != "unchanged"] + if interesting: + lines.extend(["", "| File | Result | Current custom patch |", "| --- | --- | --- |"]) + for file in interesting[:40]: + path = ( + html.escape(str(file["path"])) + .replace("|", "|") + .replace("\n", " ") + .replace("\r", " ") + ) + added, removed = str(file["added"]), str(file["removed"]) + if not re.fullmatch(r"\d+|-", added) or not re.fullmatch(r"\d+|-", removed): + raise ReportError("invalid line counts") + delta = f"+{added} / −{removed}" if file["custom_after"] else "None" + lines.append(f"| {path} | {LABELS[file['category']]} | {delta} |") + if len(interesting) > 40: + lines.extend( + [ + "", + f"{len(interesting) - 40} more affected files are listed in the full report.", + ] + ) + if counts["unchanged"]: + lines.extend( + [ + "", + f"
{counts['unchanged']} existing customizations unchanged", + "", + ] + ) + for file in [file for file in files if file["category"] == "unchanged"][:40]: + lines.append(f"- {html.escape(str(file['path']))}") + if counts["unchanged"] > 40: + lines.extend(["", f"{counts['unchanged'] - 40} more in the full report."]) + lines.extend(["", "
"]) + lines.extend( + [ + "", + "A changed generated baseline means this report cannot reliably identify which handwritten lines changed.", + ] + ) + target_base = require_sha(report["target_base_sha"]) + lines.extend(["", "
Inspect the custom-code diff", ""]) + if repository: + if not REPOSITORY.fullmatch(repository) or min(run_id, run_attempt) <= 0: + raise ReportError("invalid artifact download target") + artifact = f"castiron-custom-code-{run_id}-{run_attempt}" + lines.extend( + [ + "Download the exact patch produced by this run (requires repository access):", + "", + "```sh", + f"gh run download {run_id} --repo {repository} \\", + f" --name {artifact} --dir /tmp/{artifact}", + f"git apply --stat /tmp/{artifact}/custom-code.patch", + f"cat /tmp/{artifact}/custom-code.patch", + "```", + "", + ] + ) + lines.extend( + [ + "Or reproduce it from an SDK checkout containing the vendored reporter:", + "", + "```sh", + f"git fetch --no-tags origin {target_base} {head}", + "python3 scripts/castiron/custom_code_report.py report \\", + f" --base {target_base} \\", + f" --head {head} --fetch --require-head-hash{' --public' if report.get('public') else ''} \\", + f" --out /tmp/castiron-custom-code-{head[:12]}", + f"cat /tmp/castiron-custom-code-{head[:12]}/custom-code.patch", + "```", + "", + "This is the **current full custom patch** for mixed files, not an attribution of only the handwritten lines changed by this PR.", + "", + "
", + ] + ) + if run_url: + lines.extend(["", f"[Full report and patch]({run_url})"]) + return "\n".join(lines) + "\n" + + +def api(method: str, path: str, payload: dict[str, Any] | None = None) -> Any: + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if not token: + raise ReportError("GitHub token is required for comment publication") + args = ["gh", "api", "--hostname", "github.com", "--method", method, path] + if payload is not None: + args.extend(["--input", "-"]) + result = subprocess.run( + args, + input=json.dumps(payload).encode() if payload is not None else None, + capture_output=True, + check=False, + timeout=60, + ) + if result.returncode: + raise ReportError( + f"GitHub API request failed: {result.stderr.decode(errors='replace').strip()}" + ) + return json.loads(result.stdout) + + +def publish_comment( + report: dict[str, Any], repository: str, number: int, run_id: int, run_attempt: int +) -> str: + if not REPOSITORY.fullmatch(repository) or min(number, run_id, run_attempt) <= 0: + raise ReportError("invalid GitHub publication target") + root = f"repos/{repository}" + pull = api("GET", f"{root}/pulls/{number}") + if ( + pull["state"] != "open" + or pull["head"]["sha"] != report["head_sha"] + or pull["base"]["sha"] != report["target_base_sha"] + ): + return "Skipped stale report" + run = api("GET", f"{root}/actions/runs/{run_id}") + if ( + run["event"] != "pull_request" + or run["head_sha"] != report["head_sha"] + or not any(pr["number"] == number for pr in run["pull_requests"]) + ): + raise ReportError("workflow run does not match report PR/head") + if run["run_attempt"] != run_attempt: + return "Skipped stale report" + body = render_report( + report, + f"https://github.com/{repository}/actions/runs/{run_id}", + repository=repository, + run_id=run_id, + run_attempt=run_attempt, + ) + body += f"\n\n" + found = None + for page in range(1, 101): + comments = api("GET", f"{root}/issues/{number}/comments?per_page=100&page={page}") + for comment in comments: + if comment["user"]["login"] == "github-actions[bot]" and comment["body"].startswith( + MARKER + ): + found = comment + break + if found is not None or len(comments) < 100: + break + if found is not None: + previous = RUN_MARKER.search(found["body"]) + if previous and tuple(map(int, previous.groups())) > (run_id, run_attempt): + return "Skipped stale report" + if found["body"] == body: + return str(found["html_url"]) + # The workflow serializes publishers; recheck after pagination before writing. + pull = api("GET", f"{root}/pulls/{number}") + if ( + pull["state"] != "open" + or pull["head"]["sha"] != report["head_sha"] + or pull["base"]["sha"] != report["target_base_sha"] + ): + return "Skipped stale report" + if found is not None: + result = api("PATCH", f"{root}/issues/comments/{found['id']}", {"body": body}) + else: + result = api("POST", f"{root}/issues/{number}/comments", {"body": body}) + return str(result["html_url"]) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + hashing = commands.add_parser("hash") + hashing.add_argument("--repo", type=Path, default=Path.cwd()) + hashing.add_argument("--commit", required=True) + reporting = commands.add_parser("report") + reporting.add_argument("--repo", type=Path, default=Path.cwd()) + reporting.add_argument("--base", required=True) + reporting.add_argument("--head", required=True) + reporting.add_argument("--out", type=Path, required=True) + reporting.add_argument("--fetch", action="store_true") + reporting.add_argument("--require-head-hash", action="store_true") + reporting.add_argument("--public", action="store_true") + preparing = commands.add_parser("prepare-public") + preparing.add_argument("--source-repo", type=Path, required=True) + preparing.add_argument("--source-base", required=True) + preparing.add_argument("--source-head", required=True) + preparing.add_argument("--public-repo", type=Path, required=True) + preparing.add_argument("--public-base", required=True) + preparing.add_argument("--branch", required=True) + commenting = commands.add_parser("comment") + commenting.add_argument("--report", type=Path, required=True) + commenting.add_argument("--repository", required=True) + commenting.add_argument("--pr", type=int, required=True) + commenting.add_argument("--run-id", type=int, required=True) + commenting.add_argument("--run-attempt", type=int, required=True) + args = parser.parse_args() + try: + if args.command == "hash": + sys.stdout.write(hash_codegen_commit(args.repo, args.commit) + "\n") + elif args.command == "prepare-public": + sys.stdout.write( + json.dumps( + prepare_public_snapshot( + args.source_repo, + args.source_base, + args.source_head, + args.public_repo, + args.public_base, + args.branch, + ) + ) + + "\n" + ) + elif args.command == "comment": + if args.report.stat().st_size > 5_000_000: + raise ReportError("report artifact is too large") + report = json.loads(args.report.read_text()) + sys.stdout.write( + publish_comment(report, args.repository, args.pr, args.run_id, args.run_attempt) + + "\n" + ) + else: + args.out.mkdir(parents=True, exist_ok=True) + try: + report, patch = build_report( + args.repo, + args.base, + args.head, + fetch=args.fetch, + require_head_hash=args.require_head_hash, + public=args.public, + ) + except (ReportError, UnicodeError, KeyError, ValueError) as exc: + report = { + "schema_version": 1, + "status": "error", + "target_base_sha": require_sha(args.base), + "head_sha": require_sha(args.head), + "error": str(exc), + } + patch = b"" + (args.out / "report.json").write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n" + ) + (args.out / "custom-code.patch").write_bytes(patch) + summary = render_report(report) + (args.out / "summary.md").write_text(summary) + sys.stdout.write(summary) + return 0 if report["status"] == "ok" else 1 + except (ReportError, OSError, subprocess.TimeoutExpired) as exc: + sys.stderr.write(f"Castiron custom-code report: {exc}\n") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/castiron/ruff.toml b/scripts/castiron/ruff.toml new file mode 100644 index 0000000000..aca51abdb2 --- /dev/null +++ b/scripts/castiron/ruff.toml @@ -0,0 +1,2 @@ +extend = "../../pyproject.toml" +line-length = 100 diff --git a/scripts/castiron/test_custom_code_report.py b/scripts/castiron/test_custom_code_report.py new file mode 100644 index 0000000000..0caed5830e --- /dev/null +++ b/scripts/castiron/test_custom_code_report.py @@ -0,0 +1,372 @@ +from __future__ import annotations + +import os +import json +import base64 +import struct +import hashlib +import tempfile +import unittest +import subprocess +from pathlib import Path +from unittest import mock + +import custom_code_report as report + +GENERATION = "550e8400-e29b-41d4-a716-446655440000" + + +class CustomCodeTests(unittest.TestCase): + # Keep the vendored test stdlib-only on Python 3.10 (no typing.override yet). + def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.repo = Path(self.temporary.name) + self.git("init", "-q", "-b", "main") + self.git("config", "user.name", "Castiron test") + self.git("config", "user.email", "castiron@example.test") + + def git(self, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(self.repo), *args], check=True, capture_output=True, text=True + ).stdout.strip() + + def write(self, path: str, body: str) -> None: + target = self.repo / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + + def commit(self, message: str = "fixture") -> str: + self.git("add", "-A") + self.git("commit", "-q", "--allow-empty", "-m", message) + return self.git("rev-parse", "HEAD") + + def baseline(self) -> tuple[str, str]: + self.write("generated.py", "generated\n") + metadata = { + "generation_id": GENERATION, + "source_branch": "test", + "target": "openai-python", + "language": "python", + } + encoded = base64.b64encode(json.dumps(metadata).encode()).decode() + generated = self.commit(f"codegen\n\nGeneration metadata: {encoded}") + self.git("update-ref", "refs/remotes/origin/codegen/test", generated) + self.write( + ".castiron.stats.yml", + f"schema_version: 1\ngeneration_id: {GENERATION}\ncodegen_sha: {generated}\ncodegen_hash: {report.hash_codegen_commit(self.repo, generated)}\n", + ) + return generated, self.commit("integrated") + + def test_git_preserves_authentication_without_inherited_repository_routing(self) -> None: + environment = { + "GIT_CONFIG_GLOBAL": "/ordinary/gitconfig", + "GIT_ASKPASS": "/ordinary/askpass", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "credential.helper", + "GIT_CONFIG_VALUE_0": "example", + "GIT_DIR": "/wrong/repository", + "GIT_COMMON_DIR": "/wrong/common", + "GIT_INDEX_FILE": "/wrong/index", + } + with ( + mock.patch.dict(os.environ, environment), + mock.patch.object( + report.subprocess, + "run", + return_value=subprocess.CompletedProcess([], 0, stdout=b"ok"), + ) as run, + ): + self.assertEqual(report.git(self.repo, "status"), b"ok") + actual = run.call_args.kwargs["env"] + for name, value in environment.items(): + if name in {"GIT_DIR", "GIT_COMMON_DIR", "GIT_INDEX_FILE"}: + self.assertNotIn(name, actual) + else: + self.assertEqual(actual[name], value) + self.assertEqual(actual["GIT_NO_REPLACE_OBJECTS"], "1") + + def test_hash_vector_and_exclusions(self) -> None: + self.write("a", "hello\n") + first = self.commit() + encoded = ( + report.DOMAIN + + struct.pack(">Q", 1) + + struct.pack(">Q", 1) + + b"a100644" + + struct.pack(">Q", 6) + + b"hello\n" + ) + expected = hashlib.sha256(encoded).hexdigest() + self.assertEqual(report.hash_codegen_commit(self.repo, first), expected) + self.write(".github/workflows/test.yml", "ignored\n") + self.write(".github/actions/test/action.yml", "ignored\n") + second = self.commit() + self.assertEqual(report.hash_codegen_commit(self.repo, second), expected) + os.chmod(self.repo / "a", 0o755) + self.assertNotEqual(report.hash_codegen_commit(self.repo, self.commit()), expected) + self.write(".github/CODEOWNERS", "not ignored\n") + self.assertNotEqual(report.hash_codegen_commit(self.repo, self.commit()), expected) + + def test_classifications(self) -> None: + a, b, c = (report.Entry(b"100644", value * 40) for value in "abc") + cases = [ + ((a, a, b, b), None), + ((a, a, a, b), "newly_customized"), + ((a, b, a, a), "removed"), + ((a, b, a, c), "existing_changed"), + ((a, b, a, b), "unchanged"), + ((a, b, c, b), "baseline_changed"), + ((a, b, None, b), "no_longer_generated"), + ((a, a, None, a), "no_longer_generated"), + ((a, a, None, None), None), + ((None, b, a, b), "newly_generation_owned"), + ((None, a, a, a), "newly_generation_owned"), + ((a, a, b, a), "baseline_changed"), + ((a, b, b, b), "baseline_changed"), + ((a, a, a, None), "newly_customized"), + ] + for values, expected in cases: + with self.subTest(expected=expected): + self.assertEqual(report.classify(*values), expected) + + def test_report_ignores_handwritten_files_and_shows_custom_patch(self) -> None: + generated, base = self.baseline() + self.write("generated.py", "generated\n# custom\n") + self.write("handwritten.py", "handwritten\n") + head = self.commit() + result, patch = report.build_report(self.repo, base, head, require_head_hash=True) + self.assertEqual(result["counts"]["newly_customized"], 1) + self.assertEqual(result["files"][0]["path"], "generated.py") + self.assertIn(b"+# custom", patch) + self.assertNotIn(b"handwritten", patch) + self.assertEqual(result["after"]["commit"], generated) + body = report.render_report(result, repository="openai/example", run_id=42, run_attempt=2) + self.assertIn("Inspect the custom-code diff", body) + self.assertIn("gh run download 42 --repo openai/example", body) + self.assertIn("--name castiron-custom-code-42-2", body) + self.assertIn(f"--head {head}", body) + self.assertIn("current full custom patch", body) + + def test_bad_hash_and_lost_checkpoint_fail(self) -> None: + generated, base = self.baseline() + stats = self.repo / ".castiron.stats.yml" + stats.write_text( + stats.read_text().replace(report.hash_codegen_commit(self.repo, generated), "f" * 64) + ) + with self.assertRaisesRegex(report.ReportError, "codegen_hash mismatch"): + report.build_report(self.repo, base, self.commit()) + self.git("update-ref", "-d", "refs/remotes/origin/codegen/test") + with self.assertRaises(report.ReportError): + report.resolve_baseline(self.repo, base, fetch=False, require_hash=True) + + def test_stats_symlink_is_rejected(self) -> None: + self.baseline() + path = self.repo / ".castiron.stats.yml" + original = path.read_text() + path.unlink() + path.symlink_to(original) + with self.assertRaisesRegex(report.ReportError, "regular Git file"): + report.read_stats(self.repo, self.commit()) + + def test_public_snapshot_rejects_private_governance_and_unpublished_paths(self) -> None: + entry = report.Entry(b"100644", "a" * 40) + blocked = [ + b".castiron/private.json", + b".github/CODEOWNERS", + b".gitmodules", + b"CODEOWNERS", + b"SECURITY.md", + b"docs/SECURITY.md", + ] + for path in blocked: + with ( + self.subTest(path=path), + mock.patch.object( + report, "tree_entries", side_effect=[{path: entry}, {path: entry}] + ), + mock.patch.object(report, "read_blobs") as read, + ): + with self.assertRaisesRegex(report.ReportError, "private or governance"): + report.copy_generated_tree(self.repo, "a" * 40, self.repo, "b" * 40) + read.assert_not_called() + with ( + mock.patch.object(report, "tree_entries", side_effect=[{b"unreleased.py": entry}, {}]), + mock.patch.object(report, "read_blobs") as read, + ): + with self.assertRaisesRegex(report.ReportError, "not present on public main"): + report.copy_generated_tree(self.repo, "a" * 40, self.repo, "b" * 40) + read.assert_not_called() + + def test_zero_new_files_summary_reports_existing_changes(self) -> None: + _, clean = self.baseline() + self.write("generated.py", "generated\n# existing customization\n") + base = self.commit() + self.write("generated.py", "generated\n# existing customization updated\n") + result, _ = report.build_report(self.repo, base, self.commit()) + body = report.render_report(result) + self.assertIn("✅ No new custom-code files detected.", body) + self.assertIn("1 mixed file remains; 1 existing customization changed.", body) + self.assertNotIn("0 newly customized", body) + empty, _ = report.build_report(self.repo, clean, clean) + self.assertIn( + "0 mixed files remain; 0 existing customizations changed.", report.render_report(empty) + ) + + def test_public_snapshot_has_no_private_history_and_reports_without_private_remote( + self, + ) -> None: + self.write("private-only.txt", "must never be published\n") + private_ancestor = self.commit("private history") + (self.repo / "private-only.txt").unlink() + generated, base = self.baseline() + self.git("branch", "codegen/test", generated) + private_remote = self.repo / "private.git" + self.git("clone", "--bare", str(self.repo), str(private_remote)) + self.git("remote", "add", "origin", str(private_remote)) + + public = self.repo / "public" + public.mkdir() + report.git(public, "init", "-q", "-b", "main") + report.git(public, "config", "user.name", "Public test") + report.git(public, "config", "user.email", "public@example.test") + (public / "generated.py").write_text("generated\n") + (public / ".castiron.stats.yml").write_text( + (self.repo / ".castiron.stats.yml").read_text().split("codegen_hash:")[0] + ) + report.git(public, "add", ".") + report.git(public, "commit", "-qm", "already public") + public_base = report.git(public, "rev-parse", "HEAD").decode().strip() + public_remote = self.repo / "public.git" + report.git(public, "clone", "--bare", str(public), str(public_remote)) + report.git(public, "remote", "add", "origin", str(public_remote)) + snapshot = report.prepare_public_snapshot( + self.repo, base, base, public, public_base, "castiron/promotions/pr-1" + ) + self.assertNotEqual(snapshot["commit"], generated) + self.assertEqual(report.git(public, "rev-list", "--count", snapshot["commit"]), b"1\n") + self.assertEqual(report.hash_codegen_commit(public, snapshot["commit"]), snapshot["hash"]) + with self.assertRaises(report.ReportError): + report.git(public, "cat-file", "-e", private_ancestor) + self.assertNotIn( + b"private-only.txt", report.git(public, "ls-tree", "-r", snapshot["commit"]) + ) + report.git( + public, "push", "origin", f"{snapshot['commit']}:refs/heads/{snapshot['branch']}" + ) + (public / "generated.py").write_text("generated\n# custom\n") + report.git(public, "add", ".") + report.git(public, "commit", "-qm", "public customization") + public_head = report.git(public, "rev-parse", "HEAD").decode().strip() + result, patch = report.build_report( + public, public_base, public_head, fetch=True, require_head_hash=True, public=True + ) + self.assertEqual(result["counts"]["newly_customized"], 1) + self.assertIn(b"+# custom", patch) + self.assertIn(" --public", report.render_report(result)) + self.assertFalse(result["before"]["hash_recorded"]) + again = report.prepare_public_snapshot( + self.repo, base, base, public, public_base, "castiron/promotions/pr-1" + ) + self.assertEqual(again["commit"], snapshot["commit"]) + tree = report.git(public, "rev-parse", f"{snapshot['commit']}^{{tree}}").decode().strip() + message = report.git(public, "show", "-s", "--format=%B", snapshot["commit"]) + chained = ( + report.git( + public, + "commit-tree", + tree, + "-p", + snapshot["commit"], + "-F", + "-", + input_bytes=message, + ) + .decode() + .strip() + ) + report.git( + public, "push", "--force", "origin", f"{chained}:refs/heads/{snapshot['branch']}" + ) + with self.assertRaisesRegex(report.ReportError, "does not match the pilot generation"): + report.prepare_public_snapshot( + self.repo, base, base, public, public_base, "castiron/promotions/pr-1" + ) + + def test_comment_updates_existing_bot_comment_and_skips_stale(self) -> None: + _, base = self.baseline() + result, _ = report.build_report(self.repo, base, base) + calls: list[tuple[str, str, object]] = [] + + def fake_api(method: str, path: str, payload: object = None) -> object: + calls.append((method, path, payload)) + if "/pulls/" in path: + return {"state": "open", "head": {"sha": base}, "base": {"sha": base}} + if "/actions/runs/" in path: + return { + "event": "pull_request", + "head_sha": base, + "run_attempt": 1, + "pull_requests": [{"number": 1}], + } + if "/comments?" in path: + return [ + {"id": 7, "user": {"login": "someone"}, "body": report.MARKER}, + { + "id": 8, + "user": {"login": "github-actions[bot]"}, + "body": report.MARKER, + "html_url": "existing", + }, + ] + return {"html_url": "updated"} + + with mock.patch.object(report, "api", side_effect=fake_api): + self.assertEqual(report.publish_comment(result, "openai/example", 1, 2, 1), "updated") + self.assertEqual(calls[-1][:2], ("PATCH", "repos/openai/example/issues/comments/8")) + calls.clear() + result["head_sha"] = "f" * 40 + self.assertEqual( + report.publish_comment(result, "openai/example", 1, 2, 1), "Skipped stale report" + ) + self.assertEqual(len(calls), 1) + + def test_comment_rejects_older_runs_attempts_and_wrong_pr(self) -> None: + _, base = self.baseline() + result, _ = report.build_report(self.repo, base, base) + pull = {"state": "open", "head": {"sha": base}, "base": {"sha": base}} + run = { + "event": "pull_request", + "head_sha": base, + "run_attempt": 2, + "pull_requests": [{"number": 1}], + } + comment = { + "id": 8, + "user": {"login": "github-actions[bot]"}, + "body": report.MARKER + "\n", + "html_url": "existing", + } + with mock.patch.object(report, "api", side_effect=[pull, run]) as api: + self.assertEqual( + report.publish_comment(result, "openai/example", 1, 2, 1), "Skipped stale report" + ) + self.assertEqual(api.call_count, 2) + with mock.patch.object(report, "api", side_effect=[pull, run, [comment]]) as api: + self.assertEqual( + report.publish_comment(result, "openai/example", 1, 2, 2), "Skipped stale report" + ) + self.assertEqual(api.call_count, 3) + with mock.patch.object(report, "api", side_effect=[pull, {**run, "pull_requests": []}]): + with self.assertRaisesRegex(report.ReportError, "does not match report PR"): + report.publish_comment(result, "openai/example", 1, 2, 2) + changed = {**pull, "head": {"sha": "f" * 40}} + with mock.patch.object(report, "api", side_effect=[pull, run, [], changed]) as api: + self.assertEqual( + report.publish_comment(result, "openai/example", 1, 2, 2), "Skipped stale report" + ) + self.assertEqual(api.call_count, 4) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/openai/types/model.py b/src/openai/types/model.py index a78bdd4340..caffbee9a6 100644 --- a/src/openai/types/model.py +++ b/src/openai/types/model.py @@ -1,4 +1,5 @@ # File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. +# DO NOT MERGE: updated custom-code reporting demo; no runtime behavior change. from typing import Optional from typing_extensions import Literal