From 31fe6374352211c856f2f49cafb60f0ef846f2e2 Mon Sep 17 00:00:00 2001 From: apcha-oai <228803254+apcha-oai@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:06:42 +0000 Subject: [PATCH] chore(api): DO NOT MERGE - show custom code in SDK reviews Castiron-Internal-PR: https://github.com/openai/openai-python-internal/pull/34 Castiron-Source-SHA: 9fbcafee1844a5bf2a661ec8e5c5c1dfecb9c003 Castiron-Public-Base-SHA: 44c1560ad2f707d68e3f28d66aae602f7626e187 --- .castiron.stats.yml | 6 +- .github/workflows/castiron-custom-code.yml | 138 +++ scripts/castiron/README.md | 23 + scripts/castiron/custom_code_report.py | 920 ++++++++++++++++++++ scripts/castiron/ruff.toml | 3 + scripts/castiron/test_custom_code_report.py | 587 +++++++++++++ 6 files changed, 1675 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/castiron-custom-code.yml create mode 100644 scripts/castiron/README.md create mode 100644 scripts/castiron/custom_code_report.py create mode 100644 scripts/castiron/ruff.toml create mode 100644 scripts/castiron/test_custom_code_report.py diff --git a/.castiron.stats.yml b/.castiron.stats.yml index 5ffde321fa..a90a48f64b 100644 --- a/.castiron.stats.yml +++ b/.castiron.stats.yml @@ -1,6 +1,8 @@ schema_version: 1 -generation_id: 04cb977a-ba1d-438c-a56e-94f269e79218 +generation_id: 5df0d6f6-d156-43b6-ba92-de149763e673 openapi_spec_hash: dd725fb7d43ceec7fb2de6f8713d14b6 openapi_transformed_spec_hash: 10930179c5f116288e24e0c6fda46559 config_hash: 85382dd94c503b5d225adc7636a77c9f -codegen_sha: 6e990f52e3cbdeaae602710a1b0f2a2c944a5c35 +codegen_sha: 175308fd49c80670236b9cca66953da62b910eb5 +codegen_hash: 229f3e1c25b55fb04b07864af9ccdbc81edbdf6c341240a40fe336a3869ac9cb +public_codegen_sha: 193d1d672dc7aec341535802c14fa453372bf3db diff --git a/.github/workflows/castiron-custom-code.yml b/.github/workflows/castiron-custom-code.yml new file mode 100644 index 0000000000..5bab58d3a1 --- /dev/null +++ b/.github/workflows/castiron-custom-code.yml @@ -0,0 +1,138 @@ +# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. +name: Castiron custom code + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +# These experimental refs are restricted to trusted SDK maintainers. +# The workflow definition is trusted because its branch writers are trusted. +concurrency: + group: castiron-custom-code-${{ github.event.pull_request.number }} + cancel-in-progress: false + +env: + REPORTER_SHA256: 2855964a3b73aa57fd5a6b668c98b4aeb5dc54fc73a227091f832ca13a1f1f04 + +jobs: + report: + name: Castiron / baseline consistency + if: github.event.pull_request.head.repo.full_name == github.repository && contains(fromJSON(vars.CASTIRON_CUSTOM_CODE_BRANCHES || '[]'), github.event.pull_request.head.ref) + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + trusted: ${{ steps.trust.outputs.trusted }} + steps: + - name: Check the exact protected branch name + id: trust + env: + ALLOWED_BRANCHES: ${{ vars.CASTIRON_CUSTOM_CODE_BRANCHES || '[]' }} + PR_BRANCH: ${{ github.event.pull_request.head.ref }} + run: | + jq -e --arg branch "$PR_BRANCH" 'type == "array" and index($branch) != null' <<< "$ALLOWED_BRANCHES" > /dev/null + printf 'trusted=true\n' >> "$GITHUB_OUTPUT" + + - name: Check out the protected test branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Verify the reviewed reporter + run: printf '%s %s\n' "$REPORTER_SHA256" scripts/castiron/custom_code_report.py | sha256sum --check --strict + + - name: Test hash mismatch and snapshot isolation + run: python3 scripts/castiron/test_custom_code_report.py + + - name: Validate the codegen hash and report custom code + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PUBLIC_REPOSITORY: ${{ !github.event.repository.private }} + run: | + git fetch --quiet --no-tags origin "$BASE_SHA" "$HEAD_SHA" + mode=() + if [[ "$PUBLIC_REPOSITORY" == true ]]; then mode=(--public); fi + python3 -I scripts/castiron/custom_code_report.py report \ + --base "$BASE_SHA" --head "$HEAD_SHA" \ + --fetch --require-head-hash "${mode[@]}" \ + --out "$RUNNER_TEMP/castiron-custom-code" + + - name: Add the report to the run summary + if: always() + run: | + if test -f "$RUNNER_TEMP/castiron-custom-code/summary.md"; then + cat "$RUNNER_TEMP/castiron-custom-code/summary.md" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload report and current custom-code patch + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: castiron-custom-code-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/castiron-custom-code/ + if-no-files-found: error + retention-days: 7 + + comment: + name: Update custom-code comment + needs: report + if: always() && !cancelled() && needs.report.outputs.trusted == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + actions: read + pull-requests: write + steps: + - name: Check out the protected test branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Verify the reviewed publisher before execution + run: printf '%s %s\n' "$REPORTER_SHA256" scripts/castiron/custom_code_report.py | sha256sum --check --strict + + - name: Download this run's report + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: castiron-custom-code-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/castiron-custom-code + + - name: Create or update the single report comment + id: publish + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + run: | + python3 -I scripts/castiron/custom_code_report.py comment \ + --report "$RUNNER_TEMP/castiron-custom-code/report.json" \ + --repository "$REPOSITORY" --pr "$PR_NUMBER" --run-id "$RUN_ID" \ + --run-attempt "$RUN_ATTEMPT" + + - name: Publish a trusted failure status + if: always() && !cancelled() && steps.publish.outcome != 'success' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + const marker = ''; + const event = context.payload.pull_request; + const {data: current} = await github.rest.pulls.get({...context.repo, pull_number: event.number}); + if (current.state !== 'open' || current.head.sha !== event.head.sha || current.base.sha !== event.base.sha) return; + const comments = await github.paginate(github.rest.issues.listComments, {...context.repo, issue_number: event.number}); + const previous = comments.find(c => c.user?.type === 'Bot' && c.user?.login === 'github-actions[bot]' && c.body?.includes(marker)); + const run = Number(context.runId); + const attempt = Number(process.env.GITHUB_RUN_ATTEMPT); + const prior = previous?.body?.match(//); + if (prior && (Number(prior[1]) > run || (Number(prior[1]) === run && Number(prior[2]) > attempt))) return; + const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${run}`; + const body = `${marker}\n\n## Castiron custom code\n\n⚠️ Report unavailable for \`${event.head.sha.slice(0, 12)}\`.\n\nThe report setup or validation failed. [Inspect the workflow run](${url}).\n\n`; + if (previous) await github.rest.issues.updateComment({...context.repo, comment_id: previous.id, body}); + else await github.rest.issues.createComment({...context.repo, issue_number: event.number, body}); diff --git a/scripts/castiron/README.md b/scripts/castiron/README.md new file mode 100644 index 0000000000..1c5d51b688 --- /dev/null +++ b/scripts/castiron/README.md @@ -0,0 +1,23 @@ + +# Castiron custom-code reporting + +Castiron maintains shared templates for these files. Prefer changing those templates +for cross-SDK improvements; repository-specific customizations use the normal +three-way merge and are allowed. +The reporter uses Python 3.10+, Git, and `gh`; it does not import SDK code. + +Run `python3 scripts/castiron/test_custom_code_report.py` for focused tests. +The report comment includes commands to inspect the exact custom-code patch. +Public reporting uses only public snapshots and needs no private repository access. + +The workflow validates the recorded `codegen_hash`. +Its hash format is documented in the reporter. Only `.github/actions/` and +`.github/workflows/` are excluded from the content hash. + +During the draft rollout, `CASTIRON_CUSTOM_CODE_BRANCHES` is a repository-local +JSON array of maintainer-protected branch names. Only those branches can publish +comments. A general contributor/fork rollout requires a trusted default-branch +publisher. Never execute PR-controlled code with write credentials. +Changing the workflow may require one-time AM permission. Its reporter checksum +is a credential-safety check, not a requirement that every generated file remain +identical to its template. diff --git a/scripts/castiron/custom_code_report.py b/scripts/castiron/custom_code_report.py new file mode 100644 index 0000000000..1b5c79376c --- /dev/null +++ b/scripts/castiron/custom_code_report.py @@ -0,0 +1,920 @@ +#!/usr/bin/env python3 +# ruff: noqa: I001 +# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. +# Regenerate with Castiron; do not edit this file by hand. +"""Standalone, vendorable Castiron generated-content hashing and reporting. + +Hash v1: SHA256(domain || u64(entry count) || entries), where each entry is +u64(path length) || path bytes || six ASCII Git mode bytes || u64(blob length) +|| blob bytes. Integers are unsigned big-endian. Entries sort by raw Git path. +Only .github/actions/ and .github/workflows/ are excluded. No SDK imports. +""" + +from __future__ import annotations + +import argparse +import base64 +import collections +import hashlib +import html +import json +import os +import re +import struct +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +DOMAIN = b"castiron-codegen-v1\0" +MARKER = "" +RUN_MARKER = re.compile(r"") +SHA = re.compile(r"[0-9a-f]{40}\Z") +HASH = re.compile(r"[0-9a-f]{64}\Z") +REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z") +EXCLUDED = (b".github/actions/", b".github/workflows/") +PUBLIC_BLOCKED_PATHS = { + b".castiron", + b".github", + b".gitmodules", + b"CODEOWNERS", + b"SECURITY.md", + b"docs/CODEOWNERS", + b"docs/SECURITY.md", +} +LABELS = { + "newly_customized": "Newly customized", + "removed": "Customization removed", + "existing_changed": "Existing customization changed", + "baseline_changed": "Generated baseline changed", + "unchanged": "Existing customization unchanged", + "no_longer_generated": "No longer generated", + "newly_generation_owned": "Now generated; already existed", +} + + +class ReportError(ValueError): + """The report cannot establish a trustworthy comparison.""" + + +@dataclass(frozen=True) +class Entry: + mode: bytes + oid: str + + +def git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: + env = os.environ.copy() + # Preserve ordinary Git authentication/configuration, but do not inherit + # repository routing from the caller (for example, a Git hook). + for name in ( + "GIT_DIR", + "GIT_COMMON_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_PREFIX", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + ): + env.pop(name, None) + env["GIT_NO_REPLACE_OBJECTS"] = "1" + result = subprocess.run( + ["git", "--no-replace-objects", "-C", str(repo), *args], + input=input_bytes, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + check=False, + ) + if result.returncode: + raise ReportError(f"git {args[0]} failed: {result.stderr.decode(errors='replace').strip()}") + return result.stdout + + +def require_sha(value: str) -> str: + if not SHA.fullmatch(value): + raise ReportError("expected a full Git commit SHA") + return value + + +def included(path: bytes) -> bool: + return not path.startswith(EXCLUDED) + + +def tree_entries(repo: Path, commit: str) -> dict[bytes, Entry]: + result: dict[bytes, Entry] = {} + for record in git(repo, "ls-tree", "-r", "-z", require_sha(commit)).split(b"\0"): + if not record: + continue + metadata, path = record.split(b"\t", 1) + mode, kind, oid = metadata.split(b" ") + if not included(path): + continue + if kind != b"blob" or mode not in {b"100644", b"100755", b"120000"}: + raise ReportError(f"unsupported Git entry: {path!r}") + result[path] = Entry(mode, oid.decode("ascii")) + return result + + +def read_blobs(repo: Path, entries: dict[bytes, Entry]) -> dict[str, bytes]: + oids = sorted({entry.oid for entry in entries.values()}) + output = git( + repo, "cat-file", "--batch", input_bytes="".join(f"{oid}\n" for oid in oids).encode() + ) + result: dict[str, bytes] = {} + offset = 0 + for oid in oids: + end = output.index(b"\n", offset) + actual, kind, size = output[offset:end].split(b" ") + if actual.decode() != oid or kind != b"blob": + raise ReportError("unexpected Git object response") + offset = end + 1 + length = int(size) + result[oid] = output[offset : offset + length] + offset += length + if output[offset : offset + 1] != b"\n": + raise ReportError("truncated Git object response") + offset += 1 + if offset != len(output): + raise ReportError("unexpected trailing Git object data") + return result + + +def hash_codegen_commit(repo: Path, commit: str) -> str: + entries = tree_entries(repo, commit) + blobs = read_blobs(repo, entries) + digest = hashlib.sha256(DOMAIN + struct.pack(">Q", len(entries))) + for path, entry in sorted(entries.items()): + content = blobs[entry.oid] + digest.update(struct.pack(">Q", len(path))) + digest.update(path) + digest.update(entry.mode) + digest.update(struct.pack(">Q", len(content))) + digest.update(content) + return digest.hexdigest() + + +def read_stats(repo: Path, commit: str) -> dict[str, str]: + entry = git(repo, "ls-tree", require_sha(commit), "--", ".castiron.stats.yml") + if not entry.startswith((b"100644 blob ", b"100755 blob ")): + raise ReportError("Castiron stats must be a regular Git file") + text = git(repo, "show", f"{require_sha(commit)}:.castiron.stats.yml").decode("utf-8") + values: dict[str, str] = {} + for line in text.splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + key, sep, value = line.partition(":") + if not sep or key.strip() in values: + raise ReportError("invalid or duplicate stats field") + values[key.strip()] = value.strip().strip("\"'") + if values.get("schema_version") != "1": + raise ReportError("unsupported Castiron stats schema") + require_sha(values.get("codegen_sha", "")) + if not re.fullmatch( + r"[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}", + values.get("generation_id", ""), + ): + raise ReportError("invalid generation ID") + if "codegen_hash" in values and not HASH.fullmatch(values["codegen_hash"]): + raise ReportError("invalid codegen_hash") + if "public_codegen_sha" in values: + require_sha(values["public_codegen_sha"]) + return values + + +def public_metadata(repo: Path, commit: str) -> dict[str, str]: + message = git(repo, "show", "-s", "--format=%B", require_sha(commit)).decode("utf-8") + fields: dict[str, str] = {} + for key in ("Version", "Generation-ID", "Hash", "Branch"): + matches = re.findall(rf"^Castiron-Public-Codegen-{key}: (.+)$", message, re.MULTILINE) + if len(matches) != 1: + raise ReportError("invalid public codegen metadata") + fields[key] = matches[0] + if fields["Version"] != "1" or not HASH.fullmatch(fields["Hash"]): + raise ReportError("unsupported public codegen metadata") + if not fields["Branch"].startswith("codegen/"): + raise ReportError("invalid public codegen branch") + git(repo, "check-ref-format", f"refs/heads/{fields['Branch']}") + return fields + + +def resolve_public_baseline( + repo: Path, revision: str, *, fetch: bool, require_hash: bool, lineage: str | None = None +) -> dict[str, Any]: + stats = read_stats(repo, revision) + commit = stats.get("public_codegen_sha") + if commit is None: + # Bootstrap old public main without changing its schema-v1 stats. Search + # only the already-verified public snapshot lineage supplied by PR head. + if lineage is None: + raise ReportError("stats do not contain public_codegen_sha") + for candidate in git(repo, "rev-list", require_sha(lineage)).decode().splitlines(): + if public_metadata(repo, candidate)["Generation-ID"] == stats["generation_id"]: + commit = candidate + break + if commit is None: + raise ReportError("public snapshot lineage does not contain the base generation") + elif fetch: + git(repo, "fetch", "--quiet", "--no-tags", "origin", commit) + metadata = public_metadata(repo, commit) + if metadata["Generation-ID"] != stats["generation_id"]: + raise ReportError("public codegen generation ID does not match stats") + ref = f"refs/heads/{metadata['Branch']}" + tracking = f"refs/castiron/report/{hashlib.sha256(ref.encode()).hexdigest()}" + if fetch: + git(repo, "fetch", "--quiet", "--no-tags", "origin", f"+{ref}:{tracking}") + else: + tracking = f"refs/remotes/origin/{metadata['Branch']}" + git(repo, "merge-base", "--is-ancestor", commit, tracking) + actual = hash_codegen_commit(repo, commit) + if actual != metadata["Hash"]: + raise ReportError("public codegen content does not match snapshot metadata") + expected = stats.get("codegen_hash") + if expected is None and require_hash: + raise ReportError("head stats do not contain codegen_hash") + if expected is not None and expected != actual: + raise ReportError(f"codegen_hash mismatch: expected {expected}, got {actual}") + return { + "commit": commit, + "generation_id": stats["generation_id"], + "hash": actual, + "hash_recorded": expected is not None, + } + + +def public_snapshot_entries( + source: Path, + commit: str, + destination: Path, + public_base: str, + approved_tree: str | None = None, +) -> dict[bytes, Entry]: + entries = tree_entries(source, commit) + approved = tree_entries(destination, public_base) + if approved_tree is not None: + approved.update(tree_entries(destination, approved_tree)) + for path in entries: + if path in PUBLIC_BLOCKED_PATHS or path.startswith((b".castiron/", b".github/")): + raise ReportError(f"refusing to publish private or governance snapshot path: {path!r}") + if path not in approved: + raise ReportError( + f"snapshot path is absent from the approved public SDK tree: {path!r}" + ) + return entries + + +def copy_generated_tree( + source: Path, + commit: str, + destination: Path, + public_base: str, + approved_tree: str | None = None, +) -> str: + """Transfer approved blobs/trees only, never source commits or ancestors.""" + entries = public_snapshot_entries(source, commit, destination, public_base, approved_tree) + for oid, blob in read_blobs(source, entries).items(): + actual = git(destination, "hash-object", "-w", "--stdin", input_bytes=blob).decode().strip() + if actual != oid: + raise ReportError("source and destination Git object formats differ") + directories: dict[bytes, list[bytes]] = {b"": []} + for path, entry in entries.items(): + parent, _, name = path.rpartition(b"/") + directories.setdefault(parent, []).append( + entry.mode + b" blob " + entry.oid.encode() + b"\t" + name + b"\0" + ) + while parent: + parent = parent.rpartition(b"/")[0] + directories.setdefault(parent, []) + for directory in sorted(directories, key=lambda p: (p.count(b"/"), len(p)), reverse=True): + tree = git( + destination, "mktree", "-z", input_bytes=b"".join(directories[directory]) + ).strip() + if not directory: + return tree.decode() + parent, _, name = directory.rpartition(b"/") + directories[parent].append(b"040000 tree " + tree + b"\t" + name + b"\0") + raise ReportError("could not construct public generated tree") + + +def create_public_snapshot( + destination: Path, + tree: str, + generation: str, + content_hash: str, + branch: str, + parent: str | None, +) -> str: + message = ( + "Castiron generated snapshot\n\n" + "Castiron-Public-Codegen-Version: 1\n" + f"Castiron-Public-Codegen-Generation-ID: {generation}\n" + f"Castiron-Public-Codegen-Hash: {content_hash}\n" + f"Castiron-Public-Codegen-Branch: {branch}\n" + ) + parents = ["-p", require_sha(parent)] if parent is not None else [] + snapshot = ( + git( + destination, + "-c", + "user.name=Castiron", + "-c", + "user.email=noreply@openai.com", + "commit-tree", + tree, + *parents, + "-m", + message, + ) + .decode() + .strip() + ) + require_sha(snapshot) + if hash_codegen_commit(destination, snapshot) != content_hash: + raise ReportError("public snapshot content hash differs from private checkpoint") + return snapshot + + +def prepare_public_snapshot( + source: Path, + source_base: str, + source_head: str, + destination: Path, + public_base: str, + branch: str, +) -> dict[str, Any]: + """Prepare public-only generated history; never push anything.""" + read_stats(source, source_base) + after = read_stats(source, source_head) + public = read_stats(destination, public_base) + baseline = resolve_checkpoint(source, after, fetch=True, require_hash=True) + codegen_branch = f"codegen/{branch}" + ref = f"refs/heads/{codegen_branch}" + git(destination, "check-ref-format", ref) + remote = git(destination, "ls-remote", "--refs", "origin", ref).decode().splitlines() + previous = require_sha(remote[0].split()[0]) if remote else "" + if previous: + git(destination, "fetch", "--quiet", "--no-tags", "origin", previous) + metadata = public_metadata(destination, previous) + if ( + metadata["Branch"] != codegen_branch + or hash_codegen_commit(destination, previous) != metadata["Hash"] + ): + raise ReportError("existing public codegen branch has invalid provenance") + + base_snapshot = public.get("public_codegen_sha") + if base_snapshot: + base_snapshot = resolve_public_baseline( + destination, public_base, fetch=True, require_hash=False + )["commit"] + elif previous: + try: + base_snapshot = resolve_public_baseline( + destination, public_base, fetch=True, require_hash=False, lineage=previous + )["commit"] + except ReportError: + base_snapshot = None + if base_snapshot is None: + old = resolve_checkpoint(source, public, fetch=True, require_hash=False) + # Initial bootstrap is limited to paths already present on public main. + tree = copy_generated_tree(source, old["commit"], destination, public_base) + base_snapshot = create_public_snapshot( + destination, tree, old["generation_id"], old["hash"], codegen_branch, None + ) + + # Normal promotion has already applied the reviewed patch to the public + # index. Private HEAD can contain unchanged private-only paths, so it is + # not an approval boundary for publication. + approved_tree = git(destination, "write-tree").decode().strip() + tree = copy_generated_tree(source, baseline["commit"], destination, public_base, approved_tree) + parent = base_snapshot + if previous: + ancestors = git(destination, "rev-list", previous).decode().splitlines() + if base_snapshot in ancestors: + parent = previous + metadata = public_metadata(destination, previous) + if ( + metadata["Generation-ID"] == after["generation_id"] + and metadata["Hash"] == baseline["hash"] + and git(destination, "rev-parse", f"{previous}^{{tree}}").decode().strip() == tree + and base_snapshot in ancestors + ): + snapshot = previous + else: + snapshot = create_public_snapshot( + destination, tree, after["generation_id"], baseline["hash"], codegen_branch, parent + ) + elif ( + public_metadata(destination, base_snapshot)["Generation-ID"] == after["generation_id"] + and git(destination, "rev-parse", f"{base_snapshot}^{{tree}}").decode().strip() == tree + ): + snapshot = base_snapshot + else: + snapshot = create_public_snapshot( + destination, tree, after["generation_id"], baseline["hash"], codegen_branch, parent + ) + stats_path = destination / ".castiron.stats.yml" + current = git(source, "show", f"{require_sha(source_head)}:.castiron.stats.yml").decode("utf-8") + lines = [line for line in current.splitlines() if not line.startswith("public_codegen_sha:")] + stats_path.write_text("\n".join(lines) + f"\npublic_codegen_sha: {snapshot}\n") + git(destination, "add", "--", ".castiron.stats.yml") + return { + "commit": snapshot, + "branch": codegen_branch, + "previous": previous, + "hash": baseline["hash"], + } + + +def resolve_baseline( + repo: Path, revision: str, *, fetch: bool, require_hash: bool +) -> dict[str, Any]: + return resolve_checkpoint( + repo, read_stats(repo, revision), fetch=fetch, require_hash=require_hash + ) + + +def resolve_checkpoint( + repo: Path, stats: dict[str, str], *, fetch: bool, require_hash: bool +) -> dict[str, Any]: + commit = stats["codegen_sha"] + if fetch: + git(repo, "fetch", "--quiet", "--no-tags", "origin", commit) + message = git(repo, "show", "-s", "--format=%B", commit).decode("utf-8") + match = re.search(r"^Generation metadata: ([A-Za-z0-9+/=]+)$", message, re.MULTILINE) + if not match: + raise ReportError("codegen checkpoint has no generation metadata") + try: + decoded: object = json.loads(base64.b64decode(match[1], validate=True)) + except (ValueError, TypeError) as exc: + raise ReportError("invalid codegen checkpoint metadata") from exc + if not isinstance(decoded, dict): + raise ReportError("invalid codegen checkpoint metadata") + metadata = cast(dict[str, Any], decoded) + if metadata.get("generation_id") != stats["generation_id"]: + raise ReportError("codegen checkpoint generation ID does not match stats") + branch = metadata.get("source_branch") + if not isinstance(branch, str) or not branch: + raise ReportError("codegen checkpoint has no source branch") + ref = f"refs/heads/codegen/{branch}" + git(repo, "check-ref-format", ref) + tracking = f"refs/castiron/report/{hashlib.sha256(ref.encode()).hexdigest()}" + if fetch: + git(repo, "fetch", "--quiet", "--no-tags", "origin", f"+{ref}:{tracking}") + else: + tracking = f"refs/remotes/origin/codegen/{branch}" + git(repo, "merge-base", "--is-ancestor", commit, tracking) + actual = hash_codegen_commit(repo, commit) + expected = stats.get("codegen_hash") + if expected is None and require_hash: + raise ReportError("head stats do not contain codegen_hash") + if expected is not None and actual != expected: + raise ReportError(f"codegen_hash mismatch: expected {expected}, got {actual}") + return { + "commit": commit, + "generation_id": stats["generation_id"], + "hash": actual, + "hash_recorded": expected is not None, + } + + +def numstats(repo: Path, generated: str, integrated: str) -> dict[bytes, tuple[str, str]]: + result: dict[bytes, tuple[str, str]] = {} + output = git( + repo, + "diff", + "--numstat", + "-z", + "--no-renames", + "--no-ext-diff", + "--no-textconv", + generated, + integrated, + "--", + ) + for record in output.split(b"\0"): + if record: + added, removed, path = record.split(b"\t", 2) + result[path] = (added.decode("ascii"), removed.decode("ascii")) + return result + + +def classify(g0: Entry | None, i0: Entry | None, g1: Entry | None, i1: Entry | None) -> str | None: + before = g0 is not None and g0 != i0 + after = g1 is not None and g1 != i1 + if g0 is not None and g1 is None and (before or i1 is not None): + return "no_longer_generated" + if g0 is None and g1 is not None and i0 is not None: + return "newly_generation_owned" + if not before and not after: + return None + if g0 is not None and g1 is not None and g0 != g1: + return "baseline_changed" + if not before: + return "newly_customized" + if not after: + return "removed" + return "unchanged" if i0 == i1 else "existing_changed" + + +def build_report( + repo: Path, + base: str, + head: str, + *, + fetch: bool = False, + require_head_hash: bool = False, + public: bool = False, +) -> tuple[dict[str, Any], bytes]: + base, head = require_sha(base), require_sha(head) + comparison_base = git(repo, "merge-base", base, head).decode().strip() + if public: + after = resolve_public_baseline(repo, head, fetch=fetch, require_hash=require_head_hash) + before = resolve_public_baseline( + repo, comparison_base, fetch=fetch, require_hash=False, lineage=after["commit"] + ) + else: + before = resolve_baseline(repo, comparison_base, fetch=fetch, require_hash=False) + after = resolve_baseline(repo, head, fetch=fetch, require_hash=require_head_hash) + g0, i0 = tree_entries(repo, before["commit"]), tree_entries(repo, comparison_base) + g1, i1 = tree_entries(repo, after["commit"]), tree_entries(repo, head) + stats = numstats(repo, after["commit"], head) + files: list[dict[str, Any]] = [] + current_paths: list[str] = [] + for path in sorted(g0.keys() | g1.keys()): + category = classify(g0.get(path), i0.get(path), g1.get(path), i1.get(path)) + if category is None: + continue + custom_before = path in g0 and g0[path] != i0.get(path) + custom_after = path in g1 and g1[path] != i1.get(path) + added, removed = stats.get(path, ("0", "0")) if custom_after else ("0", "0") + files.append( + { + "path": path.decode("utf-8"), + "category": category, + "custom_before": custom_before, + "custom_after": custom_after, + "added": added, + "removed": removed, + } + ) + if custom_after: + current_paths.append(path.decode("utf-8")) + patch = b"" + if current_paths: + patch = git( + repo, + "--literal-pathspecs", + "diff", + "--binary", + "--full-index", + "--no-renames", + "--no-ext-diff", + "--no-textconv", + "--no-color", + after["commit"], + head, + "--", + *current_paths, + ) + return { + "schema_version": 1, + "status": "ok", + "public": public, + "target_base_sha": base, + "base_sha": comparison_base, + "head_sha": head, + "before": before, + "after": after, + "files": files, + "counts": { + "before": sum(f["custom_before"] for f in files), + "after": sum(f["custom_after"] for f in files), + **dict(collections.Counter(f["category"] for f in files)), + }, + }, patch + + +def render_report( + report: dict[str, Any], + run_url: str = "", + *, + repository: str = "", + run_id: int = 0, + run_attempt: int = 0, +) -> str: + head = require_sha(report["head_sha"]) + lines = [MARKER, "", "## Castiron custom code", ""] + if report.get("status") != "ok": + reason = html.escape(str(report.get("error", "Report could not be computed"))) + lines.extend([f"⚠️ Report unavailable for `{head[:12]}`.", "", reason]) + else: + files = report["files"] + counts: collections.Counter[str] = collections.Counter() + for file in files: + if file["category"] not in LABELS: + raise ReportError("unknown report category") + counts[file["category"]] += 1 + before = sum(bool(file["custom_before"]) for file in files) + after = sum(bool(file["custom_after"]) for file in files) + base = require_sha(report["base_sha"]) + newly_customized = sum( + bool(file["custom_after"]) and not file["custom_before"] for file in files + ) + removed = sum( + bool(file["custom_before"]) + and not file["custom_after"] + and file["category"] != "no_longer_generated" + for file in files + ) + if newly_customized == 0: + remaining = ( + f"{after} mixed file{'s' if after != 1 else ''} remain{'s' if after == 1 else ''}" + ) + changed = counts["existing_changed"] + details = [f"{changed} existing customization{'s' if changed != 1 else ''} changed"] + if removed: + details.append(f"{removed} customizations removed") + if counts["baseline_changed"]: + details.append(f"{counts['baseline_changed']} generated baselines changed") + lines.extend( + [ + "✅ No new custom-code files detected.", + "", + remaining + "; " + "; ".join(details) + ".", + ] + ) + else: + lines.extend( + [ + f"**Mixed files: {before} → {after}**", + "", + ( + f"{newly_customized} newly customized · {removed} customizations removed · " + f"{counts['existing_changed']} existing customizations changed · {counts['baseline_changed']} generated baselines changed" + ), + ] + ) + lines.extend(["", f"Compared `{base[:12]}` → `{head[:12]}`. Generated baselines verified."]) + if not report["before"]["hash_recorded"] or not report["after"]["hash_recorded"]: + lines.extend( + [ + "", + "Legacy baseline: content hash computed from the pinned checkpoint; not yet recorded in that revision’s stats.", + ] + ) + interesting = [file for file in files if file["category"] != "unchanged"] + if interesting: + lines.extend(["", "| File | Result | Current custom patch |", "| --- | --- | --- |"]) + for file in interesting[:40]: + path = ( + html.escape(str(file["path"])) + .replace("|", "|") + .replace("\n", " ") + .replace("\r", " ") + ) + added, removed = str(file["added"]), str(file["removed"]) + if not re.fullmatch(r"\d+|-", added) or not re.fullmatch(r"\d+|-", removed): + raise ReportError("invalid line counts") + delta = f"+{added} / −{removed}" if file["custom_after"] else "None" + lines.append(f"| {path} | {LABELS[file['category']]} | {delta} |") + if len(interesting) > 40: + lines.extend( + [ + "", + f"{len(interesting) - 40} more affected files are listed in the full report.", + ] + ) + if counts["unchanged"]: + lines.extend( + [ + "", + f"
{counts['unchanged']} existing customizations unchanged", + "", + ] + ) + for file in [file for file in files if file["category"] == "unchanged"][:40]: + lines.append(f"- {html.escape(str(file['path']))}") + if counts["unchanged"] > 40: + lines.extend(["", f"{counts['unchanged'] - 40} more in the full report."]) + lines.extend(["", "
"]) + lines.extend( + [ + "", + "A changed generated baseline means this report cannot reliably identify which handwritten lines changed.", + ] + ) + target_base = require_sha(report["target_base_sha"]) + lines.extend(["", "
Inspect the custom-code diff", ""]) + if repository: + if not REPOSITORY.fullmatch(repository) or min(run_id, run_attempt) <= 0: + raise ReportError("invalid artifact download target") + artifact = f"castiron-custom-code-{run_id}-{run_attempt}" + lines.extend( + [ + "Download the exact patch produced by this run (requires repository access):", + "", + "```sh", + f"gh run download {run_id} --repo {repository} \\", + f" --name {artifact} --dir /tmp/{artifact}", + f"git apply --stat /tmp/{artifact}/custom-code.patch", + f"cat /tmp/{artifact}/custom-code.patch", + "```", + "", + ] + ) + lines.extend( + [ + "Or reproduce it from an SDK checkout containing the vendored reporter:", + "", + "```sh", + f"git fetch --no-tags origin {target_base} {head}", + "python3 scripts/castiron/custom_code_report.py report \\", + f" --base {target_base} \\", + f" --head {head} --fetch --require-head-hash{' --public' if report.get('public') else ''} \\", + f" --out /tmp/castiron-custom-code-{head[:12]}", + f"cat /tmp/castiron-custom-code-{head[:12]}/custom-code.patch", + "```", + "", + "This is the **current full custom patch** for mixed files, not an attribution of only the handwritten lines changed by this PR.", + "", + "
", + ] + ) + if run_url: + lines.extend(["", f"[Full report and patch]({run_url})"]) + return "\n".join(lines) + "\n" + + +def api(method: str, path: str, payload: dict[str, Any] | None = None) -> Any: + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if not token: + raise ReportError("GitHub token is required for comment publication") + args = ["gh", "api", "--hostname", "github.com", "--method", method, path] + if payload is not None: + args.extend(["--input", "-"]) + result = subprocess.run( + args, + input=json.dumps(payload).encode() if payload is not None else None, + capture_output=True, + check=False, + timeout=60, + ) + if result.returncode: + raise ReportError( + f"GitHub API request failed: {result.stderr.decode(errors='replace').strip()}" + ) + return json.loads(result.stdout) + + +def publish_comment( + report: dict[str, Any], repository: str, number: int, run_id: int, run_attempt: int +) -> str: + if not REPOSITORY.fullmatch(repository) or min(number, run_id, run_attempt) <= 0: + raise ReportError("invalid GitHub publication target") + root = f"repos/{repository}" + pull = api("GET", f"{root}/pulls/{number}") + if ( + pull["state"] != "open" + or pull["head"]["sha"] != report["head_sha"] + or pull["base"]["sha"] != report["target_base_sha"] + ): + return "Skipped stale report" + run = api("GET", f"{root}/actions/runs/{run_id}") + if ( + run["event"] != "pull_request" + or run["head_sha"] != report["head_sha"] + or not any(pr["number"] == number for pr in run["pull_requests"]) + ): + raise ReportError("workflow run does not match report PR/head") + if run["run_attempt"] != run_attempt: + return "Skipped stale report" + body = render_report( + report, + f"https://github.com/{repository}/actions/runs/{run_id}", + repository=repository, + run_id=run_id, + run_attempt=run_attempt, + ) + body += f"\n\n" + found = None + for page in range(1, 101): + comments = api("GET", f"{root}/issues/{number}/comments?per_page=100&page={page}") + for comment in comments: + if comment["user"]["login"] == "github-actions[bot]" and comment["body"].startswith( + MARKER + ): + found = comment + break + if found is not None or len(comments) < 100: + break + if found is not None: + previous = RUN_MARKER.search(found["body"]) + if previous and tuple(map(int, previous.groups())) > (run_id, run_attempt): + return "Skipped stale report" + if found["body"] == body: + return str(found["html_url"]) + # The workflow serializes publishers; recheck after pagination before writing. + pull = api("GET", f"{root}/pulls/{number}") + if ( + pull["state"] != "open" + or pull["head"]["sha"] != report["head_sha"] + or pull["base"]["sha"] != report["target_base_sha"] + ): + return "Skipped stale report" + if found is not None: + result = api("PATCH", f"{root}/issues/comments/{found['id']}", {"body": body}) + else: + result = api("POST", f"{root}/issues/{number}/comments", {"body": body}) + return str(result["html_url"]) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + hashing = commands.add_parser("hash") + hashing.add_argument("--repo", type=Path, default=Path.cwd()) + hashing.add_argument("--commit", required=True) + reporting = commands.add_parser("report") + reporting.add_argument("--repo", type=Path, default=Path.cwd()) + reporting.add_argument("--base", required=True) + reporting.add_argument("--head", required=True) + reporting.add_argument("--out", type=Path, required=True) + reporting.add_argument("--fetch", action="store_true") + reporting.add_argument("--require-head-hash", action="store_true") + reporting.add_argument("--public", action="store_true") + preparing = commands.add_parser("prepare-public") + preparing.add_argument("--source-repo", type=Path, required=True) + preparing.add_argument("--source-base", required=True) + preparing.add_argument("--source-head", required=True) + preparing.add_argument("--public-repo", type=Path, required=True) + preparing.add_argument("--public-base", required=True) + preparing.add_argument("--branch", required=True) + commenting = commands.add_parser("comment") + commenting.add_argument("--report", type=Path, required=True) + commenting.add_argument("--repository", required=True) + commenting.add_argument("--pr", type=int, required=True) + commenting.add_argument("--run-id", type=int, required=True) + commenting.add_argument("--run-attempt", type=int, required=True) + args = parser.parse_args() + try: + if args.command == "hash": + sys.stdout.write(hash_codegen_commit(args.repo, args.commit) + "\n") + elif args.command == "prepare-public": + sys.stdout.write( + json.dumps( + prepare_public_snapshot( + args.source_repo, + args.source_base, + args.source_head, + args.public_repo, + args.public_base, + args.branch, + ) + ) + + "\n" + ) + elif args.command == "comment": + if args.report.stat().st_size > 5_000_000: + raise ReportError("report artifact is too large") + report = json.loads(args.report.read_text()) + sys.stdout.write( + publish_comment(report, args.repository, args.pr, args.run_id, args.run_attempt) + + "\n" + ) + else: + args.out.mkdir(parents=True, exist_ok=True) + try: + report, patch = build_report( + args.repo, + args.base, + args.head, + fetch=args.fetch, + require_head_hash=args.require_head_hash, + public=args.public, + ) + except (ReportError, UnicodeError, KeyError, ValueError) as exc: + report = { + "schema_version": 1, + "status": "error", + "target_base_sha": require_sha(args.base), + "head_sha": require_sha(args.head), + "error": str(exc), + } + patch = b"" + (args.out / "report.json").write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n" + ) + (args.out / "custom-code.patch").write_bytes(patch) + summary = render_report(report) + (args.out / "summary.md").write_text(summary) + sys.stdout.write(summary) + return 0 if report["status"] == "ok" else 1 + except (ReportError, OSError, subprocess.TimeoutExpired) as exc: + sys.stderr.write(f"Castiron custom-code report: {exc}\n") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/castiron/ruff.toml b/scripts/castiron/ruff.toml new file mode 100644 index 0000000000..1e8a301260 --- /dev/null +++ b/scripts/castiron/ruff.toml @@ -0,0 +1,3 @@ +# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. +line-length = 100 +target-version = "py310" diff --git a/scripts/castiron/test_custom_code_report.py b/scripts/castiron/test_custom_code_report.py new file mode 100644 index 0000000000..cdc7126b77 --- /dev/null +++ b/scripts/castiron/test_custom_code_report.py @@ -0,0 +1,587 @@ +# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. +# ruff: noqa: I001 +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import shutil +import struct +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path +from unittest import mock + +import custom_code_report as report + +GENERATION = "550e8400-e29b-41d4-a716-446655440000" + + +class CustomCodeTests(unittest.TestCase): + # Keep the vendored test stdlib-only on Python 3.10 (no typing.override yet). + def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.repo = Path(self.temporary.name) + self.git("init", "-q", "-b", "main") + self.git("config", "user.name", "Castiron test") + self.git("config", "user.email", "castiron@example.test") + + def git(self, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(self.repo), *args], check=True, capture_output=True, text=True + ).stdout.strip() + + def write(self, path: str, body: str) -> None: + target = self.repo / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + + def commit(self, message: str = "fixture") -> str: + self.git("add", "-A") + self.git("commit", "-q", "--allow-empty", "-m", message) + return self.git("rev-parse", "HEAD") + + def baseline(self) -> tuple[str, str]: + self.write("generated.py", "generated\n") + metadata = { + "generation_id": GENERATION, + "source_branch": "test", + "target": "openai-python", + "language": "python", + } + encoded = base64.b64encode(json.dumps(metadata).encode()).decode() + generated = self.commit(f"codegen\n\nGeneration metadata: {encoded}") + self.git("update-ref", "refs/remotes/origin/codegen/test", generated) + self.write( + ".castiron.stats.yml", + f"schema_version: 1\ngeneration_id: {GENERATION}\ncodegen_sha: {generated}\ncodegen_hash: {report.hash_codegen_commit(self.repo, generated)}\n", + ) + return generated, self.commit("integrated") + + def test_git_preserves_authentication_without_inherited_repository_routing(self) -> None: + environment = { + "GIT_CONFIG_GLOBAL": "/ordinary/gitconfig", + "GIT_ASKPASS": "/ordinary/askpass", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "credential.helper", + "GIT_CONFIG_VALUE_0": "example", + "GIT_DIR": "/wrong/repository", + "GIT_COMMON_DIR": "/wrong/common", + "GIT_INDEX_FILE": "/wrong/index", + } + with ( + mock.patch.dict(os.environ, environment), + mock.patch.object( + report.subprocess, + "run", + return_value=subprocess.CompletedProcess([], 0, stdout=b"ok"), + ) as run, + ): + self.assertEqual(report.git(self.repo, "status"), b"ok") + actual = run.call_args.kwargs["env"] + for name, value in environment.items(): + if name in {"GIT_DIR", "GIT_COMMON_DIR", "GIT_INDEX_FILE"}: + self.assertNotIn(name, actual) + else: + self.assertEqual(actual[name], value) + self.assertEqual(actual["GIT_NO_REPLACE_OBJECTS"], "1") + + @unittest.skipUnless(os.environ.get("CASTIRON_TEST_BIN"), "Castiron compiler contract test") + def test_rust_and_python_hashes_match(self) -> None: + self.write("a", "first\n") + self.write("nested/b", "second\n") + self.write(".github/workflows/ignored.yml", "ignored\n") + for change in ("initial", "mode", "bytes"): + if change == "mode": + (self.repo / "a").chmod(0o755) + elif change == "bytes": + self.write("nested/b", "changed\n") + commit = self.commit(change) + rust = subprocess.check_output( + [ + os.environ["CASTIRON_TEST_BIN"], + "sdk", + "codegen-hash", + "--repo", + str(self.repo), + "--commit", + commit, + ], + text=True, + ).strip() + self.assertEqual(rust, report.hash_codegen_commit(self.repo, commit)) + + def test_reporting_script_can_be_a_mixed_file(self) -> None: + path = "scripts/castiron/custom_code_report.py" + self.write(path, "# generated reporter\n") + _, base = self.baseline() + self.write(path, "# generated reporter\n# local customization\n") + head = self.commit() + result, _ = report.build_report(self.repo, base, head, require_head_hash=True) + changed = next(file for file in result["files"] if file["path"] == path) + self.assertEqual(changed["category"], "newly_customized") + + def test_hash_vector_and_exclusions(self) -> None: + self.write("a", "hello\n") + first = self.commit() + encoded = ( + report.DOMAIN + + struct.pack(">Q", 1) + + struct.pack(">Q", 1) + + b"a100644" + + struct.pack(">Q", 6) + + b"hello\n" + ) + expected = hashlib.sha256(encoded).hexdigest() + self.assertEqual(report.hash_codegen_commit(self.repo, first), expected) + self.write(".github/workflows/test.yml", "ignored\n") + self.write(".github/actions/test/action.yml", "ignored\n") + second = self.commit() + self.assertEqual(report.hash_codegen_commit(self.repo, second), expected) + os.chmod(self.repo / "a", 0o755) + self.assertNotEqual(report.hash_codegen_commit(self.repo, self.commit()), expected) + self.write(".github/CODEOWNERS", "not ignored\n") + self.assertNotEqual(report.hash_codegen_commit(self.repo, self.commit()), expected) + + def test_classifications(self) -> None: + a, b, c = (report.Entry(b"100644", value * 40) for value in "abc") + cases = [ + ((a, a, b, b), None), + ((a, a, a, b), "newly_customized"), + ((a, b, a, a), "removed"), + ((a, b, a, c), "existing_changed"), + ((a, b, a, b), "unchanged"), + ((a, b, c, b), "baseline_changed"), + ((a, b, None, b), "no_longer_generated"), + ((a, a, None, a), "no_longer_generated"), + ((a, a, None, None), None), + ((None, b, a, b), "newly_generation_owned"), + ((None, a, a, a), "newly_generation_owned"), + ((a, a, b, a), "baseline_changed"), + ((a, b, b, b), "baseline_changed"), + ((a, a, a, None), "newly_customized"), + ] + for values, expected in cases: + with self.subTest(expected=expected): + self.assertEqual(report.classify(*values), expected) + + def test_report_ignores_handwritten_files_and_shows_custom_patch(self) -> None: + generated, base = self.baseline() + self.write("generated.py", "generated\n# custom\n") + self.write("handwritten.py", "handwritten\n") + head = self.commit() + result, patch = report.build_report(self.repo, base, head, require_head_hash=True) + self.assertEqual(result["counts"]["newly_customized"], 1) + self.assertEqual(result["files"][0]["path"], "generated.py") + self.assertIn(b"+# custom", patch) + self.assertNotIn(b"handwritten", patch) + self.assertEqual(result["after"]["commit"], generated) + body = report.render_report(result, repository="openai/example", run_id=42, run_attempt=2) + self.assertIn("Inspect the custom-code diff", body) + self.assertIn("gh run download 42 --repo openai/example", body) + self.assertIn("--name castiron-custom-code-42-2", body) + self.assertIn(f"--head {head}", body) + self.assertIn("current full custom patch", body) + + def test_bad_hash_and_lost_checkpoint_fail(self) -> None: + generated, base = self.baseline() + stats = self.repo / ".castiron.stats.yml" + stats.write_text( + stats.read_text().replace(report.hash_codegen_commit(self.repo, generated), "f" * 64) + ) + with self.assertRaisesRegex(report.ReportError, "codegen_hash mismatch"): + report.build_report(self.repo, base, self.commit()) + self.git("update-ref", "-d", "refs/remotes/origin/codegen/test") + with self.assertRaises(report.ReportError): + report.resolve_baseline(self.repo, base, fetch=False, require_hash=True) + + def test_stats_symlink_is_rejected(self) -> None: + self.baseline() + path = self.repo / ".castiron.stats.yml" + original = path.read_text() + path.unlink() + path.symlink_to(original) + with self.assertRaisesRegex(report.ReportError, "regular Git file"): + report.read_stats(self.repo, self.commit()) + + def test_public_snapshot_rejects_private_governance_and_unpublished_paths(self) -> None: + entry = report.Entry(b"100644", "a" * 40) + blocked = [ + b".castiron/private.json", + b".github/CODEOWNERS", + b".gitmodules", + b"CODEOWNERS", + b"SECURITY.md", + b"docs/SECURITY.md", + ] + for path in blocked: + with ( + self.subTest(path=path), + mock.patch.object( + report, "tree_entries", side_effect=[{path: entry}, {path: entry}] + ), + mock.patch.object(report, "read_blobs") as read, + ): + with self.assertRaisesRegex(report.ReportError, "private or governance"): + report.copy_generated_tree(self.repo, "a" * 40, self.repo, "b" * 40) + read.assert_not_called() + with ( + mock.patch.object(report, "tree_entries", side_effect=[{b"unreleased.py": entry}, {}]), + mock.patch.object(report, "read_blobs") as read, + ): + with self.assertRaisesRegex( + report.ReportError, "absent from the approved public SDK tree" + ): + report.copy_generated_tree(self.repo, "a" * 40, self.repo, "b" * 40) + read.assert_not_called() + + def test_zero_new_files_summary_reports_existing_changes(self) -> None: + _, clean = self.baseline() + self.write("generated.py", "generated\n# existing customization\n") + base = self.commit() + self.write("generated.py", "generated\n# existing customization updated\n") + result, _ = report.build_report(self.repo, base, self.commit()) + body = report.render_report(result) + self.assertIn("✅ No new custom-code files detected.", body) + self.assertIn("1 mixed file remains; 1 existing customization changed.", body) + self.assertNotIn("0 newly customized", body) + empty, _ = report.build_report(self.repo, clean, clean) + self.assertIn( + "0 mixed files remain; 0 existing customizations changed.", report.render_report(empty) + ) + + def test_new_mixed_file_headline_includes_changed_generated_baselines(self) -> None: + _, base = self.baseline() + result, _ = report.build_report(self.repo, base, base) + for category in ("baseline_changed", "newly_generation_owned"): + result["files"] = [ + { + "path": "generated.py", + "category": category, + "custom_before": False, + "custom_after": True, + "added": "1", + "removed": "1", + } + ] + body = report.render_report(result) + self.assertNotIn("No new custom-code files", body) + self.assertIn("1 newly customized", body) + + @unittest.skipUnless(shutil.which("node"), "GitHub Actions JavaScript runtime") + def test_trusted_failure_publisher_updates_one_current_comment(self) -> None: + workflow = ( + Path(__file__).resolve().parents[2] / ".github/workflows/castiron-custom-code.yml" + ) + section = workflow.read_text().split("- name: Publish a trusted failure status\n", 1)[1] + script = textwrap.dedent(section.split("script: |\n", 1)[1]) + harness = r""" +const assert = require('node:assert/strict'); +const AsyncFunction = Object.getPrototypeOf(async function(){}).constructor; +async function check(stale, exists, priorRun, expected) { + const writes = []; + const event = {number: 1, head: {sha: 'a'.repeat(40)}, base: {sha: 'b'.repeat(40)}}; + const current = {...event, state: 'open', head: {sha: (stale ? 'c' : 'a').repeat(40)}}; + const previous = {id: 42, user: {type: 'Bot', login: 'github-actions[bot]'}, + body: `\n`}; + const github = {paginate: async () => exists ? [previous] : [], rest: { + pulls: {get: async () => ({data: current})}, + issues: {listComments() {}, updateComment: async x => writes.push(['update', x]), + createComment: async x => writes.push(['create', x])}}}; + const context = {payload: {pull_request: event}, repo: {owner: 'openai', repo: 'example'}, + runId: 20, serverUrl: 'https://github.com'}; + await new AsyncFunction('github', 'context', SCRIPT)(github, context); + assert.equal(writes.length, expected ? 1 : 0); + if (expected) { + assert.equal(writes[0][0], expected); + assert.match(writes[0][1].body, /Report unavailable/); + assert.match(writes[0][1].body, /castiron:run:v1:20:1/); + } +} +(async () => { + await check(false, true, 10, 'update'); + await check(false, false, 10, 'create'); + await check(true, true, 10, null); + await check(false, true, 21, null); +})().catch(error => { console.error(error); process.exitCode = 1; }); +""" + subprocess.run( + ["node", "-e", "const SCRIPT = " + json.dumps(script) + ";\n" + harness], + check=True, + env={**os.environ, "GITHUB_RUN_ATTEMPT": "1"}, + ) + + @unittest.skipUnless(shutil.which("jq"), "GitHub Actions jq runtime") + def test_workflow_branch_allowlist_is_case_sensitive(self) -> None: + workflow = ( + Path(__file__).resolve().parents[2] / ".github/workflows/castiron-custom-code.yml" + ) + body = workflow.read_text() + section = body.split("- name: Check the exact protected branch name\n", 1)[1] + script = textwrap.dedent(section.split("run: |\n", 1)[1].split("\n - name:", 1)[0]) + self.assertIn("needs.report.outputs.trusted == 'true'", body) + for branch, expected in (("castiron/demo", True), ("Castiron/demo", False)): + output = self.repo / "github-output" + output.write_text("") + result = subprocess.run( + ["bash", "-e", "-c", script], + env={ + **os.environ, + "ALLOWED_BRANCHES": '["castiron/demo"]', + "PR_BRANCH": branch, + "GITHUB_OUTPUT": str(output), + }, + capture_output=True, + ) + self.assertEqual(result.returncode == 0, expected) + self.assertEqual("trusted=true" in output.read_text(), expected) + + def test_removals_include_changed_baselines_but_not_handwritten_only_files(self) -> None: + _, base = self.baseline() + result, _ = report.build_report(self.repo, base, base) + result["files"] = [ + { + "path": category, + "category": category, + "custom_before": True, + "custom_after": False, + "added": "0", + "removed": "0", + } + for category in ("baseline_changed", "no_longer_generated") + ] + self.assertIn("1 customizations removed", report.render_report(result)) + result["files"].append( + { + "path": "new.py", + "category": "newly_customized", + "custom_before": False, + "custom_after": True, + "added": "1", + "removed": "0", + } + ) + self.assertIn("1 customizations removed", report.render_report(result)) + + def test_public_snapshot_has_no_private_history_and_reports_without_private_remote( + self, + ) -> None: + self.write("private-only.txt", "must never be published\n") + private_ancestor = self.commit("private history") + (self.repo / "private-only.txt").unlink() + generated, base = self.baseline() + self.git("branch", "codegen/test", generated) + private_remote = self.repo / "private.git" + self.git("clone", "--bare", str(self.repo), str(private_remote)) + self.git("remote", "add", "origin", str(private_remote)) + + public = self.repo / "public" + public.mkdir() + report.git(public, "init", "-q", "-b", "main") + report.git(public, "config", "user.name", "Public test") + report.git(public, "config", "user.email", "public@example.test") + (public / "generated.py").write_text("generated\n") + (public / ".castiron.stats.yml").write_text( + (self.repo / ".castiron.stats.yml").read_text().split("codegen_hash:")[0] + ) + report.git(public, "add", ".") + report.git(public, "commit", "-qm", "already public") + public_base = report.git(public, "rev-parse", "HEAD").decode().strip() + public_remote = self.repo / "public.git" + report.git(public, "clone", "--bare", str(public), str(public_remote)) + report.git(public, "remote", "add", "origin", str(public_remote)) + snapshot = report.prepare_public_snapshot( + self.repo, base, base, public, public_base, "castiron/promotions/pr-1" + ) + self.assertNotEqual(snapshot["commit"], generated) + self.assertEqual(report.git(public, "rev-list", "--count", snapshot["commit"]), b"1\n") + self.assertEqual(report.hash_codegen_commit(public, snapshot["commit"]), snapshot["hash"]) + with self.assertRaises(report.ReportError): + report.git(public, "cat-file", "-e", private_ancestor) + self.assertNotIn( + b"private-only.txt", report.git(public, "ls-tree", "-r", snapshot["commit"]) + ) + report.git( + public, "push", "origin", f"{snapshot['commit']}:refs/heads/{snapshot['branch']}" + ) + (public / "generated.py").write_text("generated\n# custom\n") + report.git(public, "add", ".") + report.git(public, "commit", "-qm", "public customization") + public_head = report.git(public, "rev-parse", "HEAD").decode().strip() + result, patch = report.build_report( + public, public_base, public_head, fetch=True, require_head_hash=True, public=True + ) + self.assertEqual(result["counts"]["newly_customized"], 1) + self.assertIn(b"+# custom", patch) + self.assertIn(" --public", report.render_report(result)) + self.assertFalse(result["before"]["hash_recorded"]) + again = report.prepare_public_snapshot( + self.repo, base, base, public, public_base, "castiron/promotions/pr-1" + ) + self.assertEqual(again["commit"], snapshot["commit"]) + tree = report.git(public, "rev-parse", f"{snapshot['commit']}^{{tree}}").decode().strip() + message = report.git(public, "show", "-s", "--format=%B", snapshot["commit"]) + chained = ( + report.git( + public, + "commit-tree", + tree, + "-p", + snapshot["commit"], + "-F", + "-", + input_bytes=message, + ) + .decode() + .strip() + ) + report.git( + public, "push", "--force", "origin", f"{chained}:refs/heads/{snapshot['branch']}" + ) + reused = report.prepare_public_snapshot( + self.repo, base, base, public, public_base, "castiron/promotions/pr-1" + ) + self.assertEqual(reused["commit"], chained) + + # A fresh generation may introduce files. Its public parent must be a + # previously public snapshot, never the private checkpoint parent. + next_generation = "650e8400-e29b-41d4-a716-446655440000" + self.git("checkout", "--detach", generated) + self.write("generated.py", "regenerated\n") + self.write("new.py", "new generated file\n") + self.git("add", "--", "generated.py", "new.py") + metadata = base64.b64encode( + json.dumps( + { + "generation_id": next_generation, + "source_branch": "test", + } + ).encode() + ).decode() + self.git("commit", "-qm", f"next codegen\n\nGeneration metadata: {metadata}") + next_codegen = self.git("rev-parse", "HEAD") + self.git("push", "origin", f"{next_codegen}:refs/heads/codegen/test") + self.write( + ".castiron.stats.yml", + ( + f"schema_version: 1\ngeneration_id: {next_generation}\n" + f"codegen_sha: {next_codegen}\n" + f"codegen_hash: {report.hash_codegen_commit(self.repo, next_codegen)}\n" + ), + ) + self.git("add", "--", ".castiron.stats.yml") + self.git("commit", "-qm", "next integrated SDK") + next_head = self.git("rev-parse", "HEAD") + # An unreviewed path in private HEAD is not enough to publish it. + with self.assertRaisesRegex(report.ReportError, "approved public SDK tree"): + report.prepare_public_snapshot( + self.repo, base, next_head, public, public_base, "castiron/promotions/pr-1" + ) + # Model normal promotion applying its reviewed patch to the public index. + (public / "generated.py").write_text("regenerated\n") + (public / "new.py").write_text("new generated file\n") + report.git(public, "add", "--", "generated.py", "new.py") + advanced = report.prepare_public_snapshot( + self.repo, base, next_head, public, public_base, "castiron/promotions/pr-1" + ) + self.assertEqual( + report.git(public, "rev-parse", advanced["commit"] + "^1").decode().strip(), chained + ) + self.assertEqual(report.hash_codegen_commit(public, advanced["commit"]), advanced["hash"]) + with self.assertRaises(report.ReportError): + report.git(public, "cat-file", "-e", next_codegen) + report.git( + public, "push", "origin", f"{advanced['commit']}:refs/heads/{advanced['branch']}" + ) + (public / "generated.py").write_text("regenerated\n") + (public / "new.py").write_text("new generated file\n") + report.git(public, "add", "--", ".castiron.stats.yml", "generated.py", "new.py") + report.git(public, "commit", "-qm", "next public SDK") + next_public = report.git(public, "rev-parse", "HEAD").decode().strip() + compared, _ = report.build_report( + public, public_base, next_public, fetch=True, require_head_hash=True, public=True + ) + self.assertEqual(compared["before"]["generation_id"], GENERATION) + self.assertEqual(compared["after"]["generation_id"], next_generation) + + def test_comment_updates_existing_bot_comment_and_skips_stale(self) -> None: + _, base = self.baseline() + result, _ = report.build_report(self.repo, base, base) + calls: list[tuple[str, str, object]] = [] + + def fake_api(method: str, path: str, payload: object = None) -> object: + calls.append((method, path, payload)) + if "/pulls/" in path: + return {"state": "open", "head": {"sha": base}, "base": {"sha": base}} + if "/actions/runs/" in path: + return { + "event": "pull_request", + "head_sha": base, + "run_attempt": 1, + "pull_requests": [{"number": 1}], + } + if "/comments?" in path: + return [ + {"id": 7, "user": {"login": "someone"}, "body": report.MARKER}, + { + "id": 8, + "user": {"login": "github-actions[bot]"}, + "body": report.MARKER, + "html_url": "existing", + }, + ] + return {"html_url": "updated"} + + with mock.patch.object(report, "api", side_effect=fake_api): + self.assertEqual(report.publish_comment(result, "openai/example", 1, 2, 1), "updated") + self.assertEqual(calls[-1][:2], ("PATCH", "repos/openai/example/issues/comments/8")) + calls.clear() + result["head_sha"] = "f" * 40 + self.assertEqual( + report.publish_comment(result, "openai/example", 1, 2, 1), "Skipped stale report" + ) + self.assertEqual(len(calls), 1) + + def test_comment_rejects_older_runs_attempts_and_wrong_pr(self) -> None: + _, base = self.baseline() + result, _ = report.build_report(self.repo, base, base) + pull = {"state": "open", "head": {"sha": base}, "base": {"sha": base}} + run = { + "event": "pull_request", + "head_sha": base, + "run_attempt": 2, + "pull_requests": [{"number": 1}], + } + comment = { + "id": 8, + "user": {"login": "github-actions[bot]"}, + "body": report.MARKER + "\n", + "html_url": "existing", + } + with mock.patch.object(report, "api", side_effect=[pull, run]) as api: + self.assertEqual( + report.publish_comment(result, "openai/example", 1, 2, 1), "Skipped stale report" + ) + self.assertEqual(api.call_count, 2) + with mock.patch.object(report, "api", side_effect=[pull, run, [comment]]) as api: + self.assertEqual( + report.publish_comment(result, "openai/example", 1, 2, 2), "Skipped stale report" + ) + self.assertEqual(api.call_count, 3) + with mock.patch.object(report, "api", side_effect=[pull, {**run, "pull_requests": []}]): + with self.assertRaisesRegex(report.ReportError, "does not match report PR"): + report.publish_comment(result, "openai/example", 1, 2, 2) + changed = {**pull, "head": {"sha": "f" * 40}} + with mock.patch.object(report, "api", side_effect=[pull, run, [], changed]) as api: + self.assertEqual( + report.publish_comment(result, "openai/example", 1, 2, 2), "Skipped stale report" + ) + self.assertEqual(api.call_count, 4) + + +if __name__ == "__main__": + unittest.main()