From 0b5a280c9f9b09bcb3947025d7ec52e6a40d6004 Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Mon, 16 Nov 2026 15:27:40 +0100 Subject: [PATCH] feat: keep the review guide's request in the review artifact A review now saves the body it POSTs to /v1/review-guide, the same xz-compressed bytes, as closed/bundle.json.xz in the review artifact. The artifact held the analyses and the guide but not the call graphs the guide is generated from, so no guide could be generated again on the same inputs. With the request kept, any change to the guide can be compared old against new on real pull requests. It is written only when the guide's request goes out, so never in sync mode, never for a provider the service cannot call, and never when the inputs are missing or over the service's limit. Writing it never fails the run or changes the guide. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 ++- action.yml | 3 +- scripts/remote_features.py | 22 +++++++++- tests/test_remote_features.py | 82 ++++++++++++++++++++++++++++++++++- 4 files changed, 107 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 31451aa..31c1d13 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,9 @@ required analysis inputs. The guide endpoint returns `{guide, usage, timingsMs}` saves the bare guide as `review_guide.json`. Architecture returns `{envelope, usage}`; its envelope is saved unchanged as `closed/architecture.json`, including partial, skipped and failed results. Failure of either request does not prevent the other or fail the review. +The guide's request body, the same xz-compressed bytes the service receives, is saved as +`closed/bundle.json.xz`, so the guide can be generated again later from the same inputs. It +never carries `source.json`, and it is left out when no guide request goes out. Only architecture receives optional `source.json`: tracked Python, JavaScript/TypeScript and C# files (`.py`, `.ts`, `.tsx`, `.js`, `.jsx`, `.mjs`, `.cjs`, `.cs`, `.csproj`), excluding @@ -406,7 +409,7 @@ The update runs after the baseline is saved, stops itself after 10 minutes and n | `github_token` | both | `${{ github.token }}` | Token for comments and sync delivery. | | `codeboarding_analysis_location` | both | `codeboarding_branch` | Where the synced branch's analysis is stored: `codeboarding_branch` (on `codeboarding/baseline`, for the default branch; the code is never written) or `in_place` (committed as `.codeboarding/` on the synced branch). Sync saves it there and review reads it from there. | | `warmstart_retention_days` | review | `1` | Days to keep the reusable analysis. Only the next run reads it. | -| `review_guide_url` | both | `https://jxivqwx4or6oip3krgdmdaiunm0iaybf.lambda-url.us-east-1.on.aws/` | Experimental. Shared base URL for `/v1/review-guide` and `/v1/architecture`, and for sync's `/v1/briefing` when `problem_history` is on. Saves `review_guide.json` and `closed/architecture.json` in the review artifact. Set to `''` to disable all three. Never fails the run. | +| `review_guide_url` | both | `https://jxivqwx4or6oip3krgdmdaiunm0iaybf.lambda-url.us-east-1.on.aws/` | Experimental. Shared base URL for `/v1/review-guide` and `/v1/architecture`, and for sync's `/v1/briefing` when `problem_history` is on. Saves `review_guide.json`, `closed/architecture.json` and the guide's request (`closed/bundle.json.xz`) in the review artifact. Set to `''` to disable all three. Never fails the run. | | `closed_features_source` | review | `true` | Whether architecture checks may read this repository's source. Only the architecture request carries it, capped in size, and the service keeps nothing. The model sees code around a finding, never the tree. With `false`, source-dependent checks and model review do not run, and the result says so. | | `problem_history` | sync | `false` | Opt-in. With `'true'`, each sync sends this repository's issues and merged pull requests to `review_guide_url` and commits the returned problem list as `problems.json` on the orphan branch `codeboarding/briefing`. Needs `contents: write`, `issues: read`, `pull-requests: read`, `actions: read` and `id-token: write` on the sync job. See [Problem history](#problem-history). Never fails the run. | diff --git a/action.yml b/action.yml index e0646b8..1524ae6 100644 --- a/action.yml +++ b/action.yml @@ -147,7 +147,8 @@ inputs: description: >- Experimental. Service base URL for POST /v1/review-guide and POST /v1/architecture. Defaults to production. Both receive this run's analysis independently; results are stored - as review_guide.json and closed/architecture.json. Set to an empty string to disable both. + as review_guide.json and closed/architecture.json, and the guide's request as sent as + closed/bundle.json.xz. Set to an empty string to disable both. Sync's opt-in problem_history calls POST /v1/briefing on the same URL. required: false default: 'https://jxivqwx4or6oip3krgdmdaiunm0iaybf.lambda-url.us-east-1.on.aws/' diff --git a/scripts/remote_features.py b/scripts/remote_features.py index fd5fc6e..e06e711 100644 --- a/scripts/remote_features.py +++ b/scripts/remote_features.py @@ -4,7 +4,8 @@ Runs after the review analysis. The two requests go out in parallel and independently: either one failing leaves the other alone, and nothing here fails the run. The guide is stored as review_guide.json and the architecture envelope as closed/architecture.json in the review artifact, -including partial, skipped and failed results. +including partial, skipped and failed results. The guide's request, exactly as sent, is stored as +closed/bundle.json.xz, so a guide can be generated again from the same inputs. What a request sends -------------------- @@ -110,6 +111,8 @@ SOURCE_EXTENSIONS = (".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".cs", ".csproj") SOURCE_EXCLUDE = re.compile(r"(^|/)(node_modules|dist|build|vendor|__fixtures__|docs|public)/|\.min\.js$|\.d\.ts$") MAX_SOURCE_BYTES = 16_000_000 +# The review guide's request as sent, kept in the review artifact's closed/ directory (see keep_bundle). +BUNDLE_NAME = "bundle.json.xz" class Skip(Exception): @@ -675,10 +678,27 @@ def _no_architecture(ctx: dict, status: str, reason: str) -> dict: } +def keep_bundle(artifact: Path, data: bytes | None) -> None: + """Save the review guide's request, the very bytes sent, as closed/bundle.json.xz, so the guide can + be generated again later from the same inputs. A missing or oversized body is left out with one + line in the log; this never fails the run or changes the guide.""" + try: + if not data: + raise Skip("no request body") + if len(data) > MAX_COMPRESSED_BYTES: + raise Skip(f"{len(data) / 1e6:.1f} MB is over the service's limit") + path = artifact / "closed" / BUNDLE_NAME + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + except Exception as error: + print(f"::notice::review-guide bundle not saved: {error}") + + def run_review_guide(service_url: str, data: bytes, headers: dict, artifact: Path) -> None: try: if "x-llm-provider" in headers and "x-llm-api-key" not in headers: raise Skip("the service cannot call this run's provider") + keep_bundle(artifact, data) answer = _post(f"{service_url.rstrip('/')}/v1/review-guide", data, headers) guide = answer.get("guide") if not isinstance(guide, dict): diff --git a/tests/test_remote_features.py b/tests/test_remote_features.py index 2d80da5..dc07399 100644 --- a/tests/test_remote_features.py +++ b/tests/test_remote_features.py @@ -416,7 +416,9 @@ def test_unsupported_self_hosted_and_missing_key_never_fall_back(self): self.assertEqual(remote_features.model_headers(env), {"x-llm-provider": env["LLM"]}) -class EndpointTests(unittest.TestCase): +class _ServiceCase(unittest.TestCase): + """A workspace whose inputs are stubbed, and main run against a local service.""" + def setUp(self): tmp = tempfile.TemporaryDirectory() self.addCleanup(tmp.cleanup) @@ -451,12 +453,15 @@ def run_service(self, respond=None): self.assertEqual(remote_features.main(self.env), 0) return service + +class EndpointTests(_ServiceCase): def test_exactly_two_posts_shared_inputs_and_fixed_outputs(self): service = self.run_service() self.assertEqual(_read(self.env, "review_guide.json"), GUIDE) self.assertEqual(_read(self.env, "closed/architecture.json"), ENVELOPE) self.assertEqual( - sorted(p.name for p in (Path(self.env["ARTIFACT_DIR"]) / "closed").iterdir()), ["architecture.json"] + sorted(p.name for p in (Path(self.env["ARTIFACT_DIR"]) / "closed").iterdir()), + ["architecture.json", "bundle.json.xz"], ) self.assertEqual(sorted(r["path"] for r in service.requests), ["/v1/architecture", "/v1/review-guide"]) bodies = {} @@ -573,6 +578,79 @@ def test_either_limit_omits_source_then_skips_if_required_inputs_too_large(self) self.assertEqual(_read(self.env, "closed/architecture.json")["status"], "skipped") +class KeptBundleTests(_ServiceCase): + """The review guide's request is kept in the review artifact as closed/bundle.json.xz, byte for byte, + so a guide can be generated again from the same inputs. Keeping it never fails or changes the run.""" + + def bundle(self) -> Path: + return Path(self.env["ARTIFACT_DIR"]) / "closed" / "bundle.json.xz" + + def test_written_as_the_exact_bytes_the_guide_sent(self): + service = self.run_service() + sent = next(r["body"] for r in service.requests if r["path"] == "/v1/review-guide") + self.assertEqual(self.bundle().read_bytes(), sent) + self.assertEqual(set(json.loads(lzma.decompress(sent))["files"]), INPUTS) + + def test_kept_without_architectures_source(self): + with mock.patch.object(remote_features, "collect_source", return_value=('{"a.py": "x"}', None)): + service = self.run_service() + architecture = next(r["body"] for r in service.requests if r["path"] == "/v1/architecture") + self.assertNotEqual(self.bundle().read_bytes(), architecture) + self.assertNotIn("source.json", json.loads(lzma.decompress(self.bundle().read_bytes()))["files"]) + + def test_kept_when_the_guide_request_went_out_but_failed(self): + self.run_service(lambda p, b: (500, {}) if p.endswith("review-guide") else (200, {"envelope": ENVELOPE})) + self.assertTrue(self.bundle().is_file()) + self.assertFalse((Path(self.env["ARTIFACT_DIR"]) / "review_guide.json").exists()) + + def assert_not_kept(self): + self.assertFalse(self.bundle().exists()) + self.assertTrue((Path(self.env["ARTIFACT_DIR"]) / "closed" / "architecture.json").is_file()) + + def test_not_written_when_inputs_are_missing(self): + (Path(self.env["HEAD_ANALYSIS_PATH"]).parent / "static_analysis.pkl").unlink() + self.assertEqual(self.run_service().requests, []) + self.assert_not_kept() + + def test_not_written_when_the_guide_cannot_call_the_provider(self): + self.env.update(LLM="litellm", LITELLM_API_KEY="secret") + self.assertEqual([r["path"] for r in self.run_service().requests], ["/v1/architecture"]) + self.assert_not_kept() + + def test_not_written_over_the_service_limit(self): + with mock.patch.object(remote_features, "MAX_COMPRESSED_BYTES", 1): + self.assertEqual(self.run_service().requests, []) + self.assert_not_kept() + + def test_missing_or_oversized_body_is_skipped_with_one_line(self): + artifact = Path(self.env["ARTIFACT_DIR"]) + for data in (None, b"", b"x" * (remote_features.MAX_COMPRESSED_BYTES + 1)): + with mock.patch("builtins.print") as printed: + remote_features.keep_bundle(artifact, data) + self.assertEqual(printed.call_count, 1) + self.assertIn("bundle not saved", printed.call_args.args[0]) + self.assertFalse(self.bundle().exists()) + + def test_a_failed_write_changes_nothing_else(self): + with mock.patch.object(Path, "write_bytes", side_effect=OSError("disk full")): + self.run_service() + self.assertFalse(self.bundle().exists()) + self.assertEqual(_read(self.env, "review_guide.json"), GUIDE) + self.assertEqual(_read(self.env, "closed/architecture.json"), ENVELOPE) + + def test_never_in_sync_mode(self): + # Only the review-mode step runs this script; sync's one remote call is the problem history, which + # writes no artifact at all. + action = (_SCRIPT.parent.parent / "action.yml").read_text(encoding="utf-8") + steps = [step for step in action.split("\n - name: ") if "scripts/remote_features.py" in step] + self.assertEqual(len(steps), 1) + self.assertIn("steps.guard.outputs.mode == 'review'", steps[0]) + self.assertNotIn("'sync'", steps[0]) + problem_history = (_SCRIPT.parent / "problem_history.py").read_text(encoding="utf-8") + for writes in ("bundle.json", "keep_bundle", "run_review_guide", "write_bytes"): + self.assertNotIn(writes, problem_history) + + class SourceTests(unittest.TestCase): def test_tracked_extensions_exclusions_limits_and_symlinks(self): with tempfile.TemporaryDirectory() as tmp: