From 11346426c73b712bb7835d929d8c05cf1b31905e Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Wed, 11 Nov 2026 18:09:10 +0100 Subject: [PATCH 1/7] feat: send the flow bundle as one xz document per side Large repos got no flows or merge risk: crate's bundle was 24.6 MB of gzip, over the service's 4.3 MB limit, because every fact went four times. 149,315 of its 149,316 analysis edges are call-graph edges, nested relations repeat the top level, and the methods index and the graph's nodes are one table. - `side_document()` writes each commit as one document, `head.json` and `base.json`, in place of the two analyses and two call graphs. It holds a row per method or graph node, calls by caller with their lines, the component tree, and only the analysis edges the calls do not imply. The service's `bundle.expand_side` reads it back into every field a feature reads. - `_compress` is xz -9e instead of gzip: its window spans the bundle, so the base side costs little next to a head that repeats most of it. The briefing request shares it. - A changed file over 300 KB is sent at neither commit (the same limit as source). On crate the bundle is 1.42 MB. Needs CodeBoarding/licensing-aws#56 deployed first. Co-Authored-By: Claude Opus 5.5 --- scripts/closed_features.py | 177 ++++++++++++++++++++++++++++++---- scripts/problem_history.py | 6 +- tests/test_closed_features.py | 156 +++++++++++++++++++++++++++--- tests/test_problem_history.py | 8 +- 4 files changed, 309 insertions(+), 38 deletions(-) diff --git a/scripts/closed_features.py b/scripts/closed_features.py index 29cc3b7..9ed09ac 100644 --- a/scripts/closed_features.py +++ b/scripts/closed_features.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 """Call the review-guide and architecture endpoints independently after the review analysis. -Both requests carry the same analyses, call graphs, PR metadata and changed files. Only architecture -carries optional source. Store the legacy guide in review_guide.json and the unchanged architecture +Both requests carry the same two sides, PR metadata and changed files. Only architecture carries +optional source. A side is one document holding what the service reads from that commit's analysis and +call graph (see side_document), and the bundle is xz: its window spans the whole bundle, so the base +side costs little next to a head that repeats most of it. Store the legacy guide in review_guide.json and the unchanged architecture envelope in closed/architecture.json, including partial, skipped and failed results. Credentials follow the analysis: a run that chose its own provider uses its own key and model for @@ -21,9 +23,9 @@ from __future__ import annotations -import gzip import importlib.util import json +import lzma import os import pickle import re @@ -35,11 +37,13 @@ from urllib.parse import quote, urlparse from urllib.request import Request, urlopen -# A Function URL takes 6 MB, measured after base64, and a gzipped body goes through base64 on the way -# in: 4.5 MB of bundle arrives, and 5 MB is refused with a 413. +# A Function URL takes 6 MB, measured after base64, and a compressed body goes through base64 on the +# way in: 4.5 MB of bundle arrives, and 5 MB is refused with a 413. MAX_BUNDLE_BYTES = 4_300_000 # Changed files whose contents are sent at both commits; past this a feature reads diffs only. MAX_FILES = 300 +# A file larger than this is sent neither as a changed file's contents nor as source. +MAX_FILE_BYTES = 300_000 # Fixed hosts matching closed_functions/llm.py; never supplied by a remote service. PROVIDER_HOSTS = { "ollama": "ollama.com", @@ -59,7 +63,6 @@ } SOURCE_EXTENSIONS = (".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".cs", ".csproj") SOURCE_EXCLUDE = re.compile(r"(^|/)(node_modules|dist|build|vendor|__fixtures__|docs|public)/|\.min\.js$|\.d\.ts$") -MAX_SOURCE_FILE_BYTES = 300_000 MAX_SOURCE_BYTES = 16_000_000 @@ -130,6 +133,142 @@ def call_graph(pkl: Path) -> dict: return {"nodes": nodes, "edges": edges} +def _short(q: str, path: str) -> str: + """A qualified name as a row writes it: `~.rest` when it is the file's dotted path plus `.rest`, and + a leading `~` doubled so a name of its own that starts with one reads back unchanged.""" + stem = path.rsplit(".", 1)[0].replace("/", ".") + if q.startswith(stem + "."): + return "~" + q[len(stem) :] + return "~" + q if q.startswith("~") else q + + +def side_document(analysis: dict, cg: dict) -> dict: + """One commit's analysis and call graph as the service reads them, each fact once. The service's + bundle.expand_side turns it back into both, in every field a feature reads. + + - files: per file, rows for the methods_index entries `[name, start, end, type, content_hash, cb]` + and for call-graph nodes the index lacks `[name, start, end, cb]`. A method and the graph node of + the same name share a row; cb is -1 for a method that is no graph node. A name under its file's + dotted path is written `~.rest`. Rows are numbered in order across files; `unplaced` continues the + numbering for call ends that are no node. + - calls: per caller, `[callee, line, ...]`, one line per call site. No reader takes the column, and a + site's file is always its caller's. + - components: `[id, name, description, parent, members, relations]` in tree order. Members are rows + (a key where the index lacks the method), or null for a parent holding exactly its children's. + - The analysis's edges are the calls between two different leaves (a row's leaf is the first one + listing it), less `hidden_edges`, plus `extra_edges`. On crate that leaves one extra edge of + 149,316: the analysis and the graph repeat the same edges, and only their set is read.""" + index = analysis.get("methods_index") or {} + files: dict[str, list[list]] = {} + row_of: dict[str, list] = {} # methods_index key -> its row + by_name: dict[str, list[tuple[str, list]]] = {} + for key, m in index.items(): + path, q = m.get("file_path") or "", m.get("qualified_name") or "" + if key != f"{path}|{q}": + raise ValueError(f"methods_index key {key!r} is not its file and qualified name") + row = [q, m.get("start_line"), m.get("end_line"), m.get("type"), m.get("content_hash"), -1] + files.setdefault(path, []).append(row) + row_of[key] = row + by_name.setdefault(q, []).append((path, row)) + node_row: dict[str, list] = {} + spans: list[tuple[list, int, int]] = [] + for q, n in (cg.get("nodes") or {}).items(): + same = by_name.get(q) or [] + hit = next((r for p, r in same if p == n.get("file")), same[0][1] if same else None) + if hit is None: + hit = [q, n.get("start"), n.get("end"), int(bool(n.get("cb")))] + files.setdefault(n.get("file") or "", []).append(hit) + else: + hit[5] = int(bool(n.get("cb"))) + if (hit[1], hit[2]) != (n.get("start"), n.get("end")): + spans.append((hit, n.get("start"), n.get("end"))) + node_row[q] = hit + position: dict[int, int] = {} + out_files = [] + for path in sorted(files): + rows = sorted(files[path], key=lambda r: (r[1] is None, r[1] or 0, r[0])) + for r in rows: + position[id(r)] = len(position) + out_files.append([path, [[_short(r[0], path), *r[1:]] for r in rows]]) + at = {q: position[id(r)] for q, r in node_row.items()} + unplaced = [] + calls: dict[int, list] = {} + for e in cg.get("edges") or []: + for q in (e["s"], e["d"]): + if q not in at: + at[q] = len(position) + len(unplaced) + unplaced.append(q) + lines = [s.get("line") if isinstance(s, dict) else None for s in e.get("sites") or []] + calls.setdefault(at[e["s"]], []).append([at[e["d"]], *lines]) + + def token(key): + return position[id(row_of[key])] if key in row_of else key + + comps: list[tuple[dict, int]] = [] + + def walk(cs, parent): + for c in cs or []: + comps.append((c, parent)) + walk(c.get("components"), len(comps) - 1) + + walk(analysis.get("components"), -1) + children: list[list[int]] = [[] for _ in comps] + for i, (_, parent) in enumerate(comps): + if parent >= 0: + children[parent].append(i) + members: list[list] = [ + [token(f"{g['file_path']}|{q}") for g in c.get("file_methods") or [] for q in g.get("methods") or []] + for c, _ in comps + ] + implied: list[set] = [set() for _ in comps] + for i in reversed(range(len(comps))): + implied[i] = set(members[i]) if not children[i] else set().union(*(implied[k] for k in children[i])) + relations = lambda rows: [[r.get("src_id"), r.get("dst_id"), r.get("relation")] for r in rows or []] # noqa: E731 + out_comps = [ + [ + c.get("component_id"), + c.get("name"), + c.get("description"), + parent, + None if children[i] and set(members[i]) == implied[i] else members[i], + relations(c.get("components_relations")), + ] + for i, (c, parent) in enumerate(comps) + ] + + leaf: dict[int, int] = {} + for i, own in enumerate(members): + if not children[i]: + for m in own: + if isinstance(m, int): + leaf.setdefault(m, i) + implied_edges = { + (s, d) for s, ds in calls.items() for d, *_ in ds if s in leaf and d in leaf and leaf[s] != leaf[d] + } + edges: set[tuple] = set() + + def collect_edges(o): + for r in o.get("components_relations") or []: + for e in r.get("all_edges") or []: + if e.get("source") and e.get("target"): + edges.add((token(e["source"]), token(e["target"]))) + for c in o.get("components") or []: + collect_edges(c) + + collect_edges(analysis) + return { + "version": 1, + "files": out_files, + "unplaced": unplaced, + "graph_spans": [[position[id(r)], start, end] for r, start, end in spans], + "components": out_comps, + "relations": relations(analysis.get("components_relations")), + "calls": [[s, calls[s]] for s in sorted(calls)], + "extra_edges": sorted(edges - implied_edges, key=repr), + "hidden_edges": sorted(implied_edges - edges), + } + + def _diff(repo: str, base: str, head: str) -> list[dict]: """The files changed between the two analysed commits, in the shape of a pull request's files (the compare API returns the same fields), so a feature reads exactly what was analysed.""" @@ -177,19 +316,25 @@ def collect(env: dict) -> dict: if f.get("status") != "removed": try: p = _checkout_file(checkout, f["filename"]) - files[f"{head_sha}:{f['filename']}"] = p.read_text(encoding="utf-8") + if p.stat().st_size <= MAX_FILE_BYTES: + files[f"{head_sha}:{f['filename']}"] = p.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): pass if f.get("status") != "added": was = f.get("previous_filename") or f["filename"] text = _contents(repo, base_sha, was) - if text is not None: + if text is not None and len(text.encode()) <= MAX_FILE_BYTES: files[f"{base_sha}:{was}"] = text bundle_meta = {**meta, "base_sha": base_sha, "head_sha": head_sha, "pr_number": str(pr)} + + def side(analysis_path: str, directory: Path) -> str: + analysis = json.loads(Path(analysis_path).read_text()) + return json.dumps(side_document(analysis, call_graph(directory / "static_analysis.pkl")), separators=(",", ":")) + inputs = { - "analysis.json": Path(env["HEAD_ANALYSIS_PATH"]).read_text(), - "base/analysis.json": Path(env["BASE_ANALYSIS_PATH"]).read_text(), + "head.json": side(env["HEAD_ANALYSIS_PATH"], head_dir), + "base.json": side(env["BASE_ANALYSIS_PATH"], base_dir), "metadata.json": json.dumps(bundle_meta), "pr.json": json.dumps( { @@ -200,8 +345,6 @@ def collect(env: dict) -> dict: } ), "pr-files.json": json.dumps(pr_files), - "head-cg.json": json.dumps(call_graph(head_dir / "static_analysis.pkl")), - "base-cg.json": json.dumps(call_graph(base_dir / "static_analysis.pkl")), "files.json": json.dumps(files), } return inputs @@ -301,7 +444,7 @@ def collect_source(env: dict) -> tuple[str | None, str | None]: try: path = _checkout_file(checkout, name) size = path.stat().st_size - if size > MAX_SOURCE_FILE_BYTES or not path.is_file(): + if size > MAX_FILE_BYTES or not path.is_file(): continue files[name] = path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): @@ -332,7 +475,7 @@ def _post(url: str, body: bytes, headers: dict[str, str], timeout: int = 600) -> req = Request( url, data=body, - headers={**headers, "Content-Type": "application/gzip", "User-Agent": "codeboarding-action"}, + headers={**headers, "Content-Type": "application/x-xz", "User-Agent": "codeboarding-action"}, method="POST", ) try: @@ -345,15 +488,15 @@ def _post(url: str, body: bytes, headers: dict[str, str], timeout: int = 600) -> raise RuntimeError(reason) from e -def _gzip(body: dict) -> bytes: - return gzip.compress(json.dumps(body).encode(), compresslevel=9) +def _compress(body: dict) -> bytes: + return lzma.compress(json.dumps(body).encode(), preset=9 | lzma.PRESET_EXTREME) def _pack(ctx: dict, files: dict[str, str], omitted: dict[str, str]) -> bytes: body = {"version": 1, "repo": ctx["repo"], "pr": ctx["pr"], "files": files} if omitted: body["omitted"] = omitted - return _gzip(body) + return _compress(body) def _check_size(data: bytes, what: str = "bundle") -> None: diff --git a/scripts/problem_history.py b/scripts/problem_history.py index 96d0503..0208630 100644 --- a/scripts/problem_history.py +++ b/scripts/problem_history.py @@ -38,7 +38,7 @@ _check_size, _error_message, _github, - _gzip, + _compress, _post, ) # noqa: E402 @@ -685,11 +685,11 @@ def build_request(env: dict, ledger: dict | None, facts: dict) -> dict: def pack(body: dict) -> tuple[bytes, list[str]]: - """The gzipped request and the pull requests whose findings were left out of it: the largest + """The compressed request and the pull requests whose findings were left out of it: the largest envelopes go first while it is over the service's limit, and their merges stay unread for a later run.""" findings, dropped = dict(body.get("findings") or {}), [] while True: - data = _gzip({**body, "findings": findings}) + data = _compress({**body, "findings": findings}) if len(data) <= MAX_BUNDLE_BYTES or not findings: _check_size(data, "request") return data, dropped diff --git a/tests/test_closed_features.py b/tests/test_closed_features.py index 8ba89b1..4bc7cd3 100644 --- a/tests/test_closed_features.py +++ b/tests/test_closed_features.py @@ -1,8 +1,8 @@ """Direct endpoint, source and provider contracts for the optional review requests.""" -import gzip import importlib.util import json +import lzma import os import subprocess import sys @@ -20,16 +20,7 @@ sys.modules[_SPEC.name] = closed_features _SPEC.loader.exec_module(closed_features) -INPUTS = { - "analysis.json", - "base/analysis.json", - "metadata.json", - "pr.json", - "pr-files.json", - "head-cg.json", - "base-cg.json", - "files.json", -} +INPUTS = {"head.json", "base.json", "metadata.json", "pr.json", "pr-files.json", "files.json"} GUIDE = {"version": 1, "changes": [{"id": "u1"}]} ENVELOPE = { "feature": "architecture", @@ -134,6 +125,18 @@ def test_analysed_commits_and_both_sides(self): self.assertEqual(json.loads(bundle["metadata.json"])["base_sha"], "B") self.assertEqual(json.loads(bundle["pr.json"])["commits"], ["Return two"]) self.assertEqual(set(bundle), INPUTS) + self.assertEqual(json.loads(bundle["head.json"])["version"], 1) + + def test_a_changed_file_over_the_size_limit_is_sent_at_neither_commit(self): + with tempfile.TemporaryDirectory() as tmp: + env = _workspace(Path(tmp)) + with ( + mock.patch.object(closed_features, "_github_json", side_effect=_github_json), + mock.patch.object(closed_features, "_contents", return_value="b" * 13), + mock.patch.object(closed_features, "call_graph", return_value={"nodes": {}, "edges": []}), + mock.patch.object(closed_features, "MAX_FILE_BYTES", 12), + ): + self.assertEqual(json.loads(closed_features.collect(env)["files.json"]), {"H:app.py": "head content"}) def test_changed_file_symlink_is_not_read(self): with tempfile.TemporaryDirectory() as tmp: @@ -149,6 +152,131 @@ def test_changed_file_symlink_is_not_read(self): self.assertEqual(json.loads(closed_features.collect(env)["files.json"]), {}) +def _method(q, file, start, end, h): + return { + "file_path": file, + "qualified_name": q, + "start_line": start, + "end_line": end, + "type": "FUNCTION", + "content_hash": h, + } + + +class SideDocumentTests(unittest.TestCase): + """The service's bundle.expand_side reads this exact document back (its tests hold the same one).""" + + ANALYSIS = { + "methods_index": { + "pkg/a.py|pkg.a.run": _method("pkg.a.run", "pkg/a.py", 1, 5, "h1"), + "pkg/a.py|pkg.a.helper": _method("pkg.a.helper", "pkg/a.py", 6, 8, "h2"), + "pkg/b.py|pkg.b.save": _method("pkg.b.save", "pkg/b.py", 1, 4, "h3"), + "pkg/b.py|~Weird": _method("~Weird", "pkg/b.py", 10, 12, "h4"), + }, + "components": [ + { + "component_id": "1", + "name": "Core", + "description": "d1", + "file_methods": [ + {"file_path": "pkg/a.py", "methods": ["pkg.a.run", "pkg.a.helper"]}, + {"file_path": "pkg/b.py", "methods": ["pkg.b.save", "~Weird"]}, + ], + "components": [ + { + "component_id": "1.1", + "name": "Run", + "file_methods": [{"file_path": "pkg/a.py", "methods": ["pkg.a.run", "pkg.a.helper"]}], + }, + { + "component_id": "1.2", + "name": "Store", + "file_methods": [{"file_path": "pkg/b.py", "methods": ["pkg.b.save", "~Weird"]}], + }, + ], + "components_relations": [ + { + "src_id": "1.1", + "dst_id": "1.2", + "relation": "saves", + "src_name": "Run", + "all_edges": [ + { + "source": "pkg/a.py|pkg.a.run", + "target": "pkg/b.py|pkg.b.save", + "call_sites": [{"line": 3}], + }, + {"source": "pkg/a.py|pkg.a.helper", "target": "pkg/b.py|pkg.b.save"}, + ], + } + ], + }, + { + "component_id": "2", + "name": "Also", + "file_methods": [ + {"file_path": "pkg/b.py", "methods": ["pkg.b.save"]}, + {"file_path": "pkg/c.py", "methods": ["pkg.c.gone"]}, + ], + }, + ], + "components_relations": [ + { + "src_id": "1", + "dst_id": "2", + "relation": "uses", + "key_edges": [], + "all_edges": [{"source": "pkg/a.py|pkg.a.run", "target": "pkg/b.py|pkg.b.save"}], + } + ], + "files": {"pkg/a.py": {"method_keys": []}}, + "metadata": {"depth_level": 2}, + } + GRAPH = { + "nodes": { + "pkg.a.run": {"file": "pkg/a.py", "start": 1, "end": 5, "callable": True, "cb": False}, + "pkg.a.helper": {"file": "pkg/a.py", "start": 6, "end": 9, "callable": True, "cb": False}, + "pkg.b.save": {"file": "pkg/b.py", "start": 1, "end": 4, "callable": True, "cb": False}, + "pkg.x.Only": {"file": "pkg/x.py", "start": 1, "end": 2, "callable": False, "cb": True}, + }, + "edges": [ + {"s": "pkg.a.run", "d": "pkg.b.save", "sites": [{"line": 3, "file": "pkg/a.py", "column": 9}, {"line": 4}]}, + {"s": "pkg.a.run", "d": "pkg.a.helper", "sites": [{"line": 2, "file": "pkg/a.py", "column": 1}]}, + {"s": "pkg.b.save", "d": "pkg.a.run", "sites": [{"line": 2}]}, + {"s": "pkg.x.Only", "d": "ext.lib.call", "sites": []}, + ], + } + DOCUMENT = { + "version": 1, + "files": [ + ["pkg/a.py", [["~.run", 1, 5, "FUNCTION", "h1", 0], ["~.helper", 6, 8, "FUNCTION", "h2", 0]]], + ["pkg/b.py", [["~.save", 1, 4, "FUNCTION", "h3", 0], ["~~Weird", 10, 12, "FUNCTION", "h4", -1]]], + ["pkg/x.py", [["~.Only", 1, 2, 1]]], + ], + "unplaced": ["ext.lib.call"], + "graph_spans": [[1, 6, 9]], + "components": [ + ["1", "Core", "d1", -1, None, [["1.1", "1.2", "saves"]]], + ["1.1", "Run", None, 0, [0, 1], []], + ["1.2", "Store", None, 0, [2, 3], []], + ["2", "Also", None, -1, [2, "pkg/c.py|pkg.c.gone"], []], + ], + "relations": [["1", "2", "uses"]], + "calls": [[0, [[2, 3, 4], [1, 2]]], [2, [[0, 2]]], [4, [[5]]]], + "extra_edges": [[1, 2]], + "hidden_edges": [[2, 0]], + } + + def test_each_fact_once_and_only_what_the_service_reads(self): + doc = json.loads(json.dumps(closed_features.side_document(self.ANALYSIS, self.GRAPH))) + self.assertEqual(doc, self.DOCUMENT) + + def test_a_key_that_is_not_its_file_and_name_is_refused(self): + odd = {"methods_index": {"pkg/a.py|other": _method("pkg.a.run", "pkg/a.py", 1, 5, "h1")}} + with self.assertRaises(ValueError): + closed_features.side_document(odd, {}) + + class CredentialTests(unittest.TestCase): def test_without_oidc_the_runs_own_token_proves_the_repository(self): self.assertEqual(closed_features._bearer({"GITHUB_TOKEN": "ghs_run"}), {"Authorization": "Bearer ghs_run"}) @@ -258,7 +386,7 @@ def test_exactly_two_posts_shared_inputs_and_fixed_outputs(self): for request in service.requests: self.assertEqual(request["method"], "POST") self.assertEqual(request["headers"]["Authorization"], "Bearer ghs_run") - bodies[request["path"]] = json.loads(gzip.decompress(request["body"])) + bodies[request["path"]] = json.loads(lzma.decompress(request["body"])) guide, arch = bodies["/v1/review-guide"], bodies["/v1/architecture"] self.assertEqual(set(guide["files"]), INPUTS) self.assertEqual(set(arch["files"]), INPUTS | {"source.json"}) @@ -333,7 +461,7 @@ def test_source_toggle_and_read_failure_only_omit_architecture_source(self): ): service = self.run_service() for request in service.requests: - body = json.loads(gzip.decompress(request["body"])) + body = json.loads(lzma.decompress(request["body"])) self.assertNotIn("source.json", body["files"]) self.assertEqual("omitted" in body, request["path"] == "/v1/architecture") @@ -344,7 +472,7 @@ def test_bundle_limit_omits_source_then_skips_if_required_inputs_too_large(self) ): service = self.run_service() arch = next(r for r in service.requests if r["path"] == "/v1/architecture") - body = json.loads(gzip.decompress(arch["body"])) + body = json.loads(lzma.decompress(arch["body"])) self.assertNotIn("source.json", body["files"]) self.assertIn("too large", body["omitted"]["source.json"]) with mock.patch.object(closed_features, "MAX_BUNDLE_BYTES", 1): diff --git a/tests/test_problem_history.py b/tests/test_problem_history.py index 95f9eec..73e5c47 100644 --- a/tests/test_problem_history.py +++ b/tests/test_problem_history.py @@ -1,9 +1,9 @@ """Problem history: the facts a sync run collects, the request it sends, and the ledger commit it makes.""" -import gzip import importlib.util import io import json +import lzma import random import re import sys @@ -218,7 +218,7 @@ def _patched(**names): def _service(seen, problems=3): def post(url, data, headers, timeout=None): - body = json.loads(gzip.decompress(data)) + body = json.loads(lzma.decompress(data)) seen.append({"url": url, "body": body, "headers": headers, "timeout": timeout}) merged = (body["ledger"] or {}).get("merged", []) + body["merged"] answer = {"schema": 1, "repo": body["repo"], "merged": merged, "problems": [{}] * problems} @@ -793,7 +793,7 @@ def test_findings_that_would_overflow_the_request_are_left_out(self): with mock.patch.dict(ph._check_size.__globals__, {"MAX_BUNDLE_BYTES": 1000}): with mock.patch.object(ph, "MAX_BUNDLE_BYTES", 1000): data, dropped = ph.pack(body) - self.assertEqual((json.loads(gzip.decompress(data))["findings"], dropped), ({"41": small}, ["42"])) + self.assertEqual((json.loads(lzma.decompress(data))["findings"], dropped), ({"41": small}, ["42"])) gh = FakeGitHub() gh.merged_pages = [[pr_node(41, "2026-11-06T00:00:00Z"), pr_node(42, "2026-11-05T00:00:00Z")]] @@ -1008,7 +1008,7 @@ def test_the_step_uses_only_keys_a_composite_run_step_accepts(self): self.assertIn("continue-on-error", keys) def test_shares_the_closed_features_transport(self): - for shared in (ph._bearer, ph._post, ph._github, ph._gzip, ph._check_size, ph._error_message): + for shared in (ph._bearer, ph._post, ph._github, ph._compress, ph._check_size, ph._error_message): self.assertEqual(shared.__module__, "closed_features") From 6847c8984200dd4600c8c05adc2b09ca383e3872 Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Wed, 11 Nov 2026 19:10:04 +0100 Subject: [PATCH 2/7] refactor: send only the analysis edges that are no call The service reads every call already, so the analysis edges that repeat one were rebuilt on its side from leaf membership and then corrected with hidden_edges. Without them crate's guide holds the same content and its peak memory drops from 1.8 to 1.3 GB, so extra_edges is now just the analysis edges that are no call, and the derivation and hidden_edges go. call_graph() no longer builds what side_document dropped: `callable`, and each site's file and column. It returns each call's lines. Co-Authored-By: Claude Opus 5.5 --- scripts/closed_features.py | 44 ++++++++++++----------------------- tests/test_closed_features.py | 17 +++++++------- 2 files changed, 23 insertions(+), 38 deletions(-) diff --git a/scripts/closed_features.py b/scripts/closed_features.py index 9ed09ac..dc73acd 100644 --- a/scripts/closed_features.py +++ b/scripts/closed_features.py @@ -109,8 +109,9 @@ def _github_json(path: str): def call_graph(pkl: Path) -> dict: - """A static_analysis.pkl's call graph as the service reads it: nodes and edges with call sites. - The pickle holds CodeBoarding's own classes, which the installed engine provides.""" + """A static_analysis.pkl's call graph as the service reads it: each node's file, span and whether it + is a callback, and each call's site lines (a site's file is always its caller's, and no reader takes + the column). The pickle holds CodeBoarding's own classes, which the installed engine provides.""" with pkl.open("rb") as f: results = pickle.load(f) # nosec B301 - written by this run's own analysis nodes, edges = {}, [] @@ -121,15 +122,14 @@ def call_graph(pkl: Path) -> dict: "file": n.file_path, "start": n.line_start, "end": n.line_end, - "callable": bool(n.is_callable()), "cb": bool(n.is_callback_or_anonymous()), } for e in graph.edges: - sites = [] - for s in vars(e).get("_call_sites") or []: - v = s if isinstance(s, (tuple, list, dict)) else vars(s) if hasattr(s, "__dict__") else str(s) - sites.append(list(v) if isinstance(v, tuple) else v) - edges.append({"s": e.get_source(), "d": e.get_destination(), "sites": sites}) + lines = [ + s.get("line") if isinstance(s, dict) else getattr(s, "line", None) + for s in vars(e).get("_call_sites") or [] + ] + edges.append({"s": e.get_source(), "d": e.get_destination(), "lines": lines}) return {"nodes": nodes, "edges": edges} @@ -151,13 +151,11 @@ def side_document(analysis: dict, cg: dict) -> dict: the same name share a row; cb is -1 for a method that is no graph node. A name under its file's dotted path is written `~.rest`. Rows are numbered in order across files; `unplaced` continues the numbering for call ends that are no node. - - calls: per caller, `[callee, line, ...]`, one line per call site. No reader takes the column, and a - site's file is always its caller's. + - calls: per caller, `[callee, line, ...]`, one line per call site. - components: `[id, name, description, parent, members, relations]` in tree order. Members are rows (a key where the index lacks the method), or null for a parent holding exactly its children's. - - The analysis's edges are the calls between two different leaves (a row's leaf is the first one - listing it), less `hidden_edges`, plus `extra_edges`. On crate that leaves one extra edge of - 149,316: the analysis and the graph repeat the same edges, and only their set is read.""" + - extra_edges: the analysis's edges that are no call. The rest repeat the calls: on crate one of + 149,316 is not a call.""" index = analysis.get("methods_index") or {} files: dict[str, list[list]] = {} row_of: dict[str, list] = {} # methods_index key -> its row @@ -176,10 +174,10 @@ def side_document(analysis: dict, cg: dict) -> dict: same = by_name.get(q) or [] hit = next((r for p, r in same if p == n.get("file")), same[0][1] if same else None) if hit is None: - hit = [q, n.get("start"), n.get("end"), int(bool(n.get("cb")))] + hit = [q, n.get("start"), n.get("end"), int(n["cb"])] files.setdefault(n.get("file") or "", []).append(hit) else: - hit[5] = int(bool(n.get("cb"))) + hit[5] = int(n["cb"]) if (hit[1], hit[2]) != (n.get("start"), n.get("end")): spans.append((hit, n.get("start"), n.get("end"))) node_row[q] = hit @@ -198,8 +196,7 @@ def side_document(analysis: dict, cg: dict) -> dict: if q not in at: at[q] = len(position) + len(unplaced) unplaced.append(q) - lines = [s.get("line") if isinstance(s, dict) else None for s in e.get("sites") or []] - calls.setdefault(at[e["s"]], []).append([at[e["d"]], *lines]) + calls.setdefault(at[e["s"]], []).append([at[e["d"]], *e["lines"]]) def token(key): return position[id(row_of[key])] if key in row_of else key @@ -235,16 +232,6 @@ def walk(cs, parent): ] for i, (c, parent) in enumerate(comps) ] - - leaf: dict[int, int] = {} - for i, own in enumerate(members): - if not children[i]: - for m in own: - if isinstance(m, int): - leaf.setdefault(m, i) - implied_edges = { - (s, d) for s, ds in calls.items() for d, *_ in ds if s in leaf and d in leaf and leaf[s] != leaf[d] - } edges: set[tuple] = set() def collect_edges(o): @@ -264,8 +251,7 @@ def collect_edges(o): "components": out_comps, "relations": relations(analysis.get("components_relations")), "calls": [[s, calls[s]] for s in sorted(calls)], - "extra_edges": sorted(edges - implied_edges, key=repr), - "hidden_edges": sorted(implied_edges - edges), + "extra_edges": sorted(edges - {(s, d) for s, ds in calls.items() for d, *_ in ds}, key=repr), } diff --git a/tests/test_closed_features.py b/tests/test_closed_features.py index 4bc7cd3..3255ae7 100644 --- a/tests/test_closed_features.py +++ b/tests/test_closed_features.py @@ -234,16 +234,16 @@ class SideDocumentTests(unittest.TestCase): } GRAPH = { "nodes": { - "pkg.a.run": {"file": "pkg/a.py", "start": 1, "end": 5, "callable": True, "cb": False}, - "pkg.a.helper": {"file": "pkg/a.py", "start": 6, "end": 9, "callable": True, "cb": False}, - "pkg.b.save": {"file": "pkg/b.py", "start": 1, "end": 4, "callable": True, "cb": False}, - "pkg.x.Only": {"file": "pkg/x.py", "start": 1, "end": 2, "callable": False, "cb": True}, + "pkg.a.run": {"file": "pkg/a.py", "start": 1, "end": 5, "cb": False}, + "pkg.a.helper": {"file": "pkg/a.py", "start": 6, "end": 9, "cb": False}, + "pkg.b.save": {"file": "pkg/b.py", "start": 1, "end": 4, "cb": False}, + "pkg.x.Only": {"file": "pkg/x.py", "start": 1, "end": 2, "cb": True}, }, "edges": [ - {"s": "pkg.a.run", "d": "pkg.b.save", "sites": [{"line": 3, "file": "pkg/a.py", "column": 9}, {"line": 4}]}, - {"s": "pkg.a.run", "d": "pkg.a.helper", "sites": [{"line": 2, "file": "pkg/a.py", "column": 1}]}, - {"s": "pkg.b.save", "d": "pkg.a.run", "sites": [{"line": 2}]}, - {"s": "pkg.x.Only", "d": "ext.lib.call", "sites": []}, + {"s": "pkg.a.run", "d": "pkg.b.save", "lines": [3, 4]}, + {"s": "pkg.a.run", "d": "pkg.a.helper", "lines": [2]}, + {"s": "pkg.b.save", "d": "pkg.a.run", "lines": [2]}, + {"s": "pkg.x.Only", "d": "ext.lib.call", "lines": []}, ], } DOCUMENT = { @@ -264,7 +264,6 @@ class SideDocumentTests(unittest.TestCase): "relations": [["1", "2", "uses"]], "calls": [[0, [[2, 3, 4], [1, 2]]], [2, [[0, 2]]], [4, [[5]]]], "extra_edges": [[1, 2]], - "hidden_edges": [[2, 0]], } def test_each_fact_once_and_only_what_the_service_reads(self): From 9b79791a950ab31df19d5dc45a44b2b2a9862fe0 Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Thu, 12 Nov 2026 00:13:17 +0100 Subject: [PATCH 3/7] refactor: rename closed_features to remote_features and make it readable The script calls the service's review-guide and architecture endpoints; nothing about it is closed. scripts/remote_features.py replaces scripts/closed_features.py, and the action step, problem_history's import and the tests follow. The public input closed_features_source keeps its name, since renaming it would break workflows that set it. The rewrite uses descriptive names and short functions: side_document is now a _Table of _Row objects plus _calls, _components and _analysis_edges, and collect() reads changed files through _changed_contents. The module docstring says what a request sends, why it is packed as side documents under xz, and each size limit. The side documents, collect() output and request body are byte-identical to before on 42 cases, crate's head and base with its real call graph among them, and call_graph() reads two real pickles the same. One fix: a parent's members were left out when they equalled its leaves' members, but the service rebuilds a parent from its direct children. A parent holding fewer methods than a child read back with the child's extra methods. It now compares with its children's members. No analysis in the 42 cases hit it. Co-Authored-By: Claude Opus 5.5 --- action.yml | 6 +- scripts/closed_features.py | 567 -------------- scripts/problem_history.py | 2 +- scripts/remote_features.py | 706 ++++++++++++++++++ tests/test_problem_history.py | 4 +- ...ed_features.py => test_remote_features.py} | 100 ++- 6 files changed, 772 insertions(+), 613 deletions(-) delete mode 100644 scripts/closed_features.py create mode 100644 scripts/remote_features.py rename tests/{test_closed_features.py => test_remote_features.py} (86%) diff --git a/action.yml b/action.yml index 9009995..3fd9636 100644 --- a/action.yml +++ b/action.yml @@ -736,8 +736,8 @@ runs: # After the artifact holds the analysis and before it is uploaded: each feature is one more # file in it. Uses production by default; an empty URL disables it. It never fails the run. - - name: Run closed features - id: closed_features + - name: Run remote features + id: remote_features if: steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'review' && inputs.review_guide_url != '' continue-on-error: true shell: bash @@ -777,7 +777,7 @@ runs: REQUESTY_BASE_URL: ${{ inputs.requesty_base_url }} VERCEL_API_KEY: ${{ inputs.vercel_api_key }} VERCEL_BASE_URL: ${{ inputs.vercel_base_url }} - run: python3 "$GITHUB_ACTION_PATH/scripts/closed_features.py" + run: python3 "$GITHUB_ACTION_PATH/scripts/remote_features.py" - name: Upload review artifact id: upload_review_artifact_dotcom diff --git a/scripts/closed_features.py b/scripts/closed_features.py deleted file mode 100644 index dc73acd..0000000 --- a/scripts/closed_features.py +++ /dev/null @@ -1,567 +0,0 @@ -#!/usr/bin/env python3 -"""Call the review-guide and architecture endpoints independently after the review analysis. - -Both requests carry the same two sides, PR metadata and changed files. Only architecture carries -optional source. A side is one document holding what the service reads from that commit's analysis and -call graph (see side_document), and the bundle is xz: its window spans the whole bundle, so the base -side costs little next to a head that repeats most of it. Store the legacy guide in review_guide.json and the unchanged architecture -envelope in closed/architecture.json, including partial, skipped and failed results. - -Credentials follow the analysis: a run that chose its own provider uses its own key and model for -every feature, and a hosted or licensed run uses CodeBoarding's hosted model. A run whose provider the -service cannot call (a self-hosted endpoint) makes no model call at all: its results report -what they can without one, and it never falls back to the hosted model. Either way the service needs -to know the caller is this repository's run: a fresh GitHub Actions OIDC token (plus the license, the -same bearer the hosted proxy gets), or the run's GITHUB_TOKEN when the workflow cannot mint one. -Nothing here fails the run. - -Environment: CLOSED_FUNCTIONS_URL, REPOSITORY, PR_NUMBER, ARTIFACT_DIR, HEAD_ANALYSIS_PATH, -BASE_ANALYSIS_PATH, CHECKOUT_DIR, GITHUB_TOKEN, GITHUB_API_URL, and optionally LICENSE_KEY, -CLOSED_FEATURES_SOURCE, LLM, MODEL, AGENT_MODEL, _API_KEY, _BASE_URL, -ACTIONS_ID_TOKEN_REQUEST_URL/_TOKEN. -""" - -from __future__ import annotations - -import importlib.util -import json -import lzma -import os -import pickle -import re -import subprocess -import sys -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from urllib.error import HTTPError -from urllib.parse import quote, urlparse -from urllib.request import Request, urlopen - -# A Function URL takes 6 MB, measured after base64, and a compressed body goes through base64 on the -# way in: 4.5 MB of bundle arrives, and 5 MB is refused with a 413. -MAX_BUNDLE_BYTES = 4_300_000 -# Changed files whose contents are sent at both commits; past this a feature reads diffs only. -MAX_FILES = 300 -# A file larger than this is sent neither as a changed file's contents nor as source. -MAX_FILE_BYTES = 300_000 -# Fixed hosts matching closed_functions/llm.py; never supplied by a remote service. -PROVIDER_HOSTS = { - "ollama": "ollama.com", - "openrouter": "openrouter.ai", - "openai": "api.openai.com", - "anthropic": "api.anthropic.com", - "aws_bedrock": "bedrock-runtime.{region}.amazonaws.com", - "google": "generativelanguage.googleapis.com", - "cerebras": "api.cerebras.ai", - "deepseek": "api.deepseek.com", - "glm": "open.bigmodel.cn", - "kimi": "api.moonshot.cn", - "vercel": "ai-gateway.vercel.sh", - "orcarouter": "api.orcarouter.ai", - "requesty": "router.requesty.ai", - "opper": "api.opper.ai", -} -SOURCE_EXTENSIONS = (".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".cs", ".csproj") -SOURCE_EXCLUDE = re.compile(r"(^|/)(node_modules|dist|build|vendor|__fixtures__|docs|public)/|\.min\.js$|\.d\.ts$") -MAX_SOURCE_BYTES = 16_000_000 - - -class Skip(Exception): - """A step that did not happen, for a reason worth one line in the log.""" - - -def _github( - path: str, - accept: str = "application/vnd.github+json", - *, - method: str = "GET", - payload=None, - api: str | None = None, - token: str | None = None, - timeout: int = 30, -) -> bytes: - """A GitHub API call: `path` under `api` (default GITHUB_API_URL), or a full URL.""" - api = (api or os.environ.get("GITHUB_API_URL", "https://api.github.com")).rstrip("/") - req = Request( - path if path.startswith("http") else f"{api}{path}", - data=None if payload is None else json.dumps(payload).encode(), - method=method, - headers={ - "Authorization": f"Bearer {token or os.environ['GITHUB_TOKEN']}", - "Accept": accept, - "User-Agent": "codeboarding-action", - }, - ) - with urlopen(req, timeout=timeout) as res: # nosec B310 - GitHub API URL - return res.read() - - -def _error_message(e: HTTPError) -> str: - """The `message` of an error answer's JSON body, or "" when it has none.""" - try: - return str(json.loads(e.read()).get("message") or "") - except (ValueError, AttributeError, OSError): - return "" - - -def _github_json(path: str): - return json.loads(_github(path)) - - -def call_graph(pkl: Path) -> dict: - """A static_analysis.pkl's call graph as the service reads it: each node's file, span and whether it - is a callback, and each call's site lines (a site's file is always its caller's, and no reader takes - the column). The pickle holds CodeBoarding's own classes, which the installed engine provides.""" - with pkl.open("rb") as f: - results = pickle.load(f) # nosec B301 - written by this run's own analysis - nodes, edges = {}, [] - for lang in results.results.values(): - graph = lang.cfg.graph - for name, n in graph.nodes.items(): - nodes[name] = { - "file": n.file_path, - "start": n.line_start, - "end": n.line_end, - "cb": bool(n.is_callback_or_anonymous()), - } - for e in graph.edges: - lines = [ - s.get("line") if isinstance(s, dict) else getattr(s, "line", None) - for s in vars(e).get("_call_sites") or [] - ] - edges.append({"s": e.get_source(), "d": e.get_destination(), "lines": lines}) - return {"nodes": nodes, "edges": edges} - - -def _short(q: str, path: str) -> str: - """A qualified name as a row writes it: `~.rest` when it is the file's dotted path plus `.rest`, and - a leading `~` doubled so a name of its own that starts with one reads back unchanged.""" - stem = path.rsplit(".", 1)[0].replace("/", ".") - if q.startswith(stem + "."): - return "~" + q[len(stem) :] - return "~" + q if q.startswith("~") else q - - -def side_document(analysis: dict, cg: dict) -> dict: - """One commit's analysis and call graph as the service reads them, each fact once. The service's - bundle.expand_side turns it back into both, in every field a feature reads. - - - files: per file, rows for the methods_index entries `[name, start, end, type, content_hash, cb]` - and for call-graph nodes the index lacks `[name, start, end, cb]`. A method and the graph node of - the same name share a row; cb is -1 for a method that is no graph node. A name under its file's - dotted path is written `~.rest`. Rows are numbered in order across files; `unplaced` continues the - numbering for call ends that are no node. - - calls: per caller, `[callee, line, ...]`, one line per call site. - - components: `[id, name, description, parent, members, relations]` in tree order. Members are rows - (a key where the index lacks the method), or null for a parent holding exactly its children's. - - extra_edges: the analysis's edges that are no call. The rest repeat the calls: on crate one of - 149,316 is not a call.""" - index = analysis.get("methods_index") or {} - files: dict[str, list[list]] = {} - row_of: dict[str, list] = {} # methods_index key -> its row - by_name: dict[str, list[tuple[str, list]]] = {} - for key, m in index.items(): - path, q = m.get("file_path") or "", m.get("qualified_name") or "" - if key != f"{path}|{q}": - raise ValueError(f"methods_index key {key!r} is not its file and qualified name") - row = [q, m.get("start_line"), m.get("end_line"), m.get("type"), m.get("content_hash"), -1] - files.setdefault(path, []).append(row) - row_of[key] = row - by_name.setdefault(q, []).append((path, row)) - node_row: dict[str, list] = {} - spans: list[tuple[list, int, int]] = [] - for q, n in (cg.get("nodes") or {}).items(): - same = by_name.get(q) or [] - hit = next((r for p, r in same if p == n.get("file")), same[0][1] if same else None) - if hit is None: - hit = [q, n.get("start"), n.get("end"), int(n["cb"])] - files.setdefault(n.get("file") or "", []).append(hit) - else: - hit[5] = int(n["cb"]) - if (hit[1], hit[2]) != (n.get("start"), n.get("end")): - spans.append((hit, n.get("start"), n.get("end"))) - node_row[q] = hit - position: dict[int, int] = {} - out_files = [] - for path in sorted(files): - rows = sorted(files[path], key=lambda r: (r[1] is None, r[1] or 0, r[0])) - for r in rows: - position[id(r)] = len(position) - out_files.append([path, [[_short(r[0], path), *r[1:]] for r in rows]]) - at = {q: position[id(r)] for q, r in node_row.items()} - unplaced = [] - calls: dict[int, list] = {} - for e in cg.get("edges") or []: - for q in (e["s"], e["d"]): - if q not in at: - at[q] = len(position) + len(unplaced) - unplaced.append(q) - calls.setdefault(at[e["s"]], []).append([at[e["d"]], *e["lines"]]) - - def token(key): - return position[id(row_of[key])] if key in row_of else key - - comps: list[tuple[dict, int]] = [] - - def walk(cs, parent): - for c in cs or []: - comps.append((c, parent)) - walk(c.get("components"), len(comps) - 1) - - walk(analysis.get("components"), -1) - children: list[list[int]] = [[] for _ in comps] - for i, (_, parent) in enumerate(comps): - if parent >= 0: - children[parent].append(i) - members: list[list] = [ - [token(f"{g['file_path']}|{q}") for g in c.get("file_methods") or [] for q in g.get("methods") or []] - for c, _ in comps - ] - implied: list[set] = [set() for _ in comps] - for i in reversed(range(len(comps))): - implied[i] = set(members[i]) if not children[i] else set().union(*(implied[k] for k in children[i])) - relations = lambda rows: [[r.get("src_id"), r.get("dst_id"), r.get("relation")] for r in rows or []] # noqa: E731 - out_comps = [ - [ - c.get("component_id"), - c.get("name"), - c.get("description"), - parent, - None if children[i] and set(members[i]) == implied[i] else members[i], - relations(c.get("components_relations")), - ] - for i, (c, parent) in enumerate(comps) - ] - edges: set[tuple] = set() - - def collect_edges(o): - for r in o.get("components_relations") or []: - for e in r.get("all_edges") or []: - if e.get("source") and e.get("target"): - edges.add((token(e["source"]), token(e["target"]))) - for c in o.get("components") or []: - collect_edges(c) - - collect_edges(analysis) - return { - "version": 1, - "files": out_files, - "unplaced": unplaced, - "graph_spans": [[position[id(r)], start, end] for r, start, end in spans], - "components": out_comps, - "relations": relations(analysis.get("components_relations")), - "calls": [[s, calls[s]] for s in sorted(calls)], - "extra_edges": sorted(edges - {(s, d) for s, ds in calls.items() for d, *_ in ds}, key=repr), - } - - -def _diff(repo: str, base: str, head: str) -> list[dict]: - """The files changed between the two analysed commits, in the shape of a pull request's files - (the compare API returns the same fields), so a feature reads exactly what was analysed.""" - out = _github_json(f"/repos/{repo}/compare/{base}...{head}") - return [ - { - k: f.get(k) - for k in ("filename", "status", "additions", "deletions", "changes", "patch", "previous_filename") - if k in f - } - for f in out.get("files") or [] - ] - - -def _contents(repo: str, sha: str, path: str) -> str | None: - try: - raw = _github(f"/repos/{repo}/contents/{quote(path)}?ref={sha}", accept="application/vnd.github.raw") - except HTTPError: - return None - try: - return raw.decode("utf-8") - except UnicodeDecodeError: - return None - - -def collect(env: dict) -> dict: - """The required inputs shared by both endpoints.""" - repo, pr = env["REPOSITORY"], int(env["PR_NUMBER"]) - artifact = Path(env["ARTIFACT_DIR"]) - head_dir, base_dir = Path(env["HEAD_ANALYSIS_PATH"]).parent, Path(env["BASE_ANALYSIS_PATH"]).parent - meta = json.loads((artifact / "metadata.json").read_text()) - head_sha = meta["head_sha"] - # The analysis compared against the merge base, so that is the base every feature reads too. - base_sha = meta.get("merge_base_sha") or meta.get("base_sha") - for p in (head_dir / "static_analysis.pkl", base_dir / "static_analysis.pkl"): - if not p.is_file(): - raise Skip(f"no call graph at {p}") - - pull = _github_json(f"/repos/{repo}/pulls/{pr}") - commits = _github_json(f"/repos/{repo}/pulls/{pr}/commits?per_page=100") - pr_files = _diff(repo, base_sha, head_sha) - checkout = Path(env["CHECKOUT_DIR"]) - files: dict[str, str] = {} - for f in pr_files[:MAX_FILES]: - if f.get("status") != "removed": - try: - p = _checkout_file(checkout, f["filename"]) - if p.stat().st_size <= MAX_FILE_BYTES: - files[f"{head_sha}:{f['filename']}"] = p.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - pass - if f.get("status") != "added": - was = f.get("previous_filename") or f["filename"] - text = _contents(repo, base_sha, was) - if text is not None and len(text.encode()) <= MAX_FILE_BYTES: - files[f"{base_sha}:{was}"] = text - - bundle_meta = {**meta, "base_sha": base_sha, "head_sha": head_sha, "pr_number": str(pr)} - - def side(analysis_path: str, directory: Path) -> str: - analysis = json.loads(Path(analysis_path).read_text()) - return json.dumps(side_document(analysis, call_graph(directory / "static_analysis.pkl")), separators=(",", ":")) - - inputs = { - "head.json": side(env["HEAD_ANALYSIS_PATH"], head_dir), - "base.json": side(env["BASE_ANALYSIS_PATH"], base_dir), - "metadata.json": json.dumps(bundle_meta), - "pr.json": json.dumps( - { - "title": pull.get("title") or "", - "body": pull.get("body") or "", - "headRefOid": head_sha, - "commits": [(c.get("commit") or {}).get("message", "").split("\n")[0] for c in commits], - } - ), - "pr-files.json": json.dumps(pr_files), - "files.json": json.dumps(files), - } - return inputs - - -def _load(name: str, path: Path): - spec = importlib.util.spec_from_file_location(name, path) - assert spec and spec.loader - mod = importlib.util.module_from_spec(spec) - sys.modules.setdefault(spec.name, mod) # its dataclass looks its own module up while it is defined - spec.loader.exec_module(sys.modules[spec.name]) - return sys.modules[spec.name] - - -def _relay(): - return _load("oidc_relay", Path(__file__).with_name("oidc_relay.py")) - - -def _api_key(env: dict, llm: str) -> str: - """The key as the analysis read it, so a pasted `VAR="..."` that worked there works here too.""" - check = _load("credential_check", Path(__file__).parent / "action" / "credential_check.py") - inputs = check.load_table()["providers"].get(llm, {}).get("inputs", {}) - env_var = inputs.get(f"{llm}_api_key", f"{llm.upper()}_API_KEY") - return check._clean_key(env.get(f"{llm.upper()}_API_KEY", ""), env_var) - - -def model_headers(env: dict) -> dict[str, str]: - """What tells the service which model this run uses. Nothing for a hosted or licensed run, which - use CodeBoarding's model and settings as they stand. A run's own provider sends its key and model; one the service cannot reach sends its name alone, which - the service reads as "no model here", so a run that chose its own provider is never sent to ours. - A base URL elsewhere is a provider the service cannot reach: it calls fixed hosts only.""" - llm = env.get("LLM", "").strip().lower() - if llm in ("", "hosted", "license"): - return {} - key = _api_key(env, llm) - base = env.get(f"{llm.upper()}_BASE_URL", "").strip() - host = PROVIDER_HOSTS.get(llm) - if host and llm == "aws_bedrock": - host = host.format(region=env.get("AWS_BEDROCK_REGION", "").strip()) - if not key or not host or (base and urlparse(base).hostname != host): - return {"x-llm-provider": llm} - out = {"x-llm-api-key": key, "x-llm-provider": llm} - if env.get(f"{llm.upper()}_REGION", "").strip(): - out["x-llm-region"] = env[f"{llm.upper()}_REGION"].strip() - # The analysis's own order, so a run's closed features use its analysis model. - model = env.get("AGENT_MODEL", "").strip() or env.get("MODEL", "").strip() - if model: - out["x-llm-model"] = model - return out - - -def _bearer(env: dict) -> dict[str, str]: - url, token = env.get("ACTIONS_ID_TOKEN_REQUEST_URL", ""), env.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "") - if not url or not token: - # Without `id-token: write` the run's own token proves the same thing: it can read this - # repository. It carries no license, so the owner's free bucket pays. - return {"Authorization": f"Bearer {env['GITHUB_TOKEN']}"} - relay = _relay() - license_file = None - if env.get("LICENSE_KEY", "").strip(): - license_file = Path(env.get("RUNNER_TEMP", "/tmp")) / "codeboarding-guide-license.txt" - license_file.write_text(env["LICENSE_KEY"].strip()) - try: - config = relay.RelayConfig( - upstream_base_url="", id_token_request_url=url, id_token_request_token=token, license_file=license_file - ) - return {"Authorization": relay._authorization(config)} - finally: - if license_file is not None: - license_file.unlink(missing_ok=True) - - -def _checkout_file(checkout: Path, name: str) -> Path: - """Reject traversal and symlinks (including parent directories) before reading source.""" - relative = Path(name) - if relative.is_absolute() or ".." in relative.parts: - raise OSError("path outside checkout") - path = checkout - for part in relative.parts: - path = path / part - if path.is_symlink(): - raise OSError("symlink in checkout") - return path - - -def collect_source(env: dict) -> tuple[str | None, str | None]: - """Tracked architecture source, within fixed size and extension limits, or an omission reason.""" - if env.get("CLOSED_FEATURES_SOURCE", "true").strip().lower() == "false": - return None, "turned off by the workflow" - checkout = Path(env["CHECKOUT_DIR"]) - listed = subprocess.run(["git", "-C", str(checkout), "ls-files", "-z"], capture_output=True, check=True).stdout - total = 0 - files: dict[str, str] = {} - for name in listed.decode("utf-8", "replace").split("\0"): - if not name or not name.endswith(SOURCE_EXTENSIONS) or SOURCE_EXCLUDE.search(name): - continue - try: - path = _checkout_file(checkout, name) - size = path.stat().st_size - if size > MAX_FILE_BYTES or not path.is_file(): - continue - files[name] = path.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - continue - total += size - if total > MAX_SOURCE_BYTES: - return None, f"its source is over the {MAX_SOURCE_BYTES / 1e6:.0f} MB limit" - return json.dumps(files), None - - -def _no_architecture(ctx: dict, status: str, reason: str) -> dict: - """Preserve the architecture envelope even when the request cannot complete.""" - return { - "feature": "architecture", - "schema": 1, - "status": status, - "repo": ctx["repo"], - "pr": ctx["pr"], - "headSha": ctx["head"], - "generator": None, - "coverage": None, - "data": None, - "reason": reason, - } - - -def _post(url: str, body: bytes, headers: dict[str, str], timeout: int = 600) -> dict: - req = Request( - url, - data=body, - headers={**headers, "Content-Type": "application/x-xz", "User-Agent": "codeboarding-action"}, - method="POST", - ) - try: - with urlopen(req, timeout=timeout) as res: # nosec B310 - service URL from workflow configuration - return json.loads(res.read()) - except HTTPError as e: - reason = f"the service answered {e.code}: {_error_message(e) or e.reason}" - if e.code == 402: - raise Skip(reason) from e - raise RuntimeError(reason) from e - - -def _compress(body: dict) -> bytes: - return lzma.compress(json.dumps(body).encode(), preset=9 | lzma.PRESET_EXTREME) - - -def _pack(ctx: dict, files: dict[str, str], omitted: dict[str, str]) -> bytes: - body = {"version": 1, "repo": ctx["repo"], "pr": ctx["pr"], "files": files} - if omitted: - body["omitted"] = omitted - return _compress(body) - - -def _check_size(data: bytes, what: str = "bundle") -> None: - if len(data) > MAX_BUNDLE_BYTES: - raise Skip(f"its {what} is {len(data) / 1e6:.1f} MB, over the service's limit") - - -def run_review_guide(base: str, ctx: dict, inputs: dict, headers: dict, artifact: Path) -> None: - try: - if "x-llm-provider" in headers and "x-llm-api-key" not in headers: - raise Skip("the service cannot call this run's provider") - data = _pack(ctx, inputs, {}) - _check_size(data) - answer = _post(f"{base.rstrip('/')}/v1/review-guide", data, headers) - guide = answer.get("guide") - if not isinstance(guide, dict): - raise ValueError("the service returned no guide") - (artifact / "review_guide.json").write_text(json.dumps(guide, ensure_ascii=False, indent=1)) - except Exception as e: - print(f"::notice::review-guide {'skipped' if isinstance(e, Skip) else 'failed'}: {e}") - - -def run_architecture(base: str, ctx: dict, inputs: dict, headers: dict, env: dict) -> dict: - try: - try: - source, why = collect_source(env) - except Exception as e: - source, why = None, f"reading it failed: {type(e).__name__}" - files, omitted = dict(inputs), {} - if source is None: - omitted["source.json"] = why or "the source was not collected" - else: - files["source.json"] = source - data = _pack(ctx, files, omitted) - if "source.json" in files and len(data) > MAX_BUNDLE_BYTES: - del files["source.json"] - omitted["source.json"] = "the repository is too large to send" - data = _pack(ctx, files, omitted) - _check_size(data) - answer = _post(f"{base.rstrip('/')}/v1/architecture", data, headers) - envelope = answer.get("envelope") - if not isinstance(envelope, dict) or envelope.get("status") not in ("ok", "partial", "skipped", "failed"): - raise ValueError("the service returned no architecture result") - return envelope - except Exception as e: - return _no_architecture(ctx, "skipped" if isinstance(e, Skip) else "failed", str(e)) - - -def main(env: dict | None = None) -> int: - env = dict(os.environ if env is None else env) - try: - artifact = Path(env["ARTIFACT_DIR"]) - meta = json.loads((artifact / "metadata.json").read_text()) - ctx = {"repo": env["REPOSITORY"], "pr": int(env["PR_NUMBER"]), "head": meta.get("head_sha")} - base = env["CLOSED_FUNCTIONS_URL"] - if not base.strip(): - return 0 - try: - headers = {**_bearer(env), **model_headers(env)} - inputs = collect(env) - except Exception as e: - reason = f"collecting inputs or credentials failed: {type(e).__name__}: {e}" - print(f"::notice::review-guide failed: {reason}") - architecture = _no_architecture(ctx, "failed", reason) - else: - with ThreadPoolExecutor(max_workers=2) as pool: - guide = pool.submit(run_review_guide, base, ctx, inputs, headers, artifact) - result = pool.submit(run_architecture, base, ctx, inputs, headers, env) - architecture = result.result() - guide.result() - path = artifact / "closed" / "architecture.json" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(architecture, ensure_ascii=False, indent=1)) - if architecture["status"] not in ("ok", "partial"): - print(f"::notice::architecture {architecture['status']}: {architecture.get('reason')}") - except Exception as e: # never fail the review over a closed feature - print(f"::warning::Running the closed features failed: {type(e).__name__}: {e}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/problem_history.py b/scripts/problem_history.py index 0208630..e3d2cce 100644 --- a/scripts/problem_history.py +++ b/scripts/problem_history.py @@ -31,7 +31,7 @@ from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen sys.path.insert(0, str(Path(__file__).resolve().parent)) -from closed_features import ( +from remote_features import ( MAX_BUNDLE_BYTES, Skip, _bearer, diff --git a/scripts/remote_features.py b/scripts/remote_features.py new file mode 100644 index 0000000..0bb3c57 --- /dev/null +++ b/scripts/remote_features.py @@ -0,0 +1,706 @@ +#!/usr/bin/env python3 +"""Ask the CodeBoarding service for a pull request's review guide and architecture checks. + +Runs after the review analysis. The two requests go out in parallel and independently: either one +failing leaves the other alone, and nothing here fails the run. The guide is stored as +review_guide.json and the architecture envelope as closed/architecture.json in the review artifact, +including partial, skipped and failed results. + +What a request sends +-------------------- +One JSON object of named files, compressed with xz. Both requests send the same files: + +- head.json and base.json: one side document per commit (see side_document). It holds what the + service reads from that commit's analysis.json and call graph (static_analysis.pkl), each fact once: + the methods and graph nodes as one table per file, the calls, the component tree, and the few + analysis edges that are no call. +- metadata.json: the review artifact's metadata, with the merge base as the base. +- pr.json: the pull request's title, body, head commit and commit subjects. +- pr-files.json: the files changed between the two analysed commits, as GitHub's compare API lists them. +- files.json: those files' contents at both commits, keyed `:`. + +Architecture also sends source.json, the repository's tracked source, unless the workflow turns it off +or it does not fit. + +Why it is packed this way +------------------------- +On a large repository such as crate, the two analyses and two call graphs are about 680 MB of JSON. +Most of it repeats itself: the analysis edges are the calls again, the analysis and the graph each list +every method, and the head is nearly the base. A side document writes each fact once (crate: 7 MB per +side). xz then sees the base repeat the head, because its window is 64 MB where gzip's is 32 KB: +crate's whole request is 1.4 MB. + +Size limits +----------- +- MAX_BUNDLE_BYTES (4.3 MB compressed): the most a request may carry. Past it the request is skipped, + after first dropping source.json if architecture carried it. +- MAX_FILES: how many changed files have their contents sent; past it a feature reads diffs only. +- MAX_FILE_BYTES: a larger file is sent neither in files.json nor in source.json. +- MAX_SOURCE_BYTES: the most source.json holds; past it source is not sent. + +Credentials +----------- +Credentials follow the analysis. A run that chose its own provider uses its own key and model for every +feature, and a hosted or licensed run uses CodeBoarding's hosted model. A run whose provider the service +cannot call (a self-hosted endpoint) makes no model call at all: its results report what they can +without one, and it never falls back to the hosted model. Either way the service needs to know the +caller is this repository's run: a fresh GitHub Actions OIDC token (plus the license, the same bearer the +hosted proxy gets), or the run's GITHUB_TOKEN when the workflow cannot mint one. + +Environment: CLOSED_FUNCTIONS_URL, REPOSITORY, PR_NUMBER, ARTIFACT_DIR, HEAD_ANALYSIS_PATH, +BASE_ANALYSIS_PATH, CHECKOUT_DIR, GITHUB_TOKEN, GITHUB_API_URL, and optionally LICENSE_KEY, +CLOSED_FEATURES_SOURCE, LLM, MODEL, AGENT_MODEL, _API_KEY, _BASE_URL, +ACTIONS_ID_TOKEN_REQUEST_URL/_TOKEN. +""" + +from __future__ import annotations + +import importlib.util +import json +import lzma +import os +import pickle +import re +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from urllib.error import HTTPError +from urllib.parse import quote, urlparse +from urllib.request import Request, urlopen + +# A Function URL takes 6 MB, measured after base64, and a compressed body goes through base64 on the +# way in: 4.5 MB of bundle arrives, and 5 MB is refused with a 413. +MAX_BUNDLE_BYTES = 4_300_000 +# Changed files whose contents are sent at both commits; past this a feature reads diffs only. +MAX_FILES = 300 +# A file larger than this is sent neither as a changed file's contents nor as source. +MAX_FILE_BYTES = 300_000 +# Fixed hosts matching closed_functions/llm.py; never supplied by a remote service. +PROVIDER_HOSTS = { + "ollama": "ollama.com", + "openrouter": "openrouter.ai", + "openai": "api.openai.com", + "anthropic": "api.anthropic.com", + "aws_bedrock": "bedrock-runtime.{region}.amazonaws.com", + "google": "generativelanguage.googleapis.com", + "cerebras": "api.cerebras.ai", + "deepseek": "api.deepseek.com", + "glm": "open.bigmodel.cn", + "kimi": "api.moonshot.cn", + "vercel": "ai-gateway.vercel.sh", + "orcarouter": "api.orcarouter.ai", + "requesty": "router.requesty.ai", + "opper": "api.opper.ai", +} +SOURCE_EXTENSIONS = (".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".cs", ".csproj") +SOURCE_EXCLUDE = re.compile(r"(^|/)(node_modules|dist|build|vendor|__fixtures__|docs|public)/|\.min\.js$|\.d\.ts$") +MAX_SOURCE_BYTES = 16_000_000 + + +class Skip(Exception): + """A step that did not happen, for a reason worth one line in the log.""" + + +# --- GitHub --------------------------------------------------------------------------------------- + + +def _github( + path: str, + accept: str = "application/vnd.github+json", + *, + method: str = "GET", + payload=None, + api: str | None = None, + token: str | None = None, + timeout: int = 30, +) -> bytes: + """A GitHub API call: `path` under `api` (default GITHUB_API_URL), or a full URL.""" + api = (api or os.environ.get("GITHUB_API_URL", "https://api.github.com")).rstrip("/") + request = Request( + path if path.startswith("http") else f"{api}{path}", + data=None if payload is None else json.dumps(payload).encode(), + method=method, + headers={ + "Authorization": f"Bearer {token or os.environ['GITHUB_TOKEN']}", + "Accept": accept, + "User-Agent": "codeboarding-action", + }, + ) + with urlopen(request, timeout=timeout) as response: # nosec B310 - GitHub API URL + return response.read() + + +def _error_message(error: HTTPError) -> str: + """The `message` of an error answer's JSON body, or "" when it has none.""" + try: + return str(json.loads(error.read()).get("message") or "") + except (ValueError, AttributeError, OSError): + return "" + + +def _github_json(path: str): + return json.loads(_github(path)) + + +def _diff(repo: str, base_sha: str, head_sha: str) -> list[dict]: + """The files changed between the two analysed commits, in the shape of a pull request's files + (the compare API returns the same fields), so a feature reads exactly what was analysed.""" + compared = _github_json(f"/repos/{repo}/compare/{base_sha}...{head_sha}") + fields = ("filename", "status", "additions", "deletions", "changes", "patch", "previous_filename") + return [{field: changed[field] for field in fields if field in changed} for changed in compared.get("files") or []] + + +def _contents(repo: str, sha: str, path: str) -> str | None: + """A file's text at a commit, or None when GitHub has no such file or it is not UTF-8.""" + try: + raw = _github(f"/repos/{repo}/contents/{quote(path)}?ref={sha}", accept="application/vnd.github.raw") + except HTTPError: + return None + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + return None + + +# --- Side documents ------------------------------------------------------------------------------- + + +def call_graph(pickle_path: Path) -> dict: + """A static_analysis.pkl's call graph as the service reads it: each node's file, span and whether + it is a callback, and each call's site lines. A site's file is always its caller's and no reader + takes the column, so neither is kept. The pickle holds CodeBoarding's own classes, which the + installed engine provides.""" + with pickle_path.open("rb") as f: + results = pickle.load(f) # nosec B301 - written by this run's own analysis + nodes, edges = {}, [] + for language in results.results.values(): + graph = language.cfg.graph + for name, node in graph.nodes.items(): + nodes[name] = { + "file": node.file_path, + "start": node.line_start, + "end": node.line_end, + "cb": bool(node.is_callback_or_anonymous()), + } + for edge in graph.edges: + edges.append({"s": edge.get_source(), "d": edge.get_destination(), "lines": _site_lines(edge)}) + return {"nodes": nodes, "edges": edges} + + +def _site_lines(edge) -> list: + """The line of each place the call is made; a site may be a dict or an object.""" + sites = vars(edge).get("_call_sites") or [] + return [site.get("line") if isinstance(site, dict) else getattr(site, "line", None) for site in sites] + + +@dataclass(eq=False) +class _Row: + """One row of a side document's file table: a methods_index entry, a call-graph node the index + lacks, or a method that is also a node.""" + + name: str + start: int | None + end: int | None + method: tuple | None # (type, content_hash) for a methods_index entry + callback: int = -1 # 1 or 0 once a graph node shares the row; -1 for a method that is no node + number: int = -1 # the row's position across all files, set by _Table.number_rows + + def written(self, path: str) -> list: + name = _short_name(self.name, path) + if self.method is None: + return [name, self.start, self.end, self.callback] + return [name, self.start, self.end, *self.method, self.callback] + + +def _short_name(name: str, path: str) -> str: + """A qualified name as a row writes it: `~.rest` when it is the file's dotted path plus `.rest`, + and a leading `~` doubled so a name of its own that starts with one reads back unchanged.""" + stem = path.rsplit(".", 1)[0].replace("/", ".") + if name.startswith(stem + "."): + return "~" + name[len(stem) :] + return "~" + name if name.startswith("~") else name + + +class _Table: + """The rows of one side, per file: every methods_index entry, then every call-graph node, which + shares the row of a method with its name.""" + + def __init__(self, methods_index: dict): + self.rows_by_file: dict[str, list[_Row]] = {} + self.row_of_key: dict[str, _Row] = {} + self.methods_named: dict[str, list[tuple[str, _Row]]] = {} # qualified name -> (file, row) + self.row_of_node: dict[str, _Row] = {} + self.row_count = 0 + for key, method in methods_index.items(): + self._add_method(key, method) + + def _add_method(self, key: str, method: dict) -> None: + path, name = method.get("file_path") or "", method.get("qualified_name") or "" + if key != f"{path}|{name}": + raise ValueError(f"methods_index key {key!r} is not its file and qualified name") + row = _Row( + name, + start=method.get("start_line"), + end=method.get("end_line"), + method=(method.get("type"), method.get("content_hash")), + ) + self.rows_by_file.setdefault(path, []).append(row) + self.row_of_key[key] = row + self.methods_named.setdefault(name, []).append((path, row)) + + def add_node(self, name: str, node: dict) -> _Row: + """The row of the method with the node's name, preferring the one in the node's file, or a new + row for a node that is no method.""" + methods = self.methods_named.get(name) or [] + row = next((row for path, row in methods if path == node.get("file")), methods[0][1] if methods else None) + if row is None: + row = _Row(name, start=node.get("start"), end=node.get("end"), method=None) + self.rows_by_file.setdefault(node.get("file") or "", []).append(row) + row.callback = int(node["cb"]) + self.row_of_node[name] = row + return row + + def number_rows(self) -> list: + """Number the rows, files in path order and each file's rows by start line, and return the + files as the document writes them: `[path, [row, ...]]`.""" + written = [] + for path in sorted(self.rows_by_file): + rows = sorted(self.rows_by_file[path], key=lambda row: (row.start is None, row.start or 0, row.name)) + for row in rows: + row.number = self.row_count + self.row_count += 1 + written.append([path, [row.written(path) for row in rows]]) + return written + + def member(self, key: str) -> int | str: + """A methods_index key as a component member or edge end: its row number, or the key itself + when the index lacks it.""" + row = self.row_of_key.get(key) + return key if row is None else row.number + + +def side_document(analysis: dict, graph: dict) -> dict: + """One commit's analysis and call graph as the service reads them, each fact once. The service's + bundle.expand_side turns it back into both, in every field a feature reads. + + - files: per file, rows for the methods_index entries `[name, start, end, type, content_hash, cb]` + and for call-graph nodes the index lacks `[name, start, end, cb]`. A method and the graph node of + the same name share a row; cb is -1 for a method that is no graph node. A name under its file's + dotted path is written `~.rest`. Rows are numbered in order across files; `unplaced` continues the + numbering for call ends that are no node. + - graph_spans: `[row, start, end]` for a node whose span is not its method's. + - calls: per caller, `[callee, line, ...]`, one line per call site. + - components: `[id, name, description, parent, members, relations]` in tree order. Members are rows + (a key where the index lacks the method), or null for a parent holding exactly its children's. + - extra_edges: the analysis's edges that are no call. The rest repeat the calls: on crate one of + 149,316 is not a call.""" + table = _Table(analysis.get("methods_index") or {}) + spans = [] + for name, node in (graph.get("nodes") or {}).items(): + row = table.add_node(name, node) + if (row.start, row.end) != (node.get("start"), node.get("end")): + spans.append((row, node.get("start"), node.get("end"))) + files = table.number_rows() + calls, unplaced = _calls(graph.get("edges") or [], table) + called = {(caller, callee) for caller, callees in calls for callee, *_ in callees} + edges = {(table.member(source), table.member(target)) for source, target in _analysis_edges(analysis)} + return { + "version": 1, + "files": files, + "unplaced": unplaced, + "graph_spans": [[row.number, start, end] for row, start, end in spans], + "components": _components(analysis.get("components"), table), + "relations": _relations(analysis.get("components_relations")), + "calls": calls, + "extra_edges": sorted(edges - called, key=repr), + } + + +def _calls(edges: list[dict], table: _Table) -> tuple[list, list[str]]: + """The calls grouped by caller, `[caller, [[callee, line, ...], ...]]` in row order, and the call + ends that are no node, numbered after the last row.""" + number_of = {name: row.number for name, row in table.row_of_node.items()} + unplaced: list[str] = [] + by_caller: dict[int, list] = {} + for edge in edges: + for name in (edge["s"], edge["d"]): + if name not in number_of: + number_of[name] = table.row_count + len(unplaced) + unplaced.append(name) + by_caller.setdefault(number_of[edge["s"]], []).append([number_of[edge["d"]], *edge["lines"]]) + return [[caller, by_caller[caller]] for caller in sorted(by_caller)], unplaced + + +def _components(components: list | None, table: _Table) -> list: + """The component tree in pre-order, each `[id, name, description, parent, members, relations]`. + A parent's members are null when they are exactly its children's, which the service rebuilds.""" + flat = _flatten(components) + children: list[list[int]] = [[] for _ in flat] + for position, (_, parent) in enumerate(flat): + if parent >= 0: + children[parent].append(position) + members = [_members(component, table) for component, _ in flat] + written = [] + for position, (component, parent) in enumerate(flat): + from_children = set().union(*(members[child] for child in children[position])) + implied = children[position] and set(members[position]) == from_children + written.append( + [ + component.get("component_id"), + component.get("name"), + component.get("description"), + parent, + None if implied else members[position], + _relations(component.get("components_relations")), + ] + ) + return written + + +def _flatten(components: list | None) -> list[tuple[dict, int]]: + """The component tree in pre-order, each component with its parent's position (-1 at the top).""" + flat: list[tuple[dict, int]] = [] + + def visit(level: list | None, parent: int) -> None: + for component in level or []: + flat.append((component, parent)) + visit(component.get("components"), len(flat) - 1) + + visit(components, -1) + return flat + + +def _members(component: dict, table: _Table) -> list: + return [ + table.member(f"{group['file_path']}|{name}") + for group in component.get("file_methods") or [] + for name in group.get("methods") or [] + ] + + +def _relations(relations: list | None) -> list: + """Each relation as `[src_id, dst_id, label]`.""" + return [[relation.get("src_id"), relation.get("dst_id"), relation.get("relation")] for relation in relations or []] + + +def _analysis_edges(scope: dict): + """Every method edge of every relation in the analysis, top level and nested.""" + for relation in scope.get("components_relations") or []: + for edge in relation.get("all_edges") or []: + if edge.get("source") and edge.get("target"): + yield edge["source"], edge["target"] + for component in scope.get("components") or []: + yield from _analysis_edges(component) + + +# --- Collecting the inputs ------------------------------------------------------------------------ + + +def collect(env: dict) -> dict: + """The files both requests send, named as the service reads them.""" + repo, pr = env["REPOSITORY"], int(env["PR_NUMBER"]) + head_analysis, base_analysis = Path(env["HEAD_ANALYSIS_PATH"]), Path(env["BASE_ANALYSIS_PATH"]) + metadata = json.loads((Path(env["ARTIFACT_DIR"]) / "metadata.json").read_text()) + head_sha = metadata["head_sha"] + # The analysis compared against the merge base, so that is the base every feature reads too. + base_sha = metadata.get("merge_base_sha") or metadata.get("base_sha") + for analysis_path in (head_analysis, base_analysis): + pickle_path = analysis_path.parent / "static_analysis.pkl" + if not pickle_path.is_file(): + raise Skip(f"no call graph at {pickle_path}") + + pull = _github_json(f"/repos/{repo}/pulls/{pr}") + commits = _github_json(f"/repos/{repo}/pulls/{pr}/commits?per_page=100") + changed = _diff(repo, base_sha, head_sha) + contents = _changed_contents(repo, changed, Path(env["CHECKOUT_DIR"]), head_sha, base_sha) + return { + "head.json": _side(head_analysis), + "base.json": _side(base_analysis), + "metadata.json": json.dumps({**metadata, "base_sha": base_sha, "head_sha": head_sha, "pr_number": str(pr)}), + "pr.json": json.dumps( + { + "title": pull.get("title") or "", + "body": pull.get("body") or "", + "headRefOid": head_sha, + "commits": [(commit.get("commit") or {}).get("message", "").split("\n")[0] for commit in commits], + } + ), + "pr-files.json": json.dumps(changed), + "files.json": json.dumps(contents), + } + + +def _side(analysis_path: Path) -> str: + """The side document of the commit whose analysis this is; its call graph lies beside it.""" + analysis = json.loads(analysis_path.read_text()) + graph = call_graph(analysis_path.parent / "static_analysis.pkl") + return json.dumps(side_document(analysis, graph), separators=(",", ":")) + + +def _changed_contents(repo: str, changed: list[dict], checkout: Path, head_sha: str, base_sha: str) -> dict: + """The first MAX_FILES changed files at both commits, keyed `:`: the head from the + checkout, the base from GitHub. A file over MAX_FILE_BYTES or not UTF-8 is left out.""" + contents: dict[str, str] = {} + for change in changed[:MAX_FILES]: + if change.get("status") != "removed": + text = _checkout_text(checkout, change["filename"]) + if text is not None: + contents[f"{head_sha}:{change['filename']}"] = text + if change.get("status") != "added": + base_name = change.get("previous_filename") or change["filename"] + text = _contents(repo, base_sha, base_name) + if text is not None and len(text.encode()) <= MAX_FILE_BYTES: + contents[f"{base_sha}:{base_name}"] = text + return contents + + +def _checkout_text(checkout: Path, name: str) -> str | None: + """A checked-out file's text, or None when it is unsafe to read, over MAX_FILE_BYTES or not UTF-8.""" + try: + path = _checkout_file(checkout, name) + if path.stat().st_size <= MAX_FILE_BYTES: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + pass + return None + + +def _checkout_file(checkout: Path, name: str) -> Path: + """Reject traversal and symlinks (including parent directories) before reading source.""" + relative = Path(name) + if relative.is_absolute() or ".." in relative.parts: + raise OSError("path outside checkout") + path = checkout + for part in relative.parts: + path = path / part + if path.is_symlink(): + raise OSError("symlink in checkout") + return path + + +def collect_source(env: dict) -> tuple[str | None, str | None]: + """Tracked architecture source, within fixed size and extension limits, or an omission reason.""" + if env.get("CLOSED_FEATURES_SOURCE", "true").strip().lower() == "false": + return None, "turned off by the workflow" + checkout = Path(env["CHECKOUT_DIR"]) + listed = subprocess.run(["git", "-C", str(checkout), "ls-files", "-z"], capture_output=True, check=True).stdout + total = 0 + files: dict[str, str] = {} + for name in listed.decode("utf-8", "replace").split("\0"): + if not name or not name.endswith(SOURCE_EXTENSIONS) or SOURCE_EXCLUDE.search(name): + continue + try: + path = _checkout_file(checkout, name) + size = path.stat().st_size + if size > MAX_FILE_BYTES or not path.is_file(): + continue + files[name] = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + total += size + if total > MAX_SOURCE_BYTES: + return None, f"its source is over the {MAX_SOURCE_BYTES / 1e6:.0f} MB limit" + return json.dumps(files), None + + +# --- Credentials ---------------------------------------------------------------------------------- + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules.setdefault(spec.name, module) # its dataclass looks its own module up while it is defined + spec.loader.exec_module(sys.modules[spec.name]) + return sys.modules[spec.name] + + +def _relay(): + return _load("oidc_relay", Path(__file__).with_name("oidc_relay.py")) + + +def _api_key(env: dict, llm: str) -> str: + """The key as the analysis read it, so a pasted `VAR="..."` that worked there works here too.""" + check = _load("credential_check", Path(__file__).parent / "action" / "credential_check.py") + inputs = check.load_table()["providers"].get(llm, {}).get("inputs", {}) + env_var = inputs.get(f"{llm}_api_key", f"{llm.upper()}_API_KEY") + return check._clean_key(env.get(f"{llm.upper()}_API_KEY", ""), env_var) + + +def model_headers(env: dict) -> dict[str, str]: + """What tells the service which model this run uses. Nothing for a hosted or licensed run, which + use CodeBoarding's model and settings as they stand. A run's own provider sends its key and model; + one the service cannot reach sends its name alone, which the service reads as "no model here", so a + run that chose its own provider is never sent to ours. A base URL elsewhere is a provider the + service cannot reach: it calls fixed hosts only.""" + llm = env.get("LLM", "").strip().lower() + if llm in ("", "hosted", "license"): + return {} + key = _api_key(env, llm) + base_url = env.get(f"{llm.upper()}_BASE_URL", "").strip() + host = PROVIDER_HOSTS.get(llm) + if host and llm == "aws_bedrock": + host = host.format(region=env.get("AWS_BEDROCK_REGION", "").strip()) + if not key or not host or (base_url and urlparse(base_url).hostname != host): + return {"x-llm-provider": llm} + headers = {"x-llm-api-key": key, "x-llm-provider": llm} + if env.get(f"{llm.upper()}_REGION", "").strip(): + headers["x-llm-region"] = env[f"{llm.upper()}_REGION"].strip() + # The analysis's own order, so a run's remote features use its analysis model. + model = env.get("AGENT_MODEL", "").strip() or env.get("MODEL", "").strip() + if model: + headers["x-llm-model"] = model + return headers + + +def _bearer(env: dict) -> dict[str, str]: + url, token = env.get("ACTIONS_ID_TOKEN_REQUEST_URL", ""), env.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "") + if not url or not token: + # Without `id-token: write` the run's own token proves the same thing: it can read this + # repository. It carries no license, so the owner's free bucket pays. + return {"Authorization": f"Bearer {env['GITHUB_TOKEN']}"} + relay = _relay() + license_file = None + if env.get("LICENSE_KEY", "").strip(): + license_file = Path(env.get("RUNNER_TEMP", "/tmp")) / "codeboarding-guide-license.txt" + license_file.write_text(env["LICENSE_KEY"].strip()) + try: + config = relay.RelayConfig( + upstream_base_url="", id_token_request_url=url, id_token_request_token=token, license_file=license_file + ) + return {"Authorization": relay._authorization(config)} + finally: + if license_file is not None: + license_file.unlink(missing_ok=True) + + +# --- Requests ------------------------------------------------------------------------------------- + + +def _post(url: str, body: bytes, headers: dict[str, str], timeout: int = 600) -> dict: + request = Request( + url, + data=body, + headers={**headers, "Content-Type": "application/x-xz", "User-Agent": "codeboarding-action"}, + method="POST", + ) + try: + with urlopen(request, timeout=timeout) as response: # nosec B310 - service URL from workflow configuration + return json.loads(response.read()) + except HTTPError as error: + reason = f"the service answered {error.code}: {_error_message(error) or error.reason}" + if error.code == 402: + raise Skip(reason) from error + raise RuntimeError(reason) from error + + +def _compress(body: dict) -> bytes: + return lzma.compress(json.dumps(body).encode(), preset=9 | lzma.PRESET_EXTREME) + + +def _pack(ctx: dict, files: dict[str, str], omitted: dict[str, str]) -> bytes: + """A request body: the named files, and why any optional one is missing.""" + body = {"version": 1, "repo": ctx["repo"], "pr": ctx["pr"], "files": files} + if omitted: + body["omitted"] = omitted + return _compress(body) + + +def _check_size(data: bytes, what: str = "bundle") -> None: + if len(data) > MAX_BUNDLE_BYTES: + raise Skip(f"its {what} is {len(data) / 1e6:.1f} MB, over the service's limit") + + +def _no_architecture(ctx: dict, status: str, reason: str) -> dict: + """Preserve the architecture envelope even when the request cannot complete.""" + return { + "feature": "architecture", + "schema": 1, + "status": status, + "repo": ctx["repo"], + "pr": ctx["pr"], + "headSha": ctx["head"], + "generator": None, + "coverage": None, + "data": None, + "reason": reason, + } + + +def run_review_guide(service_url: str, ctx: dict, inputs: dict, headers: dict, artifact: Path) -> None: + try: + if "x-llm-provider" in headers and "x-llm-api-key" not in headers: + raise Skip("the service cannot call this run's provider") + data = _pack(ctx, inputs, {}) + _check_size(data) + answer = _post(f"{service_url.rstrip('/')}/v1/review-guide", data, headers) + guide = answer.get("guide") + if not isinstance(guide, dict): + raise ValueError("the service returned no guide") + (artifact / "review_guide.json").write_text(json.dumps(guide, ensure_ascii=False, indent=1)) + except Exception as error: + print(f"::notice::review-guide {'skipped' if isinstance(error, Skip) else 'failed'}: {error}") + + +def run_architecture(service_url: str, ctx: dict, inputs: dict, headers: dict, env: dict) -> dict: + try: + try: + source, why = collect_source(env) + except Exception as error: + source, why = None, f"reading it failed: {type(error).__name__}" + files, omitted = dict(inputs), {} + if source is None: + omitted["source.json"] = why or "the source was not collected" + else: + files["source.json"] = source + data = _pack(ctx, files, omitted) + if "source.json" in files and len(data) > MAX_BUNDLE_BYTES: + del files["source.json"] + omitted["source.json"] = "the repository is too large to send" + data = _pack(ctx, files, omitted) + _check_size(data) + answer = _post(f"{service_url.rstrip('/')}/v1/architecture", data, headers) + envelope = answer.get("envelope") + if not isinstance(envelope, dict) or envelope.get("status") not in ("ok", "partial", "skipped", "failed"): + raise ValueError("the service returned no architecture result") + return envelope + except Exception as error: + return _no_architecture(ctx, "skipped" if isinstance(error, Skip) else "failed", str(error)) + + +def main(env: dict | None = None) -> int: + env = dict(os.environ if env is None else env) + try: + artifact = Path(env["ARTIFACT_DIR"]) + metadata = json.loads((artifact / "metadata.json").read_text()) + ctx = {"repo": env["REPOSITORY"], "pr": int(env["PR_NUMBER"]), "head": metadata.get("head_sha")} + service_url = env["CLOSED_FUNCTIONS_URL"] + if not service_url.strip(): + return 0 + try: + headers = {**_bearer(env), **model_headers(env)} + inputs = collect(env) + except Exception as error: + reason = f"collecting inputs or credentials failed: {type(error).__name__}: {error}" + print(f"::notice::review-guide failed: {reason}") + architecture = _no_architecture(ctx, "failed", reason) + else: + with ThreadPoolExecutor(max_workers=2) as pool: + guide = pool.submit(run_review_guide, service_url, ctx, inputs, headers, artifact) + result = pool.submit(run_architecture, service_url, ctx, inputs, headers, env) + architecture = result.result() + guide.result() + path = artifact / "closed" / "architecture.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(architecture, ensure_ascii=False, indent=1)) + if architecture["status"] not in ("ok", "partial"): + print(f"::notice::architecture {architecture['status']}: {architecture.get('reason')}") + except Exception as error: # never fail the review over a remote feature + print(f"::warning::Running the remote features failed: {type(error).__name__}: {error}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_problem_history.py b/tests/test_problem_history.py index 73e5c47..6a7f197 100644 --- a/tests/test_problem_history.py +++ b/tests/test_problem_history.py @@ -1007,9 +1007,9 @@ def test_the_step_uses_only_keys_a_composite_run_step_accepts(self): self.assertLessEqual(keys, allowed) self.assertIn("continue-on-error", keys) - def test_shares_the_closed_features_transport(self): + def test_shares_the_remote_features_transport(self): for shared in (ph._bearer, ph._post, ph._github, ph._compress, ph._check_size, ph._error_message): - self.assertEqual(shared.__module__, "closed_features") + self.assertEqual(shared.__module__, "remote_features") if __name__ == "__main__": diff --git a/tests/test_closed_features.py b/tests/test_remote_features.py similarity index 86% rename from tests/test_closed_features.py rename to tests/test_remote_features.py index 3255ae7..cb47c32 100644 --- a/tests/test_closed_features.py +++ b/tests/test_remote_features.py @@ -14,11 +14,11 @@ from pathlib import Path from unittest import mock -_SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "closed_features.py" -_SPEC = importlib.util.spec_from_file_location("closed_features", _SCRIPT) -closed_features = importlib.util.module_from_spec(_SPEC) -sys.modules[_SPEC.name] = closed_features -_SPEC.loader.exec_module(closed_features) +_SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "remote_features.py" +_SPEC = importlib.util.spec_from_file_location("remote_features", _SCRIPT) +remote_features = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = remote_features +_SPEC.loader.exec_module(remote_features) INPUTS = {"head.json", "base.json", "metadata.json", "pr.json", "pr-files.json", "files.json"} GUIDE = {"version": 1, "changes": [{"id": "u1"}]} @@ -114,11 +114,11 @@ def test_analysed_commits_and_both_sides(self): with tempfile.TemporaryDirectory() as tmp: env = _workspace(Path(tmp)) with ( - mock.patch.object(closed_features, "_github_json", side_effect=_github_json) as gh, - mock.patch.object(closed_features, "_contents", return_value="base content") as base, - mock.patch.object(closed_features, "call_graph", return_value={"nodes": {}, "edges": []}), + mock.patch.object(remote_features, "_github_json", side_effect=_github_json) as gh, + mock.patch.object(remote_features, "_contents", return_value="base content") as base, + mock.patch.object(remote_features, "call_graph", return_value={"nodes": {}, "edges": []}), ): - bundle = closed_features.collect(env) + bundle = remote_features.collect(env) self.assertIn("/repos/o/r/compare/B...H", [c.args[0] for c in gh.call_args_list]) self.assertEqual(json.loads(bundle["files.json"]), {"H:app.py": "head content", "B:app.py": "base content"}) base.assert_called_once_with("o/r", "B", "app.py") @@ -131,12 +131,12 @@ def test_a_changed_file_over_the_size_limit_is_sent_at_neither_commit(self): with tempfile.TemporaryDirectory() as tmp: env = _workspace(Path(tmp)) with ( - mock.patch.object(closed_features, "_github_json", side_effect=_github_json), - mock.patch.object(closed_features, "_contents", return_value="b" * 13), - mock.patch.object(closed_features, "call_graph", return_value={"nodes": {}, "edges": []}), - mock.patch.object(closed_features, "MAX_FILE_BYTES", 12), + mock.patch.object(remote_features, "_github_json", side_effect=_github_json), + mock.patch.object(remote_features, "_contents", return_value="b" * 13), + mock.patch.object(remote_features, "call_graph", return_value={"nodes": {}, "edges": []}), + mock.patch.object(remote_features, "MAX_FILE_BYTES", 12), ): - self.assertEqual(json.loads(closed_features.collect(env)["files.json"]), {"H:app.py": "head content"}) + self.assertEqual(json.loads(remote_features.collect(env)["files.json"]), {"H:app.py": "head content"}) def test_changed_file_symlink_is_not_read(self): with tempfile.TemporaryDirectory() as tmp: @@ -145,11 +145,11 @@ def test_changed_file_symlink_is_not_read(self): path.unlink() path.symlink_to(Path(env["ARTIFACT_DIR"]) / "metadata.json") with ( - mock.patch.object(closed_features, "_github_json", side_effect=_github_json), - mock.patch.object(closed_features, "_contents", return_value=None), - mock.patch.object(closed_features, "call_graph", return_value={}), + mock.patch.object(remote_features, "_github_json", side_effect=_github_json), + mock.patch.object(remote_features, "_contents", return_value=None), + mock.patch.object(remote_features, "call_graph", return_value={}), ): - self.assertEqual(json.loads(closed_features.collect(env)["files.json"]), {}) + self.assertEqual(json.loads(remote_features.collect(env)["files.json"]), {}) def _method(q, file, start, end, h): @@ -267,24 +267,44 @@ class SideDocumentTests(unittest.TestCase): } def test_each_fact_once_and_only_what_the_service_reads(self): - doc = json.loads(json.dumps(closed_features.side_document(self.ANALYSIS, self.GRAPH))) + doc = json.loads(json.dumps(remote_features.side_document(self.ANALYSIS, self.GRAPH))) self.assertEqual(doc, self.DOCUMENT) + def test_a_parent_is_left_out_only_when_its_childrens_members_rebuild_it(self): + # The service rebuilds a parent from its children's members, so a parent holding fewer than a + # child does is written out even though its leaves hold exactly its members. + def component(cid, names, children=()): + return { + "component_id": cid, + "file_methods": [{"file_path": "a.py", "methods": list(names)}], + "components": list(children), + } + + analysis = { + "methods_index": {f"a.py|{q}": _method(q, "a.py", i, i, "h") for i, q in enumerate("xyz", 1)}, + "components": [ + component("G", "xy", [component("M", "xyz", [component("L1", "x"), component("L2", "y")])]), + component("P", "xy", [component("C1", "x"), component("C2", "y")]), + ], + } + members = [c[4] for c in remote_features.side_document(analysis, {})["components"]] + self.assertEqual(members, [[0, 1], [0, 1, 2], [0], [1], None, [0], [1]]) + def test_a_key_that_is_not_its_file_and_name_is_refused(self): odd = {"methods_index": {"pkg/a.py|other": _method("pkg.a.run", "pkg/a.py", 1, 5, "h1")}} with self.assertRaises(ValueError): - closed_features.side_document(odd, {}) + remote_features.side_document(odd, {}) class CredentialTests(unittest.TestCase): def test_without_oidc_the_runs_own_token_proves_the_repository(self): - self.assertEqual(closed_features._bearer({"GITHUB_TOKEN": "ghs_run"}), {"Authorization": "Bearer ghs_run"}) + self.assertEqual(remote_features._bearer({"GITHUB_TOKEN": "ghs_run"}), {"Authorization": "Bearer ghs_run"}) def test_oidc_bearer_includes_license_and_removes_temporary_file(self): mint = _Server(lambda p, b: (200, {"value": "jwt.token.sig"})) self.addCleanup(mint.close) with tempfile.TemporaryDirectory() as tmp: - headers = closed_features._bearer( + headers = remote_features._bearer( { "ACTIONS_ID_TOKEN_REQUEST_URL": f"{mint.url}/token?x=1", "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "req", @@ -298,10 +318,10 @@ def test_oidc_bearer_includes_license_and_removes_temporary_file(self): def test_hosted_and_license_send_no_model(self): for llm in ("", "hosted", "license"): - self.assertEqual(closed_features.model_headers({"LLM": llm, "OPENAI_API_KEY": "leftover"}), {}) + self.assertEqual(remote_features.model_headers({"LLM": llm, "OPENAI_API_KEY": "leftover"}), {}) def test_each_fixed_provider_and_analysis_model(self): - for provider, host in closed_features.PROVIDER_HOSTS.items(): + for provider, host in remote_features.PROVIDER_HOSTS.items(): with self.subTest(provider=provider): env = { "LLM": provider, @@ -313,7 +333,7 @@ def test_each_fixed_provider_and_analysis_model(self): env["AWS_BEDROCK_REGION"] = "eu-west-1" host = host.format(region="eu-west-1") env[f"{provider.upper()}_BASE_URL"] = f"https://{host}/v1" - headers = closed_features.model_headers(env) + headers = remote_features.model_headers(env) self.assertEqual(headers["x-llm-api-key"], "k") self.assertEqual(headers["x-llm-model"], "analysis") if provider == "aws_bedrock": @@ -327,7 +347,7 @@ def test_key_is_unwrapped_the_way_the_analysis_reads_it(self): ({**bedrock, "AWS_BEDROCK_API_KEY": "AWS_BEARER_TOKEN_BEDROCK=abc="}, "abc="), ): with self.subTest(llm=env["LLM"]): - self.assertEqual(closed_features.model_headers(env)["x-llm-api-key"], want) + self.assertEqual(remote_features.model_headers(env)["x-llm-api-key"], want) def test_unsupported_self_hosted_and_missing_key_never_fall_back(self): for env in ( @@ -336,7 +356,7 @@ def test_unsupported_self_hosted_and_missing_key_never_fall_back(self): {"LLM": "openai", "OPENAI_API_KEY": "k", "OPENAI_BASE_URL": "https://gateway.corp/v1"}, {"LLM": "anthropic"}, ): - self.assertEqual(closed_features.model_headers(env), {"x-llm-provider": env["LLM"]}) + self.assertEqual(remote_features.model_headers(env), {"x-llm-provider": env["LLM"]}) class EndpointTests(unittest.TestCase): @@ -349,7 +369,7 @@ def setUp(self): ("_contents", {"return_value": "base"}), ("call_graph", {"return_value": {"nodes": {}, "edges": []}}), ): - patch = mock.patch.object(closed_features, name, **kwargs) + patch = mock.patch.object(remote_features, name, **kwargs) patch.start() self.addCleanup(patch.stop) @@ -370,7 +390,7 @@ def run_service(self, respond=None): self.addCleanup(service.close) self.env["CLOSED_FUNCTIONS_URL"] = service.url + "/" with mock.patch("builtins.print"): - self.assertEqual(closed_features.main(self.env), 0) + self.assertEqual(remote_features.main(self.env), 0) return service def test_exactly_two_posts_shared_inputs_and_fixed_outputs(self): @@ -445,8 +465,8 @@ def respond(path, body): def test_empty_url_disables_both(self): self.env["CLOSED_FUNCTIONS_URL"] = "" - with mock.patch.object(closed_features, "_post") as post: - self.assertEqual(closed_features.main(self.env), 0) + with mock.patch.object(remote_features, "_post") as post: + self.assertEqual(remote_features.main(self.env), 0) post.assert_not_called() self.assertFalse((Path(self.env["ARTIFACT_DIR"]) / "closed").exists()) @@ -454,7 +474,7 @@ def test_source_toggle_and_read_failure_only_omit_architecture_source(self): for failure in (False, True): self.env["CLOSED_FEATURES_SOURCE"] = "false" with ( - mock.patch.object(closed_features, "collect_source", side_effect=OSError()) + mock.patch.object(remote_features, "collect_source", side_effect=OSError()) if failure else nullcontext() ): @@ -466,15 +486,15 @@ def test_source_toggle_and_read_failure_only_omit_architecture_source(self): def test_bundle_limit_omits_source_then_skips_if_required_inputs_too_large(self): with ( - mock.patch.object(closed_features, "MAX_BUNDLE_BYTES", 1000), - mock.patch.object(closed_features, "collect_source", return_value=(os.urandom(2000).hex(), None)), + mock.patch.object(remote_features, "MAX_BUNDLE_BYTES", 1000), + mock.patch.object(remote_features, "collect_source", return_value=(os.urandom(2000).hex(), None)), ): service = self.run_service() arch = next(r for r in service.requests if r["path"] == "/v1/architecture") body = json.loads(lzma.decompress(arch["body"])) self.assertNotIn("source.json", body["files"]) self.assertIn("too large", body["omitted"]["source.json"]) - with mock.patch.object(closed_features, "MAX_BUNDLE_BYTES", 1): + with mock.patch.object(remote_features, "MAX_BUNDLE_BYTES", 1): self.assertEqual(self.run_service().requests, []) self.assertEqual(_read(self.env, "closed/architecture.json")["status"], "skipped") @@ -511,11 +531,11 @@ def test_tracked_extensions_exclusions_limits_and_symlinks(self): (root / "link.py").symlink_to(root / "a.py") subprocess.run(["git", "-C", tmp, "add", "."], check=True) (root / "untracked.py").write_text("not tracked") - text, why = closed_features.collect_source({"CHECKOUT_DIR": tmp}) + text, why = remote_features.collect_source({"CHECKOUT_DIR": tmp}) self.assertEqual(set(json.loads(text)), set(names[:9])) self.assertIsNone(why) - with mock.patch.object(closed_features, "MAX_SOURCE_BYTES", 1): - text, why = closed_features.collect_source({"CHECKOUT_DIR": tmp}) + with mock.patch.object(remote_features, "MAX_SOURCE_BYTES", 1): + text, why = remote_features.collect_source({"CHECKOUT_DIR": tmp}) self.assertIsNone(text) self.assertIn("limit", why) @@ -525,11 +545,11 @@ def test_symlink_parent_and_traversal_rejected(self): (root / "link").symlink_to(root, target_is_directory=True) for name in ("link/a.py", "../a.py", "/tmp/a.py"): with self.assertRaises(OSError): - closed_features._checkout_file(root, name) + remote_features._checkout_file(root, name) def test_disabled_source_does_not_read_checkout(self): self.assertEqual( - closed_features.collect_source({"CLOSED_FEATURES_SOURCE": "false"}), (None, "turned off by the workflow") + remote_features.collect_source({"CLOSED_FEATURES_SOURCE": "false"}), (None, "turned off by the workflow") ) From db83d05e1e2f332a02e8733ebf0fe103d6c08cd9 Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Thu, 12 Nov 2026 21:12:30 +0100 Subject: [PATCH 4/7] refactor: read the side documents' analyses through the installed engine remote_features.py parsed the legacy analysis.json itself: methods_index, each component's file_methods and every relation's all_edges. It now reads an analysis only through the pinned release's parse_unified_analysis, the same engine that unpickles the call graph beside it. The review analyses both commits with that release, so both are always in the format it reads, and a format change in Core needs no change here. The check that a methods_index key is its file and qualified name goes: the keys are built from Core's method entries. InstalledCoreTests runs in the core-compatibility job against the pinned release. tests/fixtures/analysis holds one analysis as Core 0.15.5 and Core main (9cf50fb9, after #571) wrote it. Each release reads only its own, and every fixture it reads must give the same side document. Checked: on CodeBoarding's own analysis (two baseline syncs, 153 files, 22 components) and this repo's, the side document is byte-identical to the one built from the raw document, and Core main reading the v1 rewrite gives the same bytes as 0.15.5 reading the legacy original. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test.yml | 2 + scripts/remote_features.py | 145 ++++++----- tests/fixtures/analysis/core-0.15.5.json | 289 +++++++++++++++++++++ tests/fixtures/analysis/core-9cf50fb9.json | 217 ++++++++++++++++ tests/test_remote_features.py | 243 ++++++++++------- 5 files changed, 736 insertions(+), 160 deletions(-) create mode 100644 tests/fixtures/analysis/core-0.15.5.json create mode 100644 tests/fixtures/analysis/core-9cf50fb9.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8d6efd..2cb3eec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,6 +42,8 @@ jobs: python -m pip install --disable-pip-version-check "$requirement" - name: Check the provider table against the installed release run: python -m unittest tests.test_provider_table_drift -v + - name: Build side documents with the installed release + run: python -m unittest tests.test_remote_features.InstalledCoreTests -v - name: Install sync artifacts with the installed CodeBoarding release shell: bash env: diff --git a/scripts/remote_features.py b/scripts/remote_features.py index 0bb3c57..35c180d 100644 --- a/scripts/remote_features.py +++ b/scripts/remote_features.py @@ -13,7 +13,8 @@ - head.json and base.json: one side document per commit (see side_document). It holds what the service reads from that commit's analysis.json and call graph (static_analysis.pkl), each fact once: the methods and graph nodes as one table per file, the calls, the component tree, and the few - analysis edges that are no call. + analysis edges that are no call. Analyses are read through the installed engine's parser, never as a + document, so whatever format the pinned release writes is read. - metadata.json: the review artifact's metadata, with the merge base as the base. - pr.json: the pull request's title, body, head commit and commit subjects. - pr-files.json: the files changed between the two analysed commits, as GitHub's compare API lists them. @@ -197,13 +198,13 @@ def _site_lines(edge) -> list: @dataclass(eq=False) class _Row: - """One row of a side document's file table: a methods_index entry, a call-graph node the index - lacks, or a method that is also a node.""" + """One row of a side document's file table: a method of the analysis, a call-graph node that is no + method, or a method that is also a node.""" name: str start: int | None end: int | None - method: tuple | None # (type, content_hash) for a methods_index entry + method: tuple | None # (type, content_hash) for a method of the analysis callback: int = -1 # 1 or 0 once a graph node shares the row; -1 for a method that is no node number: int = -1 # the row's position across all files, set by _Table.number_rows @@ -224,30 +225,24 @@ def _short_name(name: str, path: str) -> str: class _Table: - """The rows of one side, per file: every methods_index entry, then every call-graph node, which + """The rows of one side, per file: every method of the analysis, then every call-graph node, which shares the row of a method with its name.""" - def __init__(self, methods_index: dict): + def __init__(self, files: dict): self.rows_by_file: dict[str, list[_Row]] = {} self.row_of_key: dict[str, _Row] = {} self.methods_named: dict[str, list[tuple[str, _Row]]] = {} # qualified name -> (file, row) self.row_of_node: dict[str, _Row] = {} self.row_count = 0 - for key, method in methods_index.items(): - self._add_method(key, method) - - def _add_method(self, key: str, method: dict) -> None: - path, name = method.get("file_path") or "", method.get("qualified_name") or "" - if key != f"{path}|{name}": - raise ValueError(f"methods_index key {key!r} is not its file and qualified name") - row = _Row( - name, - start=method.get("start_line"), - end=method.get("end_line"), - method=(method.get("type"), method.get("content_hash")), - ) + for path, entry in files.items(): + for method in entry.methods: + self._add_method(path, method) + + def _add_method(self, path: str, method) -> None: + name = method.qualified_name + row = _Row(name, start=method.start_line, end=method.end_line, method=(method.node_type, method.content_hash)) self.rows_by_file.setdefault(path, []).append(row) - self.row_of_key[key] = row + self.row_of_key[f"{path}|{name}"] = row self.methods_named.setdefault(name, []).append((path, row)) def add_node(self, name: str, node: dict) -> _Row: @@ -275,28 +270,29 @@ def number_rows(self) -> list: return written def member(self, key: str) -> int | str: - """A methods_index key as a component member or edge end: its row number, or the key itself - when the index lacks it.""" + """A method's `path|qualified_name` key as a component member or edge end: its row number, or + the key itself when the analysis lacks the method.""" row = self.row_of_key.get(key) return key if row is None else row.number -def side_document(analysis: dict, graph: dict) -> dict: - """One commit's analysis and call graph as the service reads them, each fact once. The service's - bundle.expand_side turns it back into both, in every field a feature reads. +def side_document(root, sub_analyses: dict, graph: dict) -> dict: + """One commit's analysis, as Core parses it (the root and each component's sub-analysis by id), and + its call graph as the service reads them, each fact once. The service's bundle.expand_side turns it + back into both, in every field a feature reads. - - files: per file, rows for the methods_index entries `[name, start, end, type, content_hash, cb]` - and for call-graph nodes the index lacks `[name, start, end, cb]`. A method and the graph node of + - files: per file, rows for the analysis's methods `[name, start, end, type, content_hash, cb]` + and for call-graph nodes that are no method `[name, start, end, cb]`. A method and the graph node of the same name share a row; cb is -1 for a method that is no graph node. A name under its file's dotted path is written `~.rest`. Rows are numbered in order across files; `unplaced` continues the numbering for call ends that are no node. - graph_spans: `[row, start, end]` for a node whose span is not its method's. - calls: per caller, `[callee, line, ...]`, one line per call site. - components: `[id, name, description, parent, members, relations]` in tree order. Members are rows - (a key where the index lacks the method), or null for a parent holding exactly its children's. + (a key where the analysis lacks the method), or null for a parent holding exactly its children's. - extra_edges: the analysis's edges that are no call. The rest repeat the calls: on crate one of 149,316 is not a call.""" - table = _Table(analysis.get("methods_index") or {}) + table = _Table(root.files) spans = [] for name, node in (graph.get("nodes") or {}).items(): row = table.add_node(name, node) @@ -305,14 +301,14 @@ def side_document(analysis: dict, graph: dict) -> dict: files = table.number_rows() calls, unplaced = _calls(graph.get("edges") or [], table) called = {(caller, callee) for caller, callees in calls for callee, *_ in callees} - edges = {(table.member(source), table.member(target)) for source, target in _analysis_edges(analysis)} + edges = {(table.member(source), table.member(target)) for source, target in _analysis_edges(root, sub_analyses)} return { "version": 1, "files": files, "unplaced": unplaced, "graph_spans": [[row.number, start, end] for row, start, end in spans], - "components": _components(analysis.get("components"), table), - "relations": _relations(analysis.get("components_relations")), + "components": _components(root, sub_analyses, table), + "relations": _relations(root.components_relations), "calls": calls, "extra_edges": sorted(edges - called, key=repr), } @@ -333,66 +329,71 @@ def _calls(edges: list[dict], table: _Table) -> tuple[list, list[str]]: return [[caller, by_caller[caller]] for caller in sorted(by_caller)], unplaced -def _components(components: list | None, table: _Table) -> list: +def _components(root, sub_analyses: dict, table: _Table) -> list: """The component tree in pre-order, each `[id, name, description, parent, members, relations]`. A parent's members are null when they are exactly its children's, which the service rebuilds.""" - flat = _flatten(components) + flat = _flatten(root, sub_analyses) children: list[list[int]] = [[] for _ in flat] - for position, (_, parent) in enumerate(flat): + for position, (_, parent, _) in enumerate(flat): if parent >= 0: children[parent].append(position) - members = [_members(component, table) for component, _ in flat] + members = [ + [ + table.member(f"{group.file_path}|{method.qualified_name}") + for group in component.file_methods + for method in group.methods + ] + for component, _, _ in flat + ] written = [] - for position, (component, parent) in enumerate(flat): + for position, (component, parent, relations) in enumerate(flat): from_children = set().union(*(members[child] for child in children[position])) implied = children[position] and set(members[position]) == from_children written.append( [ - component.get("component_id"), - component.get("name"), - component.get("description"), + component.component_id, + component.name, + component.description, parent, None if implied else members[position], - _relations(component.get("components_relations")), + _relations(relations), ] ) return written -def _flatten(components: list | None) -> list[tuple[dict, int]]: - """The component tree in pre-order, each component with its parent's position (-1 at the top).""" - flat: list[tuple[dict, int]] = [] +def _flatten(root, sub_analyses: dict) -> list[tuple[object, int, list]]: + """The component tree in pre-order, each component with its parent's position (-1 at the top) and + the relations among its children.""" + flat: list[tuple[object, int, list]] = [] - def visit(level: list | None, parent: int) -> None: - for component in level or []: - flat.append((component, parent)) - visit(component.get("components"), len(flat) - 1) + def visit(analysis, parent: int) -> None: + for component in analysis.components: + sub_analysis = sub_analyses.get(component.component_id) + flat.append((component, parent, sub_analysis.components_relations if sub_analysis else [])) + if sub_analysis is not None: + visit(sub_analysis, len(flat) - 1) - visit(components, -1) + visit(root, -1) return flat -def _members(component: dict, table: _Table) -> list: - return [ - table.member(f"{group['file_path']}|{name}") - for group in component.get("file_methods") or [] - for name in group.get("methods") or [] - ] +def _relations(relations: list) -> list: + """Each relation as `[src_id, dst_id, label]`.""" + return [[relation.src_id, relation.dst_id, relation.relation] for relation in relations] -def _relations(relations: list | None) -> list: - """Each relation as `[src_id, dst_id, label]`.""" - return [[relation.get("src_id"), relation.get("dst_id"), relation.get("relation")] for relation in relations or []] +def _analysis_edges(root, sub_analyses: dict): + """Every method edge of every relation in the analysis, top level and nested, as `path|qualified_name` + keys.""" + for analysis in (root, *sub_analyses.values()): + for relation in analysis.components_relations: + for edge in relation.all_edges: + yield _method_key(edge.source), _method_key(edge.target) -def _analysis_edges(scope: dict): - """Every method edge of every relation in the analysis, top level and nested.""" - for relation in scope.get("components_relations") or []: - for edge in relation.get("all_edges") or []: - if edge.get("source") and edge.get("target"): - yield edge["source"], edge["target"] - for component in scope.get("components") or []: - yield from _analysis_edges(component) +def _method_key(reference) -> str: + return f"{reference.reference_file or ''}|{reference.qualified_name}" # --- Collecting the inputs ------------------------------------------------------------------------ @@ -434,9 +435,17 @@ def collect(env: dict) -> dict: def _side(analysis_path: Path) -> str: """The side document of the commit whose analysis this is; its call graph lies beside it.""" - analysis = json.loads(analysis_path.read_text()) + root, sub_analyses = read_analysis(analysis_path) graph = call_graph(analysis_path.parent / "static_analysis.pkl") - return json.dumps(side_document(analysis, graph), separators=(",", ":")) + return json.dumps(side_document(root, sub_analyses, graph), separators=(",", ":")) + + +def read_analysis(analysis_path: Path) -> tuple: + """The analysis as the installed engine parses it: the root and each component's sub-analysis by id. + The run analysed both commits with that engine, so both are in the format it reads.""" + from diagram_analysis.analysis_json import parse_unified_analysis + + return parse_unified_analysis(json.loads(analysis_path.read_text())) def _changed_contents(repo: str, changed: list[dict], checkout: Path, head_sha: str, base_sha: str) -> dict: diff --git a/tests/fixtures/analysis/core-0.15.5.json b/tests/fixtures/analysis/core-0.15.5.json new file mode 100644 index 0000000..d122b4f --- /dev/null +++ b/tests/fixtures/analysis/core-0.15.5.json @@ -0,0 +1,289 @@ +{ + "metadata": { + "generated_at": "2026-11-12T00:00:00Z", + "source_tree_hash": "", + "repo_name": "pkg", + "depth_level": 2, + "depth_cap": 2, + "file_coverage_summary": { + "total_files": 3, + "analyzed": 3, + "not_analyzed": 0, + "not_analyzed_by_reason": {} + }, + "tree_spec": {} + }, + "description": "fixture", + "files": { + "pkg/a.py": { + "method_keys": [ + "pkg/a.py|pkg.a.run", + "pkg/a.py|pkg.a.helper" + ], + "content_hash": "", + "module_hash": "" + }, + "pkg/b.py": { + "method_keys": [ + "pkg/b.py|pkg.b.save", + "pkg/b.py|~Weird" + ], + "content_hash": "", + "module_hash": "" + }, + "pkg/c.py": { + "method_keys": [ + "pkg/c.py|pkg.c.load" + ], + "content_hash": "", + "module_hash": "" + } + }, + "methods_index": { + "pkg/a.py|pkg.a.run": { + "file_path": "pkg/a.py", + "qualified_name": "pkg.a.run", + "start_line": 1, + "end_line": 5, + "type": "FUNCTION", + "content_hash": "h1" + }, + "pkg/a.py|pkg.a.helper": { + "file_path": "pkg/a.py", + "qualified_name": "pkg.a.helper", + "start_line": 6, + "end_line": 8, + "type": "FUNCTION", + "content_hash": "h2" + }, + "pkg/b.py|pkg.b.save": { + "file_path": "pkg/b.py", + "qualified_name": "pkg.b.save", + "start_line": 1, + "end_line": 4, + "type": "FUNCTION", + "content_hash": "h3" + }, + "pkg/b.py|~Weird": { + "file_path": "pkg/b.py", + "qualified_name": "~Weird", + "start_line": 10, + "end_line": 12, + "type": "FUNCTION", + "content_hash": "h4" + }, + "pkg/c.py|pkg.c.load": { + "file_path": "pkg/c.py", + "qualified_name": "pkg.c.load", + "start_line": 1, + "end_line": 3, + "type": "FUNCTION", + "content_hash": "h5" + } + }, + "components": [ + { + "name": "Core", + "description": "d1", + "key_entities": [], + "source_cluster_ids": [], + "file_methods": [ + { + "file_path": "pkg/a.py", + "methods": [ + "pkg.a.run", + "pkg.a.helper" + ] + }, + { + "file_path": "pkg/b.py", + "methods": [ + "pkg.b.save", + "~Weird" + ] + } + ], + "component_id": "1", + "can_expand": true, + "components": [ + { + "name": "Run", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "file_methods": [ + { + "file_path": "pkg/a.py", + "methods": [ + "pkg.a.run", + "pkg.a.helper" + ] + } + ], + "component_id": "1.1", + "can_expand": false, + "components": [], + "components_relations": [] + }, + { + "name": "Store", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "file_methods": [ + { + "file_path": "pkg/b.py", + "methods": [ + "pkg.b.save", + "~Weird" + ] + } + ], + "component_id": "1.2", + "can_expand": false, + "components": [], + "components_relations": [] + } + ], + "components_relations": [ + { + "relation": "saves", + "src_name": "Run", + "dst_name": "Store", + "evidence": "", + "key_edges": [], + "src_id": "1.1", + "dst_id": "1.2", + "is_static": false, + "all_edges": [ + { + "source": "pkg/a.py|pkg.a.run", + "target": "pkg/b.py|pkg.b.save", + "call_sites": [ + { + "line": 3, + "column": 8 + } + ], + "description": "" + }, + { + "source": "pkg/a.py|pkg.a.helper", + "target": "pkg/b.py|pkg.b.save", + "call_sites": [], + "description": "" + } + ] + } + ] + }, + { + "name": "Also", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "file_methods": [ + { + "file_path": "pkg/c.py", + "methods": [ + "pkg.c.load" + ] + } + ], + "component_id": "2", + "can_expand": false, + "components": [], + "components_relations": [] + } + ], + "components_relations": [ + { + "relation": "saves", + "src_name": "Run", + "dst_name": "Store", + "evidence": "", + "key_edges": [], + "src_id": "1.1", + "dst_id": "1.2", + "is_static": false, + "all_edges": [ + { + "source": "pkg/a.py|pkg.a.run", + "target": "pkg/b.py|pkg.b.save", + "call_sites": [ + { + "line": 3, + "column": 8 + } + ], + "description": "" + }, + { + "source": "pkg/a.py|pkg.a.helper", + "target": "pkg/b.py|pkg.b.save", + "call_sites": [], + "description": "" + } + ] + }, + { + "relation": "uses", + "src_name": "Run", + "dst_name": "Also", + "evidence": "", + "key_edges": [], + "src_id": "1.1", + "dst_id": "2", + "is_static": false, + "all_edges": [ + { + "source": "pkg/a.py|pkg.a.run", + "target": "pkg/c.py|pkg.c.load", + "call_sites": [ + { + "line": 4, + "column": 4 + } + ], + "description": "" + } + ] + }, + { + "relation": "loads", + "src_name": "Store", + "dst_name": "Also", + "evidence": "", + "key_edges": [], + "src_id": "1.2", + "dst_id": "2", + "is_static": false, + "all_edges": [ + { + "source": "pkg/b.py|~Weird", + "target": "pkg/c.py|pkg.c.load", + "call_sites": [], + "description": "" + } + ] + }, + { + "relation": "reads", + "src_name": "Also", + "dst_name": "Store", + "evidence": "", + "key_edges": [], + "src_id": "2", + "dst_id": "1.2", + "is_static": false, + "all_edges": [ + { + "source": "pkg/c.py|pkg.c.load", + "target": "pkg/b.py|pkg.b.save", + "call_sites": [], + "description": "" + } + ] + } + ] +} diff --git a/tests/fixtures/analysis/core-9cf50fb9.json b/tests/fixtures/analysis/core-9cf50fb9.json new file mode 100644 index 0000000..4ddba2e --- /dev/null +++ b/tests/fixtures/analysis/core-9cf50fb9.json @@ -0,0 +1,217 @@ +{ + "metadata": { + "generated_at": "2026-11-12T00:00:00Z", + "source_tree_hash": "", + "repo_name": "pkg", + "depth_level": 2, + "depth_cap": 2, + "file_coverage_summary": { + "total_files": 3, + "analyzed": 3, + "not_analyzed": 0, + "not_analyzed_by_reason": {} + }, + "tree_spec": {}, + "format_version": "v1" + }, + "description": "fixture", + "files": { + "pkg/a.py": { + "content_hash": "", + "module_hash": "", + "methods": { + "pkg.a.run": [ + 1, + 5, + "FUNCTION", + "h1", + "W2ZvP4dicxdV" + ], + "pkg.a.helper": [ + 6, + 8, + "FUNCTION", + "h2", + "fHTj2iSkCxvz" + ] + } + }, + "pkg/b.py": { + "content_hash": "", + "module_hash": "", + "methods": { + "pkg.b.save": [ + 1, + 4, + "FUNCTION", + "h3", + "HXVcn6KDWNds" + ], + "~Weird": [ + 10, + 12, + "FUNCTION", + "h4", + "VHMX26c_wkXu" + ] + } + }, + "pkg/c.py": { + "content_hash": "", + "module_hash": "", + "methods": { + "pkg.c.load": [ + 1, + 3, + "FUNCTION", + "h5", + "ai1syDPa_D53" + ] + } + } + }, + "components": [ + { + "name": "Core", + "component_id": "1", + "description": "d1", + "key_entities": [], + "source_cluster_ids": [], + "can_expand": true, + "components": [ + { + "name": "Run", + "component_id": "1.1", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "can_expand": false, + "file_methods": [ + "W2ZvP4dicxdV", + "fHTj2iSkCxvz" + ], + "components": [], + "components_relations": [] + }, + { + "name": "Store", + "component_id": "1.2", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "can_expand": false, + "file_methods": [ + "HXVcn6KDWNds", + "VHMX26c_wkXu" + ], + "components": [], + "components_relations": [] + } + ], + "components_relations": [ + { + "relation": "saves", + "src_name": "Run", + "dst_name": "Store", + "evidence": "", + "src_id": "1.1", + "dst_id": "1.2", + "is_static": false, + "key_edges": [] + } + ] + }, + { + "name": "Also", + "component_id": "2", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "can_expand": false, + "file_methods": [ + "ai1syDPa_D53" + ], + "components": [], + "components_relations": [] + } + ], + "components_relations": [ + { + "relation": "saves", + "src_name": "Run", + "dst_name": "Store", + "evidence": "", + "src_id": "1.1", + "dst_id": "1.2", + "is_static": false, + "key_edges": [] + }, + { + "relation": "uses", + "src_name": "Run", + "dst_name": "Also", + "evidence": "", + "src_id": "1.1", + "dst_id": "2", + "is_static": false, + "key_edges": [] + }, + { + "relation": "loads", + "src_name": "Store", + "dst_name": "Also", + "evidence": "", + "src_id": "1.2", + "dst_id": "2", + "is_static": false, + "key_edges": [] + }, + { + "relation": "reads", + "src_name": "Also", + "dst_name": "Store", + "evidence": "", + "src_id": "2", + "dst_id": "1.2", + "is_static": false, + "key_edges": [] + } + ], + "edges": [ + [ + "fHTj2iSkCxvz", + "HXVcn6KDWNds", + [] + ], + [ + "W2ZvP4dicxdV", + "HXVcn6KDWNds", + [ + [ + 3, + 8 + ] + ] + ], + [ + "W2ZvP4dicxdV", + "ai1syDPa_D53", + [ + [ + 4, + 4 + ] + ] + ], + [ + "VHMX26c_wkXu", + "ai1syDPa_D53", + [] + ], + [ + "ai1syDPa_D53", + "HXVcn6KDWNds", + [] + ] + ] +} diff --git a/tests/test_remote_features.py b/tests/test_remote_features.py index cb47c32..573775a 100644 --- a/tests/test_remote_features.py +++ b/tests/test_remote_features.py @@ -9,9 +9,11 @@ import tempfile import threading import unittest +from importlib import metadata from contextlib import nullcontext from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from types import SimpleNamespace from unittest import mock _SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "remote_features.py" @@ -110,6 +112,11 @@ def _read(env, name): class BundleTests(unittest.TestCase): + def setUp(self): + patch = mock.patch.object(remote_features, "read_analysis", return_value=(_analysis([]), {})) + patch.start() + self.addCleanup(patch.stop) + def test_analysed_commits_and_both_sides(self): with tempfile.TemporaryDirectory() as tmp: env = _workspace(Path(tmp)) @@ -152,85 +159,80 @@ def test_changed_file_symlink_is_not_read(self): self.assertEqual(json.loads(remote_features.collect(env)["files.json"]), {}) -def _method(q, file, start, end, h): - return { - "file_path": file, - "qualified_name": q, - "start_line": start, - "end_line": end, - "type": "FUNCTION", - "content_hash": h, - } +def _method(name, start, end, content_hash): + return SimpleNamespace( + qualified_name=name, start_line=start, end_line=end, node_type="FUNCTION", content_hash=content_hash + ) + + +def _component(cid, name=None, members=None, description=None): + """A component as Core parses it; members are qualified names by file.""" + groups = [ + SimpleNamespace(file_path=path, methods=[SimpleNamespace(qualified_name=q) for q in names]) + for path, names in (members or {}).items() + ] + return SimpleNamespace(component_id=cid, name=name, description=description, file_methods=groups) + + +def _relation(src, dst, label, *edges): + """A relation as Core parses it; edges are `path|qualified_name` pairs.""" + + def reference(key): + path, name = key.split("|") + return SimpleNamespace(reference_file=path, qualified_name=name) + + return SimpleNamespace( + src_id=src, + dst_id=dst, + relation=label, + all_edges=[SimpleNamespace(source=reference(s), target=reference(t)) for s, t in edges], + ) + + +def _analysis(components, relations=(), files=None): + """A scope as Core parses it: the root carries the file index, a sub-analysis does not need one.""" + return SimpleNamespace( + components=list(components), + components_relations=list(relations), + files={path: SimpleNamespace(methods=methods) for path, methods in (files or {}).items()}, + ) class SideDocumentTests(unittest.TestCase): """The service's bundle.expand_side reads this exact document back (its tests hold the same one).""" - ANALYSIS = { - "methods_index": { - "pkg/a.py|pkg.a.run": _method("pkg.a.run", "pkg/a.py", 1, 5, "h1"), - "pkg/a.py|pkg.a.helper": _method("pkg.a.helper", "pkg/a.py", 6, 8, "h2"), - "pkg/b.py|pkg.b.save": _method("pkg.b.save", "pkg/b.py", 1, 4, "h3"), - "pkg/b.py|~Weird": _method("~Weird", "pkg/b.py", 10, 12, "h4"), - }, - "components": [ - { - "component_id": "1", - "name": "Core", - "description": "d1", - "file_methods": [ - {"file_path": "pkg/a.py", "methods": ["pkg.a.run", "pkg.a.helper"]}, - {"file_path": "pkg/b.py", "methods": ["pkg.b.save", "~Weird"]}, - ], - "components": [ - { - "component_id": "1.1", - "name": "Run", - "file_methods": [{"file_path": "pkg/a.py", "methods": ["pkg.a.run", "pkg.a.helper"]}], - }, - { - "component_id": "1.2", - "name": "Store", - "file_methods": [{"file_path": "pkg/b.py", "methods": ["pkg.b.save", "~Weird"]}], - }, - ], - "components_relations": [ - { - "src_id": "1.1", - "dst_id": "1.2", - "relation": "saves", - "src_name": "Run", - "all_edges": [ - { - "source": "pkg/a.py|pkg.a.run", - "target": "pkg/b.py|pkg.b.save", - "call_sites": [{"line": 3}], - }, - {"source": "pkg/a.py|pkg.a.helper", "target": "pkg/b.py|pkg.b.save"}, - ], - } - ], - }, - { - "component_id": "2", - "name": "Also", - "file_methods": [ - {"file_path": "pkg/b.py", "methods": ["pkg.b.save"]}, - {"file_path": "pkg/c.py", "methods": ["pkg.c.gone"]}, - ], - }, - ], - "components_relations": [ - { - "src_id": "1", - "dst_id": "2", - "relation": "uses", - "key_edges": [], - "all_edges": [{"source": "pkg/a.py|pkg.a.run", "target": "pkg/b.py|pkg.b.save"}], - } + ROOT = _analysis( + [ + _component( + "1", + "Core", + {"pkg/a.py": ["pkg.a.run", "pkg.a.helper"], "pkg/b.py": ["pkg.b.save", "~Weird"]}, + description="d1", + ), + _component("2", "Also", {"pkg/b.py": ["pkg.b.save"], "pkg/c.py": ["pkg.c.gone"]}), ], - "files": {"pkg/a.py": {"method_keys": []}}, - "metadata": {"depth_level": 2}, + [_relation("1", "2", "uses", ("pkg/a.py|pkg.a.run", "pkg/b.py|pkg.b.save"))], + files={ + "pkg/a.py": [_method("pkg.a.run", 1, 5, "h1"), _method("pkg.a.helper", 6, 8, "h2")], + "pkg/b.py": [_method("pkg.b.save", 1, 4, "h3"), _method("~Weird", 10, 12, "h4")], + }, + ) + SUB_ANALYSES = { + "1": _analysis( + [ + _component("1.1", "Run", {"pkg/a.py": ["pkg.a.run", "pkg.a.helper"]}), + _component("1.2", "Store", {"pkg/b.py": ["pkg.b.save", "~Weird"]}), + ], + [ + _relation( + "1.1", + "1.2", + "saves", + ("pkg/a.py|pkg.a.run", "pkg/b.py|pkg.b.save"), + ("pkg/a.py|pkg.a.helper", "pkg/b.py|pkg.b.save"), + ) + ], + ) } GRAPH = { "nodes": { @@ -267,33 +269,89 @@ class SideDocumentTests(unittest.TestCase): } def test_each_fact_once_and_only_what_the_service_reads(self): - doc = json.loads(json.dumps(remote_features.side_document(self.ANALYSIS, self.GRAPH))) + doc = json.loads(json.dumps(remote_features.side_document(self.ROOT, self.SUB_ANALYSES, self.GRAPH))) self.assertEqual(doc, self.DOCUMENT) def test_a_parent_is_left_out_only_when_its_childrens_members_rebuild_it(self): # The service rebuilds a parent from its children's members, so a parent holding fewer than a # child does is written out even though its leaves hold exactly its members. - def component(cid, names, children=()): - return { - "component_id": cid, - "file_methods": [{"file_path": "a.py", "methods": list(names)}], - "components": list(children), - } - - analysis = { - "methods_index": {f"a.py|{q}": _method(q, "a.py", i, i, "h") for i, q in enumerate("xyz", 1)}, - "components": [ - component("G", "xy", [component("M", "xyz", [component("L1", "x"), component("L2", "y")])]), - component("P", "xy", [component("C1", "x"), component("C2", "y")]), - ], + def component(cid, names): + return _component(cid, members={"a.py": list(names)}) + + root = _analysis( + [component("G", "xy"), component("P", "xy")], + files={"a.py": [_method(q, i, i, "h") for i, q in enumerate("xyz", 1)]}, + ) + sub_analyses = { + "G": _analysis([component("M", "xyz")]), + "M": _analysis([component("L1", "x"), component("L2", "y")]), + "P": _analysis([component("C1", "x"), component("C2", "y")]), } - members = [c[4] for c in remote_features.side_document(analysis, {})["components"]] + members = [c[4] for c in remote_features.side_document(root, sub_analyses, {})["components"]] self.assertEqual(members, [[0, 1], [0, 1, 2], [0], [1], None, [0], [1]]) - def test_a_key_that_is_not_its_file_and_name_is_refused(self): - odd = {"methods_index": {"pkg/a.py|other": _method("pkg.a.run", "pkg/a.py", 1, 5, "h1")}} - with self.assertRaises(ValueError): - remote_features.side_document(odd, {}) + +def _core_installed() -> bool: + try: + metadata.version("codeboarding") + except metadata.PackageNotFoundError: + return False + return True + + +@unittest.skipUnless(_core_installed(), "the pinned CodeBoarding release is not installed") +class InstalledCoreTests(unittest.TestCase): + """read_analysis and side_document against the release action.yml pins. It runs in the + `core-compatibility` CI job. + + tests/fixtures/analysis holds one analysis as two Core releases wrote it: core-0.15.5.json and + core-9cf50fb9.json (main after #571). Each release reads only its own, so the pinned one must read at + least one, and every one it reads must give the same document. A pin whose release reads none needs + that analysis written by it, which is the point: the document has not been checked against its parser.""" + + GRAPH = { + "nodes": { + **SideDocumentTests.GRAPH["nodes"], + "pkg.c.load": {"file": "pkg/c.py", "start": 1, "end": 3, "cb": False}, + }, + "edges": [*SideDocumentTests.GRAPH["edges"], {"s": "pkg.c.load", "d": "pkg.b.save", "lines": [2]}], + } + DOCUMENT = { + "version": 1, + "files": [ + ["pkg/a.py", [["~.run", 1, 5, "FUNCTION", "h1", 0], ["~.helper", 6, 8, "FUNCTION", "h2", 0]]], + ["pkg/b.py", [["~.save", 1, 4, "FUNCTION", "h3", 0], ["~~Weird", 10, 12, "FUNCTION", "h4", -1]]], + ["pkg/c.py", [["~.load", 1, 3, "FUNCTION", "h5", 0]]], + ["pkg/x.py", [["~.Only", 1, 2, 1]]], + ], + "unplaced": ["ext.lib.call"], + "graph_spans": [[1, 6, 9]], + "components": [ + ["1", "Core", "d1", -1, None, [["1.1", "1.2", "saves"]]], + ["1.1", "Run", "", 0, [0, 1], []], + ["1.2", "Store", "", 0, [2, 3], []], + ["2", "Also", "", -1, [4], []], + ], + "relations": [["1.1", "1.2", "saves"], ["1.1", "2", "uses"], ["1.2", "2", "loads"], ["2", "1.2", "reads"]], + "calls": [[0, [[2, 3, 4], [1, 2]]], [2, [[0, 2]]], [4, [[2, 2]]], [5, [[6]]]], + # Of the analysis's edges, the calls are left to the graph: load -> save is one. + "extra_edges": [[0, 4], [1, 2], [3, 4]], + } + + def test_the_pinned_release_reads_an_analysis_it_wrote_into_the_side_document(self): + read = [] + for fixture in sorted((Path(__file__).parent / "fixtures" / "analysis").glob("*.json")): + try: + root, sub_analyses = remote_features.read_analysis(fixture) + except (TypeError, ValueError): # another release's format + continue + read.append(fixture.name) + with self.subTest(fixture=fixture.name): + doc = json.loads(json.dumps(remote_features.side_document(root, sub_analyses, self.GRAPH))) + self.assertEqual(doc, self.DOCUMENT) + self.assertTrue( + read, "the pinned release reads none of tests/fixtures/analysis; add the analysis as it writes it" + ) class CredentialTests(unittest.TestCase): @@ -368,6 +426,7 @@ def setUp(self): ("_github_json", {"side_effect": _github_json}), ("_contents", {"return_value": "base"}), ("call_graph", {"return_value": {"nodes": {}, "edges": []}}), + ("read_analysis", {"return_value": (_analysis([]), {})}), ): patch = mock.patch.object(remote_features, name, **kwargs) patch.start() From 1d392aff5f245aaf3356878a76d177b37e49e083 Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Thu, 12 Nov 2026 14:27:42 +0100 Subject: [PATCH 5/7] fix: check requests against both of the service's size limits before sending The service refuses a request on two sizes: what is sent (4.3 MB of xz, the Function URL's limit) and what it decompresses to (256 MiB of JSON for a feature, 32 MiB for a briefing). The Action checked only the first, so a briefing of 32-43 MiB of JSON, which xz takes under 4.3 MB, went out whole, was refused, and went out again on every sync. _encode now enforces both in one step: it measures the JSON, which is cheap, then compresses once. The limits sit together and name the service's constants they follow. The briefing leaves findings out against the JSON limit first, largest first, counted per entry, and only recompresses when the compressed size is still over, aimed by the ratio it just measured, at most 3 compressions in all. It used to recompress once per dropped finding. Co-Authored-By: Claude Opus 5.5 --- scripts/problem_history.py | 40 +++++++++++++++--------- scripts/remote_features.py | 57 ++++++++++++++++++++++++----------- tests/test_problem_history.py | 52 +++++++++++++++++++++++++------- tests/test_remote_features.py | 34 +++++++++++++-------- 4 files changed, 127 insertions(+), 56 deletions(-) diff --git a/scripts/problem_history.py b/scripts/problem_history.py index e3d2cce..fea3bc1 100644 --- a/scripts/problem_history.py +++ b/scripts/problem_history.py @@ -32,13 +32,13 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from remote_features import ( - MAX_BUNDLE_BYTES, + MAX_BRIEFING_JSON_BYTES, Skip, + TooLarge, _bearer, - _check_size, + _encode, _error_message, _github, - _compress, _post, ) # noqa: E402 @@ -71,6 +71,8 @@ # The review artifact's retention-days in action.yml; an unread merge older than this has nothing left to find. REVIEW_RETENTION_DAYS = 14 MAX_BODY_CHARS = 2000 +# Compressions of one request at most: the first, and retries aimed by the ratio the one before measured. +MAX_COMPRESSIONS = 3 SERVICE_TIMEOUT = 300 # A composite action's step cannot set timeout-minutes, so the script stops itself at the step's budget. STEP_LIMIT_SECONDS = 600 @@ -685,17 +687,27 @@ def build_request(env: dict, ledger: dict | None, facts: dict) -> dict: def pack(body: dict) -> tuple[bytes, list[str]]: - """The compressed request and the pull requests whose findings were left out of it: the largest - envelopes go first while it is over the service's limit, and their merges stay unread for a later run.""" - findings, dropped = dict(body.get("findings") or {}), [] - while True: - data = _compress({**body, "findings": findings}) - if len(data) <= MAX_BUNDLE_BYTES or not findings: - _check_size(data, "request") - return data, dropped - largest = max(findings, key=lambda pr: len(json.dumps(findings[pr]))) - del findings[largest] - dropped.append(largest) + """The compressed request and the pull requests whose findings were left out of it, largest first; + their merges stay unread for a later run. Findings are left out until the JSON is under the service's + limit, counted per entry rather than serialised again, and then it is compressed once. Only when that + is over the limit on what is sent is it compressed again, down to a little under what the ratio just + measured says fits, at most MAX_COMPRESSIONS times in all.""" + findings = body.get("findings") or {} + # An entry's bytes in the request: `"pr": {...}` and its separator, exact while one entry is left. + cost = {pr: len(json.dumps({pr: envelope})) for pr, envelope in findings.items()} + kept, dropped = sorted(findings, key=cost.get), [] + size, budget = len(json.dumps({**body, "findings": findings})), MAX_BRIEFING_JSON_BYTES + for compressions in range(1, MAX_COMPRESSIONS + 1): + while kept and size > budget: + dropped.append(kept.pop()) + size -= cost[dropped[-1]] + request = {**body, "findings": {pr: findings[pr] for pr in findings if pr not in dropped}} + try: + return _encode(request, MAX_BRIEFING_JSON_BYTES, "request"), dropped + except TooLarge as e: + if not kept or e.fits is None or compressions == MAX_COMPRESSIONS: + raise + budget = e.fits * 9 // 10 def _newest_pr(*lists) -> int | None: diff --git a/scripts/remote_features.py b/scripts/remote_features.py index 35c180d..6fa19bd 100644 --- a/scripts/remote_features.py +++ b/scripts/remote_features.py @@ -33,8 +33,10 @@ Size limits ----------- -- MAX_BUNDLE_BYTES (4.3 MB compressed): the most a request may carry. Past it the request is skipped, - after first dropping source.json if architecture carried it. +- The service sets two limits on every request, and _encode checks both before anything is sent: the + JSON, measured first because that is cheap, against MAX_FEATURE_JSON_BYTES (MAX_BRIEFING_JSON_BYTES + for the problem history), then the xz once against MAX_COMPRESSED_BYTES. Past either a feature request + is skipped, after first dropping source.json if architecture carried it. - MAX_FILES: how many changed files have their contents sent; past it a feature reads diffs only. - MAX_FILE_BYTES: a larger file is sent neither in files.json nor in source.json. - MAX_SOURCE_BYTES: the most source.json holds; past it source is not sent. @@ -71,9 +73,14 @@ from urllib.parse import quote, urlparse from urllib.request import Request, urlopen -# A Function URL takes 6 MB, measured after base64, and a compressed body goes through base64 on the -# way in: 4.5 MB of bundle arrives, and 5 MB is refused with a 413. -MAX_BUNDLE_BYTES = 4_300_000 +# The service's limits on a request, which move with it. A Function URL takes 6 MB, measured after +# base64, and a compressed body goes through base64 on the way in: 4.5 MB of it arrives, and 5 MB is +# refused with a 413. Decompressed, the service refuses a feature request over 256 MiB and a problem +# history request over 32 MiB: MAX_BUNDLE_BYTES and MAX_BRIEFING_BYTES in licensing-aws +# closed_functions/handler.py. +MAX_COMPRESSED_BYTES = 4_300_000 +MAX_FEATURE_JSON_BYTES = 256 * 1024 * 1024 +MAX_BRIEFING_JSON_BYTES = 32 * 1024 * 1024 # Changed files whose contents are sent at both commits; past this a feature reads diffs only. MAX_FILES = 300 # A file larger than this is sent neither as a changed file's contents nor as source. @@ -104,6 +111,15 @@ class Skip(Exception): """A step that did not happen, for a reason worth one line in the log.""" +class TooLarge(Skip): + """A request over one of the service's limits. Over the compressed one, `fits` is how much JSON the + compression ratio just measured would bring under it.""" + + def __init__(self, message: str, fits: int | None = None): + super().__init__(message) + self.fits = fits + + # --- GitHub --------------------------------------------------------------------------------------- @@ -605,21 +621,25 @@ def _post(url: str, body: bytes, headers: dict[str, str], timeout: int = 600) -> raise RuntimeError(reason) from error -def _compress(body: dict) -> bytes: - return lzma.compress(json.dumps(body).encode(), preset=9 | lzma.PRESET_EXTREME) +def _encode(body: dict, max_json_bytes: int, what: str = "bundle") -> bytes: + """The request as sent, xz-compressed JSON, or TooLarge when it is over either of the service's + limits. The JSON is measured before compressing it, which is the slow part, and it is compressed once.""" + text = json.dumps(body).encode() + if len(text) > max_json_bytes: + raise TooLarge(f"its {what} is {len(text) / 1e6:.0f} MB of JSON, over the service's limit") + data = lzma.compress(text, preset=9 | lzma.PRESET_EXTREME) + if len(data) > MAX_COMPRESSED_BYTES: + message = f"its {what} is {len(data) / 1e6:.1f} MB compressed, over the service's limit" + raise TooLarge(message, fits=len(text) * MAX_COMPRESSED_BYTES // len(data)) + return data def _pack(ctx: dict, files: dict[str, str], omitted: dict[str, str]) -> bytes: - """A request body: the named files, and why any optional one is missing.""" + """A feature request: the named files, and why any optional one is missing.""" body = {"version": 1, "repo": ctx["repo"], "pr": ctx["pr"], "files": files} if omitted: body["omitted"] = omitted - return _compress(body) - - -def _check_size(data: bytes, what: str = "bundle") -> None: - if len(data) > MAX_BUNDLE_BYTES: - raise Skip(f"its {what} is {len(data) / 1e6:.1f} MB, over the service's limit") + return _encode(body, MAX_FEATURE_JSON_BYTES) def _no_architecture(ctx: dict, status: str, reason: str) -> dict: @@ -643,7 +663,6 @@ def run_review_guide(service_url: str, ctx: dict, inputs: dict, headers: dict, a if "x-llm-provider" in headers and "x-llm-api-key" not in headers: raise Skip("the service cannot call this run's provider") data = _pack(ctx, inputs, {}) - _check_size(data) answer = _post(f"{service_url.rstrip('/')}/v1/review-guide", data, headers) guide = answer.get("guide") if not isinstance(guide, dict): @@ -664,12 +683,14 @@ def run_architecture(service_url: str, ctx: dict, inputs: dict, headers: dict, e omitted["source.json"] = why or "the source was not collected" else: files["source.json"] = source - data = _pack(ctx, files, omitted) - if "source.json" in files and len(data) > MAX_BUNDLE_BYTES: + try: + data = _pack(ctx, files, omitted) + except TooLarge: + if "source.json" not in files: + raise del files["source.json"] omitted["source.json"] = "the repository is too large to send" data = _pack(ctx, files, omitted) - _check_size(data) answer = _post(f"{service_url.rstrip('/')}/v1/architecture", data, headers) envelope = answer.get("envelope") if not isinstance(envelope, dict) or envelope.get("status") not in ("ok", "partial", "skipped", "failed"): diff --git a/tests/test_problem_history.py b/tests/test_problem_history.py index 6a7f197..26c9c04 100644 --- a/tests/test_problem_history.py +++ b/tests/test_problem_history.py @@ -391,8 +391,11 @@ def test_every_body_is_cut_to_2000_characters_and_an_oversized_request_is_refuse gh.issue_pages[(("OPEN",), "DESC")] = [[issue_node(1, body="x" * 5000), issue_node(2)]] self.run_update(gh) self.assertEqual([i["body"] for i in self.body["issues"]], ["x" * 2000, "It breaks when saving."]) - with mock.patch.dict(ph._check_size.__globals__, {"MAX_BUNDLE_BYTES": 10}): - with self.assertRaisesRegex(problem_history.Skip, "its request is"): + with mock.patch.dict(ph._encode.__globals__, {"MAX_COMPRESSED_BYTES": 10}): + with self.assertRaisesRegex(problem_history.Skip, "its request is .* compressed"): + problem_history.pack({"issues": []}) + with mock.patch.object(ph, "MAX_BRIEFING_JSON_BYTES", 10): + with self.assertRaisesRegex(problem_history.Skip, "its request is .* of JSON"): problem_history.pack({"issues": []}) def test_a_cut_counts_utf16_units_and_never_splits_a_surrogate_pair(self): @@ -784,22 +787,49 @@ def test_an_unread_merge_older_than_the_review_artifacts_is_not_asked_again(self retention = re.search(r"retention-days: (\d+)", upload).group(1) self.assertEqual(int(retention), ph.REVIEW_RETENTION_DAYS) - def test_findings_that_would_overflow_the_request_are_left_out(self): + def pack_counting(self, body, **limits): + """pack(body) under `limits` (MAX_BRIEFING_JSON_BYTES, MAX_COMPRESSED_BYTES), and how often it compressed.""" + compress = mock.Mock(wraps=lzma.compress) + with ( + mock.patch.object( + ph, "MAX_BRIEFING_JSON_BYTES", limits.get("MAX_BRIEFING_JSON_BYTES", ph.MAX_BRIEFING_JSON_BYTES) + ), + mock.patch.dict(ph._encode.__globals__, limits), + mock.patch.object(lzma, "compress", compress), + ): + data, dropped = ph.pack(body) + return json.loads(lzma.decompress(data))["findings"], dropped, compress.call_count + + def test_findings_over_the_services_json_limit_are_left_out_before_compressing(self): + # xz takes these to almost nothing, so only their JSON is over the service's limit. + small, big = {**ENVELOPE, "data": {"findings": ["a"]}}, {**ENVELOPE, "data": {"findings": ["b" * 3000]}} + body = {"issues": [], "findings": {"41": small, "42": big, "43": small}} + limit = len(json.dumps({**body, "findings": {"41": small, "43": small}})) + kept, dropped, compressions = self.pack_counting(body, MAX_BRIEFING_JSON_BYTES=limit) + self.assertEqual((kept, dropped, compressions), ({"41": small, "43": small}, ["42"], 1)) + + def test_findings_over_the_compressed_limit_are_left_out_in_a_few_compressions(self): + rnd = random.Random(1) + findings = { + str(pr): {**ENVELOPE, "data": {"findings": [rnd.randbytes(100 + 20 * pr).hex()]}} for pr in range(60) + } + kept, dropped, compressions = self.pack_counting( + {"issues": [], "findings": findings}, MAX_COMPRESSED_BYTES=20_000 + ) + self.assertLessEqual(compressions, ph.MAX_COMPRESSIONS) + self.assertTrue(dropped and kept) + self.assertEqual(list(kept), [pr for pr in findings if pr not in dropped]) + self.assertLess(max(map(int, kept)), min(map(int, dropped))) + small, big = {**ENVELOPE, "data": {"findings": ["a"]}}, { **ENVELOPE, "data": {"findings": [random.Random(1).randbytes(3000).hex()]}, } - body = {"issues": [], "findings": {"41": small, "42": big}} - with mock.patch.dict(ph._check_size.__globals__, {"MAX_BUNDLE_BYTES": 1000}): - with mock.patch.object(ph, "MAX_BUNDLE_BYTES", 1000): - data, dropped = ph.pack(body) - self.assertEqual((json.loads(lzma.decompress(data))["findings"], dropped), ({"41": small}, ["42"])) - gh = FakeGitHub() gh.merged_pages = [[pr_node(41, "2026-11-06T00:00:00Z"), pr_node(42, "2026-11-05T00:00:00Z")]] gh.add_review(41, 41, small) gh.add_review(42, 42, big) - with mock.patch.object(ph, "MAX_BUNDLE_BYTES", 1200): + with mock.patch.dict(ph._encode.__globals__, {"MAX_COMPRESSED_BYTES": 1200}): notice = self.run_update(gh) self.assertEqual(list(self.body["findings"]), ["41"]) self.assertIn("left out the findings of #42 to stay under the request limit", notice) @@ -1008,7 +1038,7 @@ def test_the_step_uses_only_keys_a_composite_run_step_accepts(self): self.assertIn("continue-on-error", keys) def test_shares_the_remote_features_transport(self): - for shared in (ph._bearer, ph._post, ph._github, ph._compress, ph._check_size, ph._error_message): + for shared in (ph._bearer, ph._post, ph._github, ph._encode, ph._error_message): self.assertEqual(shared.__module__, "remote_features") diff --git a/tests/test_remote_features.py b/tests/test_remote_features.py index 573775a..7c835c1 100644 --- a/tests/test_remote_features.py +++ b/tests/test_remote_features.py @@ -134,7 +134,7 @@ def test_analysed_commits_and_both_sides(self): self.assertEqual(set(bundle), INPUTS) self.assertEqual(json.loads(bundle["head.json"])["version"], 1) - def test_a_changed_file_over_the_size_limit_is_sent_at_neither_commit(self): + def test_a_changed_file_over_the_size_limit_is_left_out_only_at_that_commit(self): with tempfile.TemporaryDirectory() as tmp: env = _workspace(Path(tmp)) with ( @@ -543,19 +543,27 @@ def test_source_toggle_and_read_failure_only_omit_architecture_source(self): self.assertNotIn("source.json", body["files"]) self.assertEqual("omitted" in body, request["path"] == "/v1/architecture") - def test_bundle_limit_omits_source_then_skips_if_required_inputs_too_large(self): - with ( - mock.patch.object(remote_features, "MAX_BUNDLE_BYTES", 1000), - mock.patch.object(remote_features, "collect_source", return_value=(os.urandom(2000).hex(), None)), + def test_either_limit_omits_source_then_skips_if_required_inputs_too_large(self): + with mock.patch.object(remote_features, "collect_source", return_value=(None, "off")): + without_source = max(len(lzma.decompress(r["body"])) for r in self.run_service().requests) + # Random source is over the compressed limit; repeated source compresses to almost nothing and is + # over the JSON limit only. + for limit, room, source in ( + ("MAX_COMPRESSED_BYTES", 1000, os.urandom(2000).hex()), + ("MAX_FEATURE_JSON_BYTES", without_source + 1000, "x" * 100_000), ): - service = self.run_service() - arch = next(r for r in service.requests if r["path"] == "/v1/architecture") - body = json.loads(lzma.decompress(arch["body"])) - self.assertNotIn("source.json", body["files"]) - self.assertIn("too large", body["omitted"]["source.json"]) - with mock.patch.object(remote_features, "MAX_BUNDLE_BYTES", 1): - self.assertEqual(self.run_service().requests, []) - self.assertEqual(_read(self.env, "closed/architecture.json")["status"], "skipped") + with ( + mock.patch.object(remote_features, limit, room), + mock.patch.object(remote_features, "collect_source", return_value=(source, None)), + ): + service = self.run_service() + arch = next(r for r in service.requests if r["path"] == "/v1/architecture") + body = json.loads(lzma.decompress(arch["body"])) + self.assertNotIn("source.json", body["files"]) + self.assertIn("too large", body["omitted"]["source.json"]) + with mock.patch.object(remote_features, limit, 1): + self.assertEqual(self.run_service().requests, []) + self.assertEqual(_read(self.env, "closed/architecture.json")["status"], "skipped") class SourceTests(unittest.TestCase): From 175d7cd0347589e877aa4741d42cd8010862b2e4 Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Thu, 12 Nov 2026 17:58:53 +0100 Subject: [PATCH 6/7] perf: compress each feature request once, one at a time The guide and architecture each compressed their own body in parallel, nearly the same bytes twice, and architecture compressed again after dropping source.json. At preset 9 | EXTREME each compressor takes about 700 MB. Both features can send the body without source, so it is compressed once and carries why source is missing, which only architecture reads. Architecture's body with source is compressed only when there is source; when it does not fit, the shared bytes stand in. A run compresses once without source and at most twice with it, and never two at a time. The JSON check needs no ratio guess: the 256 MiB limit is 62 times the 4.3 MB one, so it already turns away whatever could only fit at better than 62:1, and no tighter cut-off holds for every request. Co-Authored-By: Claude Opus 5.5 --- scripts/remote_features.py | 66 ++++++++++++++++++----------------- tests/test_remote_features.py | 47 ++++++++++++++----------- 2 files changed, 61 insertions(+), 52 deletions(-) diff --git a/scripts/remote_features.py b/scripts/remote_features.py index 6fa19bd..fd5fc6e 100644 --- a/scripts/remote_features.py +++ b/scripts/remote_features.py @@ -8,7 +8,8 @@ What a request sends -------------------- -One JSON object of named files, compressed with xz. Both requests send the same files: +One JSON object of named files, compressed with xz. Both requests send the same files, and the same +bytes unless architecture's carries source: - head.json and base.json: one side document per commit (see side_document). It holds what the service reads from that commit's analysis.json and call graph (static_analysis.pkl), each fact once: @@ -35,8 +36,12 @@ ----------- - The service sets two limits on every request, and _encode checks both before anything is sent: the JSON, measured first because that is cheap, against MAX_FEATURE_JSON_BYTES (MAX_BRIEFING_JSON_BYTES - for the problem history), then the xz once against MAX_COMPRESSED_BYTES. Past either a feature request - is skipped, after first dropping source.json if architecture carried it. + for the problem history), then the xz once against MAX_COMPRESSED_BYTES. The JSON limit is 62 times + the compressed one, so the cheap check already turns away whatever could only fit at better than 62:1. + These bundles compress about 10:1, but no tighter cut-off holds for every request. +- xz at this preset takes about 700 MB and seconds per 10 MB, so a run compresses one body at a time and + each at most once: the body without source, which both features send, and architecture's with source + when there is source. When that one does not fit, architecture sends the shared one. - MAX_FILES: how many changed files have their contents sent; past it a feature reads diffs only. - MAX_FILE_BYTES: a larger file is sent neither in files.json nor in source.json. - MAX_SOURCE_BYTES: the most source.json holds; past it source is not sent. @@ -634,12 +639,24 @@ def _encode(body: dict, max_json_bytes: int, what: str = "bundle") -> bytes: return data -def _pack(ctx: dict, files: dict[str, str], omitted: dict[str, str]) -> bytes: - """A feature request: the named files, and why any optional one is missing.""" - body = {"version": 1, "repo": ctx["repo"], "pr": ctx["pr"], "files": files} - if omitted: - body["omitted"] = omitted - return _encode(body, MAX_FEATURE_JSON_BYTES) +def _requests(ctx: dict, inputs: dict[str, str], env: dict) -> tuple[bytes, bytes]: + """The review guide's request and architecture's, compressed one after the other. Both can send the + body without source, so it is compressed once; it says why source is missing, which only architecture + reads. Architecture's body with source is compressed only when there is source, and the shared one + stands in when it does not fit. TooLarge when not even the shared one fits.""" + try: + source, why = collect_source(env) + except Exception as error: + source, why = None, f"reading it failed: {type(error).__name__}" + body = {"version": 1, "repo": ctx["repo"], "pr": ctx["pr"], "files": inputs} + missing = (why or "the source was not collected") if source is None else "the repository is too large to send" + shared = _encode({**body, "omitted": {"source.json": missing}}, MAX_FEATURE_JSON_BYTES) + if source is None: + return shared, shared + try: + return shared, _encode({**body, "files": {**inputs, "source.json": source}}, MAX_FEATURE_JSON_BYTES) + except TooLarge: + return shared, shared def _no_architecture(ctx: dict, status: str, reason: str) -> dict: @@ -658,11 +675,10 @@ def _no_architecture(ctx: dict, status: str, reason: str) -> dict: } -def run_review_guide(service_url: str, ctx: dict, inputs: dict, headers: dict, artifact: Path) -> None: +def run_review_guide(service_url: str, data: bytes, headers: dict, artifact: Path) -> None: try: if "x-llm-provider" in headers and "x-llm-api-key" not in headers: raise Skip("the service cannot call this run's provider") - data = _pack(ctx, inputs, {}) answer = _post(f"{service_url.rstrip('/')}/v1/review-guide", data, headers) guide = answer.get("guide") if not isinstance(guide, dict): @@ -672,25 +688,8 @@ def run_review_guide(service_url: str, ctx: dict, inputs: dict, headers: dict, a print(f"::notice::review-guide {'skipped' if isinstance(error, Skip) else 'failed'}: {error}") -def run_architecture(service_url: str, ctx: dict, inputs: dict, headers: dict, env: dict) -> dict: +def run_architecture(service_url: str, ctx: dict, data: bytes, headers: dict) -> dict: try: - try: - source, why = collect_source(env) - except Exception as error: - source, why = None, f"reading it failed: {type(error).__name__}" - files, omitted = dict(inputs), {} - if source is None: - omitted["source.json"] = why or "the source was not collected" - else: - files["source.json"] = source - try: - data = _pack(ctx, files, omitted) - except TooLarge: - if "source.json" not in files: - raise - del files["source.json"] - omitted["source.json"] = "the repository is too large to send" - data = _pack(ctx, files, omitted) answer = _post(f"{service_url.rstrip('/')}/v1/architecture", data, headers) envelope = answer.get("envelope") if not isinstance(envelope, dict) or envelope.get("status") not in ("ok", "partial", "skipped", "failed"): @@ -711,15 +710,18 @@ def main(env: dict | None = None) -> int: return 0 try: headers = {**_bearer(env), **model_headers(env)} - inputs = collect(env) + guide_request, architecture_request = _requests(ctx, collect(env), env) + except TooLarge as error: + print(f"::notice::review-guide skipped: {error}") + architecture = _no_architecture(ctx, "skipped", str(error)) except Exception as error: reason = f"collecting inputs or credentials failed: {type(error).__name__}: {error}" print(f"::notice::review-guide failed: {reason}") architecture = _no_architecture(ctx, "failed", reason) else: with ThreadPoolExecutor(max_workers=2) as pool: - guide = pool.submit(run_review_guide, service_url, ctx, inputs, headers, artifact) - result = pool.submit(run_architecture, service_url, ctx, inputs, headers, env) + guide = pool.submit(run_review_guide, service_url, guide_request, headers, artifact) + result = pool.submit(run_architecture, service_url, ctx, architecture_request, headers) architecture = result.result() guide.result() path = artifact / "closed" / "architecture.json" diff --git a/tests/test_remote_features.py b/tests/test_remote_features.py index 7c835c1..2d80da5 100644 --- a/tests/test_remote_features.py +++ b/tests/test_remote_features.py @@ -9,9 +9,8 @@ import tempfile import threading import unittest -from importlib import metadata -from contextlib import nullcontext from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from importlib import metadata from pathlib import Path from types import SimpleNamespace from unittest import mock @@ -529,19 +528,27 @@ def test_empty_url_disables_both(self): post.assert_not_called() self.assertFalse((Path(self.env["ARTIFACT_DIR"]) / "closed").exists()) - def test_source_toggle_and_read_failure_only_omit_architecture_source(self): + def run_counting(self, **patches): + """run_service with `patches` on remote_features, and how often it compressed.""" + compress = mock.Mock(wraps=lzma.compress) + with mock.patch.object(lzma, "compress", compress), mock.patch.dict(remote_features.__dict__, patches): + service = self.run_service() + return {r["path"]: r["body"] for r in service.requests}, compress.call_count + + def test_without_source_both_send_one_body_compressed_once(self): for failure in (False, True): self.env["CLOSED_FEATURES_SOURCE"] = "false" - with ( - mock.patch.object(remote_features, "collect_source", side_effect=OSError()) - if failure - else nullcontext() - ): - service = self.run_service() - for request in service.requests: - body = json.loads(lzma.decompress(request["body"])) - self.assertNotIn("source.json", body["files"]) - self.assertEqual("omitted" in body, request["path"] == "/v1/architecture") + patches = {"collect_source": mock.Mock(side_effect=OSError())} if failure else {} + bodies, compressions = self.run_counting(**patches) + self.assertEqual((compressions, len(set(bodies.values()))), (1, 1)) + body = json.loads(lzma.decompress(bodies["/v1/architecture"])) + self.assertNotIn("source.json", body["files"]) + self.assertIn("reading it failed" if failure else "turned off", body["omitted"]["source.json"]) + + def test_source_is_compressed_once_more_and_only_for_architecture(self): + bodies, compressions = self.run_counting(collect_source=mock.Mock(return_value=("{}", None))) + self.assertEqual(compressions, 2) + self.assertNotEqual(bodies["/v1/architecture"], bodies["/v1/review-guide"]) def test_either_limit_omits_source_then_skips_if_required_inputs_too_large(self): with mock.patch.object(remote_features, "collect_source", return_value=(None, "off")): @@ -552,13 +559,13 @@ def test_either_limit_omits_source_then_skips_if_required_inputs_too_large(self) ("MAX_COMPRESSED_BYTES", 1000, os.urandom(2000).hex()), ("MAX_FEATURE_JSON_BYTES", without_source + 1000, "x" * 100_000), ): - with ( - mock.patch.object(remote_features, limit, room), - mock.patch.object(remote_features, "collect_source", return_value=(source, None)), - ): - service = self.run_service() - arch = next(r for r in service.requests if r["path"] == "/v1/architecture") - body = json.loads(lzma.decompress(arch["body"])) + bodies, compressions = self.run_counting( + **{limit: room, "collect_source": mock.Mock(return_value=(source, None))} + ) + # Architecture sends the guide's body, which already says why source is missing. + self.assertEqual(bodies["/v1/architecture"], bodies["/v1/review-guide"]) + self.assertEqual(compressions, 2 if limit == "MAX_COMPRESSED_BYTES" else 1) + body = json.loads(lzma.decompress(bodies["/v1/architecture"])) self.assertNotIn("source.json", body["files"]) self.assertIn("too large", body["omitted"]["source.json"]) with mock.patch.object(remote_features, limit, 1): From 140f60599cedb8fed935211822b634561285a6d1 Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Thu, 12 Nov 2026 21:13:08 +0100 Subject: [PATCH 7/7] fix: build the review diff from Core's parsed analysis, not the document load_analysis already parsed both sides with the pinned release's parse_unified_analysis and rolled relations up with its project_relations_to_level, but the diff then read members, file hashes and method hashes from the raw document: file_methods as {file_path, methods} groups and methods_index. Core #571's "v1" document has neither, so once the Action pins a release with it, the review diff fails with "'str' object has no attribute 'get'". The diff now reads only what Core parsed: each component's file_methods, the root's file index and Core's projected relations at every level. The review analyses both commits with the pinned release, so both sides are always in the format it reads, and a format change in Core needs no change here. The core-compatibility job rendered the committed .codeboarding/analysis.json, which after a pin moves can still be in the format of the release before, which the new one refuses. It now runs InstalledCoreTests: tests/fixtures/ analysis holds one analysis as Core 0.15.5 and Core main (9cf50fb9) wrote it, each release reads only its own, and every fixture the pinned one reads must give the same view. Checked on CodeBoarding's own analysis (two baseline syncs, 22 components, depth 1 and 2) and this repo's: output and summary are byte-identical to main's script on the legacy documents with 0.15.5, and to that again on their v1 rewrites with Core main, where main's script fails. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test.yml | 12 +- scripts/diff_to_mermaid.py | 56 ++-- tests/fixtures/analysis/core-0.15.5.json | 289 +++++++++++++++++++++ tests/fixtures/analysis/core-9cf50fb9.json | 217 ++++++++++++++++ tests/test_action_sync.py | 2 +- tests/test_diff_to_mermaid.py | 191 ++++++++++---- 6 files changed, 688 insertions(+), 79 deletions(-) create mode 100644 tests/fixtures/analysis/core-0.15.5.json create mode 100644 tests/fixtures/analysis/core-9cf50fb9.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8d6efd..d9f8887 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,14 +53,10 @@ jobs: printf '{"components": []}\n' > "$ANALYSIS_DIR/analysis.json" scripts/action/install-sync.sh cmp "$ANALYSIS_DIR/analysis.json" "$CHECKOUT_DIR/.codeboarding/analysis.json" - - name: Render with the installed CodeBoarding release - run: >- - python scripts/diff_to_mermaid.py - --base .codeboarding/analysis.json - --head .codeboarding/analysis.json - --out "${RUNNER_TEMP}/diagram.md" - --direction LR - --render-depth 1 + # Not the committed .codeboarding/analysis.json: after a pin moves, that can still be in the + # format of the release before, which the new one refuses. + - name: Read and diff analyses with the installed CodeBoarding release + run: python -m unittest tests.test_diff_to_mermaid.InstalledCoreTests -v lint: runs-on: ubuntu-latest diff --git a/scripts/diff_to_mermaid.py b/scripts/diff_to_mermaid.py index 4d8283d..7b2f69b 100644 --- a/scripts/diff_to_mermaid.py +++ b/scripts/diff_to_mermaid.py @@ -20,7 +20,9 @@ analysed code changed, whatever the component diff says: a component can still read as changed when the analysis grouped the same code differently. -Self-contained stdlib. +Stdlib only, apart from the installed CodeBoarding release: both analyses are read through its parser +and never as a document, so whatever format that release writes is read. The review analyses both +commits with it, so both sides are always in that format. """ from __future__ import annotations @@ -63,23 +65,46 @@ def _relation_dict(relation) -> dict: } -def _project_analysis(data: dict, root_analysis, sub_analyses: dict, id_to_name: dict, projector) -> dict: - """Attach Core's global-relation projection to every nested JSON level.""" +def _diff_view(root_analysis, sub_analyses: dict, id_to_name: dict, projector) -> dict: + """The dict the diff reads, built from Core's parsed analysis rather than the document: the component + tree with each component's methods by file, Core's projection of the global relations at every level, + and the content hash of every analysed file and method.""" global_relations = list(root_analysis.components_relations) - def project_level(container: dict, analysis) -> None: + def level(analysis) -> dict: level_ids = {component.component_id for component in analysis.components} - container["components_relations"] = [ - _relation_dict(relation) for relation in projector(global_relations, level_ids, id_to_name) - ] - for component in container.get("components") or []: - component_id = component.get("component_id", "") - sub_analysis = sub_analyses.get(component_id) + components = [] + for component in analysis.components: + view = { + "name": component.name, + "component_id": component.component_id, + "file_methods": [ + {"file_path": group.file_path, "methods": [method.qualified_name for method in group.methods]} + for group in component.file_methods + ], + } + sub_analysis = sub_analyses.get(component.component_id) if sub_analysis is not None: - project_level(component, sub_analysis) - - project_level(data, root_analysis) - return data + view.update(level(sub_analysis)) + components.append(view) + return { + "components": components, + "components_relations": [ + _relation_dict(relation) for relation in projector(global_relations, level_ids, id_to_name) + ], + } + + files = root_analysis.files + return { + **level(root_analysis), + # files[""] only holds pathless methods; it is no file. + "files": {path: {"content_hash": entry.content_hash} for path, entry in files.items() if path}, + "methods_index": { + f"{path}|{method.qualified_name}": {"file_path": path, "content_hash": method.content_hash} + for path, entry in files.items() + for method in entry.methods + }, + } def read_analysis_json(path: Path) -> dict: @@ -101,8 +126,7 @@ def load_analysis(path: Path) -> dict: root_analysis, sub_analyses = parse_unified_analysis(data) except (KeyError, TypeError, ValueError) as exc: sys.exit(f"::error::Could not parse analysis JSON at {path}: {exc}") - return _project_analysis( - data, + return _diff_view( root_analysis, sub_analyses, build_id_to_name_map(root_analysis, sub_analyses), diff --git a/tests/fixtures/analysis/core-0.15.5.json b/tests/fixtures/analysis/core-0.15.5.json new file mode 100644 index 0000000..d122b4f --- /dev/null +++ b/tests/fixtures/analysis/core-0.15.5.json @@ -0,0 +1,289 @@ +{ + "metadata": { + "generated_at": "2026-11-12T00:00:00Z", + "source_tree_hash": "", + "repo_name": "pkg", + "depth_level": 2, + "depth_cap": 2, + "file_coverage_summary": { + "total_files": 3, + "analyzed": 3, + "not_analyzed": 0, + "not_analyzed_by_reason": {} + }, + "tree_spec": {} + }, + "description": "fixture", + "files": { + "pkg/a.py": { + "method_keys": [ + "pkg/a.py|pkg.a.run", + "pkg/a.py|pkg.a.helper" + ], + "content_hash": "", + "module_hash": "" + }, + "pkg/b.py": { + "method_keys": [ + "pkg/b.py|pkg.b.save", + "pkg/b.py|~Weird" + ], + "content_hash": "", + "module_hash": "" + }, + "pkg/c.py": { + "method_keys": [ + "pkg/c.py|pkg.c.load" + ], + "content_hash": "", + "module_hash": "" + } + }, + "methods_index": { + "pkg/a.py|pkg.a.run": { + "file_path": "pkg/a.py", + "qualified_name": "pkg.a.run", + "start_line": 1, + "end_line": 5, + "type": "FUNCTION", + "content_hash": "h1" + }, + "pkg/a.py|pkg.a.helper": { + "file_path": "pkg/a.py", + "qualified_name": "pkg.a.helper", + "start_line": 6, + "end_line": 8, + "type": "FUNCTION", + "content_hash": "h2" + }, + "pkg/b.py|pkg.b.save": { + "file_path": "pkg/b.py", + "qualified_name": "pkg.b.save", + "start_line": 1, + "end_line": 4, + "type": "FUNCTION", + "content_hash": "h3" + }, + "pkg/b.py|~Weird": { + "file_path": "pkg/b.py", + "qualified_name": "~Weird", + "start_line": 10, + "end_line": 12, + "type": "FUNCTION", + "content_hash": "h4" + }, + "pkg/c.py|pkg.c.load": { + "file_path": "pkg/c.py", + "qualified_name": "pkg.c.load", + "start_line": 1, + "end_line": 3, + "type": "FUNCTION", + "content_hash": "h5" + } + }, + "components": [ + { + "name": "Core", + "description": "d1", + "key_entities": [], + "source_cluster_ids": [], + "file_methods": [ + { + "file_path": "pkg/a.py", + "methods": [ + "pkg.a.run", + "pkg.a.helper" + ] + }, + { + "file_path": "pkg/b.py", + "methods": [ + "pkg.b.save", + "~Weird" + ] + } + ], + "component_id": "1", + "can_expand": true, + "components": [ + { + "name": "Run", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "file_methods": [ + { + "file_path": "pkg/a.py", + "methods": [ + "pkg.a.run", + "pkg.a.helper" + ] + } + ], + "component_id": "1.1", + "can_expand": false, + "components": [], + "components_relations": [] + }, + { + "name": "Store", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "file_methods": [ + { + "file_path": "pkg/b.py", + "methods": [ + "pkg.b.save", + "~Weird" + ] + } + ], + "component_id": "1.2", + "can_expand": false, + "components": [], + "components_relations": [] + } + ], + "components_relations": [ + { + "relation": "saves", + "src_name": "Run", + "dst_name": "Store", + "evidence": "", + "key_edges": [], + "src_id": "1.1", + "dst_id": "1.2", + "is_static": false, + "all_edges": [ + { + "source": "pkg/a.py|pkg.a.run", + "target": "pkg/b.py|pkg.b.save", + "call_sites": [ + { + "line": 3, + "column": 8 + } + ], + "description": "" + }, + { + "source": "pkg/a.py|pkg.a.helper", + "target": "pkg/b.py|pkg.b.save", + "call_sites": [], + "description": "" + } + ] + } + ] + }, + { + "name": "Also", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "file_methods": [ + { + "file_path": "pkg/c.py", + "methods": [ + "pkg.c.load" + ] + } + ], + "component_id": "2", + "can_expand": false, + "components": [], + "components_relations": [] + } + ], + "components_relations": [ + { + "relation": "saves", + "src_name": "Run", + "dst_name": "Store", + "evidence": "", + "key_edges": [], + "src_id": "1.1", + "dst_id": "1.2", + "is_static": false, + "all_edges": [ + { + "source": "pkg/a.py|pkg.a.run", + "target": "pkg/b.py|pkg.b.save", + "call_sites": [ + { + "line": 3, + "column": 8 + } + ], + "description": "" + }, + { + "source": "pkg/a.py|pkg.a.helper", + "target": "pkg/b.py|pkg.b.save", + "call_sites": [], + "description": "" + } + ] + }, + { + "relation": "uses", + "src_name": "Run", + "dst_name": "Also", + "evidence": "", + "key_edges": [], + "src_id": "1.1", + "dst_id": "2", + "is_static": false, + "all_edges": [ + { + "source": "pkg/a.py|pkg.a.run", + "target": "pkg/c.py|pkg.c.load", + "call_sites": [ + { + "line": 4, + "column": 4 + } + ], + "description": "" + } + ] + }, + { + "relation": "loads", + "src_name": "Store", + "dst_name": "Also", + "evidence": "", + "key_edges": [], + "src_id": "1.2", + "dst_id": "2", + "is_static": false, + "all_edges": [ + { + "source": "pkg/b.py|~Weird", + "target": "pkg/c.py|pkg.c.load", + "call_sites": [], + "description": "" + } + ] + }, + { + "relation": "reads", + "src_name": "Also", + "dst_name": "Store", + "evidence": "", + "key_edges": [], + "src_id": "2", + "dst_id": "1.2", + "is_static": false, + "all_edges": [ + { + "source": "pkg/c.py|pkg.c.load", + "target": "pkg/b.py|pkg.b.save", + "call_sites": [], + "description": "" + } + ] + } + ] +} diff --git a/tests/fixtures/analysis/core-9cf50fb9.json b/tests/fixtures/analysis/core-9cf50fb9.json new file mode 100644 index 0000000..4ddba2e --- /dev/null +++ b/tests/fixtures/analysis/core-9cf50fb9.json @@ -0,0 +1,217 @@ +{ + "metadata": { + "generated_at": "2026-11-12T00:00:00Z", + "source_tree_hash": "", + "repo_name": "pkg", + "depth_level": 2, + "depth_cap": 2, + "file_coverage_summary": { + "total_files": 3, + "analyzed": 3, + "not_analyzed": 0, + "not_analyzed_by_reason": {} + }, + "tree_spec": {}, + "format_version": "v1" + }, + "description": "fixture", + "files": { + "pkg/a.py": { + "content_hash": "", + "module_hash": "", + "methods": { + "pkg.a.run": [ + 1, + 5, + "FUNCTION", + "h1", + "W2ZvP4dicxdV" + ], + "pkg.a.helper": [ + 6, + 8, + "FUNCTION", + "h2", + "fHTj2iSkCxvz" + ] + } + }, + "pkg/b.py": { + "content_hash": "", + "module_hash": "", + "methods": { + "pkg.b.save": [ + 1, + 4, + "FUNCTION", + "h3", + "HXVcn6KDWNds" + ], + "~Weird": [ + 10, + 12, + "FUNCTION", + "h4", + "VHMX26c_wkXu" + ] + } + }, + "pkg/c.py": { + "content_hash": "", + "module_hash": "", + "methods": { + "pkg.c.load": [ + 1, + 3, + "FUNCTION", + "h5", + "ai1syDPa_D53" + ] + } + } + }, + "components": [ + { + "name": "Core", + "component_id": "1", + "description": "d1", + "key_entities": [], + "source_cluster_ids": [], + "can_expand": true, + "components": [ + { + "name": "Run", + "component_id": "1.1", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "can_expand": false, + "file_methods": [ + "W2ZvP4dicxdV", + "fHTj2iSkCxvz" + ], + "components": [], + "components_relations": [] + }, + { + "name": "Store", + "component_id": "1.2", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "can_expand": false, + "file_methods": [ + "HXVcn6KDWNds", + "VHMX26c_wkXu" + ], + "components": [], + "components_relations": [] + } + ], + "components_relations": [ + { + "relation": "saves", + "src_name": "Run", + "dst_name": "Store", + "evidence": "", + "src_id": "1.1", + "dst_id": "1.2", + "is_static": false, + "key_edges": [] + } + ] + }, + { + "name": "Also", + "component_id": "2", + "description": "", + "key_entities": [], + "source_cluster_ids": [], + "can_expand": false, + "file_methods": [ + "ai1syDPa_D53" + ], + "components": [], + "components_relations": [] + } + ], + "components_relations": [ + { + "relation": "saves", + "src_name": "Run", + "dst_name": "Store", + "evidence": "", + "src_id": "1.1", + "dst_id": "1.2", + "is_static": false, + "key_edges": [] + }, + { + "relation": "uses", + "src_name": "Run", + "dst_name": "Also", + "evidence": "", + "src_id": "1.1", + "dst_id": "2", + "is_static": false, + "key_edges": [] + }, + { + "relation": "loads", + "src_name": "Store", + "dst_name": "Also", + "evidence": "", + "src_id": "1.2", + "dst_id": "2", + "is_static": false, + "key_edges": [] + }, + { + "relation": "reads", + "src_name": "Also", + "dst_name": "Store", + "evidence": "", + "src_id": "2", + "dst_id": "1.2", + "is_static": false, + "key_edges": [] + } + ], + "edges": [ + [ + "fHTj2iSkCxvz", + "HXVcn6KDWNds", + [] + ], + [ + "W2ZvP4dicxdV", + "HXVcn6KDWNds", + [ + [ + 3, + 8 + ] + ] + ], + [ + "W2ZvP4dicxdV", + "ai1syDPa_D53", + [ + [ + 4, + 4 + ] + ] + ], + [ + "VHMX26c_wkXu", + "ai1syDPa_D53", + [] + ], + [ + "ai1syDPa_D53", + "HXVcn6KDWNds", + [] + ] + ] +} diff --git a/tests/test_action_sync.py b/tests/test_action_sync.py index 872036d..30ed246 100644 --- a/tests/test_action_sync.py +++ b/tests/test_action_sync.py @@ -317,7 +317,7 @@ def test_empty_review_is_successful(self) -> None: ) (fake_core / "diagram_analysis" / "analysis_json.py").write_text( "from types import SimpleNamespace\n" - "def parse_unified_analysis(data): return SimpleNamespace(components=[], components_relations=[]), {}\n" + "def parse_unified_analysis(data): return SimpleNamespace(components=[], components_relations=[], files={}), {}\n" "def build_id_to_name_map(*args): return {}\n", encoding="utf-8", ) diff --git a/tests/test_diff_to_mermaid.py b/tests/test_diff_to_mermaid.py index 0624787..48778e0 100644 --- a/tests/test_diff_to_mermaid.py +++ b/tests/test_diff_to_mermaid.py @@ -6,6 +6,7 @@ import sys import tempfile import unittest +from importlib import metadata from pathlib import Path from types import ModuleType, SimpleNamespace from unittest.mock import patch @@ -27,6 +28,15 @@ def comp(name, files=None, cid=None, subs=None, subrels=None): return c +def core_component(cid, name, files=None): + """A component as Core's parser gives it, with the attributes the loader reads.""" + groups = [ + SimpleNamespace(file_path=path, methods=[SimpleNamespace(qualified_name=name) for name in names]) + for path, names in (files or {}).items() + ] + return SimpleNamespace(component_id=cid, name=name, file_methods=groups) + + def rel(src, dst, label="calls"): return {"src_name": src, "dst_name": dst, "src_id": src, "dst_id": dst, "relation": label} @@ -75,14 +85,12 @@ def test_the_count_rides_on_the_machine_readable_summary(self): class TestDiff(unittest.TestCase): def test_core_loader_projects_global_relation_into_rendered_mermaid(self): - root_a = SimpleNamespace(component_id="1") - root_b = SimpleNamespace(component_id="2") - child_a = SimpleNamespace(component_id="1.1") - child_b = SimpleNamespace(component_id="2.1") + root_a, root_b = core_component("1", "Root A"), core_component("2", "Root B") + child_a, child_b = core_component("1.1", "Child A"), core_component("2.1", "Child B") def parse_unified_analysis(data): relations = [SimpleNamespace(**relation) for relation in data.get("components_relations", [])] - root = SimpleNamespace(components=[root_a, root_b], components_relations=relations) + root = SimpleNamespace(components=[root_a, root_b], components_relations=relations, files={}) return root, { "1": SimpleNamespace(components=[child_a]), "2": SimpleNamespace(components=[child_b]), @@ -124,34 +132,13 @@ def project_relations_to_level(relations, level_ids, id_to_name): "output_generators": ModuleType("output_generators"), "output_generators.rendering": rendering, } - analysis = { - "components": [ - {"name": "Root A", "component_id": "1", "components": [{"name": "Child A", "component_id": "1.1"}]}, - {"name": "Root B", "component_id": "2", "components": [{"name": "Child B", "component_id": "2.1"}]}, - ], - "components_relations": [], - } + relation = {"relation": "calls", "src_name": "Child A", "dst_name": "Child B", "src_id": "1.1", "dst_id": "2.1"} with tempfile.TemporaryDirectory() as tmp, patch.dict(sys.modules, modules): base_path = Path(tmp) / "base.json" head_path = Path(tmp) / "head.json" - base_path.write_text(json.dumps(analysis)) - head_path.write_text( - json.dumps( - { - **analysis, - "components_relations": [ - { - "relation": "calls", - "src_name": "Child A", - "dst_name": "Child B", - "src_id": "1.1", - "dst_id": "2.1", - } - ], - } - ) - ) + base_path.write_text(json.dumps({"components_relations": []})) + head_path.write_text(json.dumps({"components_relations": [relation]})) diff = dm.build_diff(dm.load_analysis(base_path), dm.load_analysis(head_path)) text, meta = dm.render_mermaid(diff) @@ -159,11 +146,11 @@ def project_relations_to_level(relations, level_ids, id_to_name): self.assertEqual(meta["n_edges"], 1) self.assertTrue(meta["changed"]) - def test_projects_global_relations_at_every_component_level(self): - child = SimpleNamespace(component_id="1.1") - peer = SimpleNamespace(component_id="1.2") - root_a = SimpleNamespace(component_id="1") - root_b = SimpleNamespace(component_id="2") + def test_reads_members_hashes_and_relations_from_cores_objects(self): + child = core_component("1.1", "Child", {"a.py": ["a.run"]}) + peer = core_component("1.2", "Peer", {"b.py": ["b.save"]}) + root_a = core_component("1", "Root A", {"a.py": ["a.run"], "b.py": ["b.save"]}) + root_b = core_component("2", "Root B") global_relation = SimpleNamespace( relation="calls", src_name="Child", @@ -178,25 +165,20 @@ def test_projects_global_relations_at_every_component_level(self): src_id="1.1", dst_id="1.2", ) + + def method(name, content_hash): + return SimpleNamespace(qualified_name=name, content_hash=content_hash) + root_analysis = SimpleNamespace( - components=[root_a, root_b], components_relations=[global_relation, sibling_relation] + components=[root_a, root_b], + components_relations=[global_relation, sibling_relation], + files={ + "a.py": SimpleNamespace(content_hash="fa", methods=[method("a.run", "ma")]), + "b.py": SimpleNamespace(content_hash="fb", methods=[method("b.save", "mb")]), + "": SimpleNamespace(content_hash="", methods=[method("loose", "ml")]), + }, ) sub_analysis = SimpleNamespace(components=[child, peer]) - data = { - "components": [ - { - "name": "Root A", - "component_id": "1", - "components": [ - {"name": "Child", "component_id": "1.1"}, - {"name": "Peer", "component_id": "1.2"}, - ], - "components_relations": [{"relation": "stale"}], - }, - {"name": "Root B", "component_id": "2"}, - ], - "components_relations": [], - } def projector(relations, level_ids, _id_to_name): self.assertEqual(relations, [global_relation, sibling_relation]) @@ -214,18 +196,35 @@ def projector(relations, level_ids, _id_to_name): return [sibling_relation] self.fail(f"unexpected component level: {level_ids}") - projected = dm._project_analysis( - data, + view = dm._diff_view( root_analysis, {"1": sub_analysis}, {"1": "Root A", "2": "Root B", "1.1": "Child", "1.2": "Peer"}, projector, ) - self.assertEqual(projected["components_relations"][0]["src_id"], "1") - nested_relations = projected["components"][0]["components_relations"] + self.assertEqual(view["components_relations"][0]["src_id"], "1") + nested_relations = view["components"][0]["components_relations"] self.assertEqual(nested_relations[0]["src_id"], "1.1") self.assertEqual(nested_relations[0]["dst_id"], "1.2") + self.assertEqual( + view["components"][0]["file_methods"], + [{"file_path": "a.py", "methods": ["a.run"]}, {"file_path": "b.py", "methods": ["b.save"]}], + ) + self.assertEqual( + view["components"][0]["components"][1]["file_methods"], [{"file_path": "b.py", "methods": ["b.save"]}] + ) + self.assertNotIn("components", view["components"][1]) + # files[""] holds pathless methods, which still count, but it is no file. + self.assertEqual(view["files"], {"a.py": {"content_hash": "fa"}, "b.py": {"content_hash": "fb"}}) + self.assertEqual( + view["methods_index"], + { + "a.py|a.run": {"file_path": "a.py", "content_hash": "ma"}, + "b.py|b.save": {"file_path": "b.py", "content_hash": "mb"}, + "|loose": {"file_path": "", "content_hash": "ml"}, + }, + ) def test_added_modified_deleted_unchanged(self): base = {"components": [comp("A", {"a.py": ["f"]}), comp("B"), comp("D")], "components_relations": []} @@ -531,5 +530,89 @@ def test_no_edge_labels(self): self.assertNotIn(' -- "', text) +def _core_installed() -> bool: + try: + metadata.version("codeboarding") + except metadata.PackageNotFoundError: + return False + return True + + +@unittest.skipUnless(_core_installed(), "the pinned CodeBoarding release is not installed") +class InstalledCoreTests(unittest.TestCase): + """The loader against the release action.yml pins. It runs in the `core-compatibility` CI job. + + tests/fixtures/analysis holds one analysis as two Core releases wrote it: core-0.15.5.json and + core-9cf50fb9.json (main after #571). Each release reads only its own, so the pinned one must read at + least one, and every one it reads must give the same view. A pin whose release reads none needs that + analysis written by it, which is the point: the diff has not been checked against its parser.""" + + def test_the_pinned_release_reads_an_analysis_it_wrote_into_the_diffs_view(self) -> None: + components_relations = [ + {"relation": "uses", "src_name": "Core", "dst_name": "Also", "src_id": "1", "dst_id": "2"}, + {"relation": "reads", "src_name": "Also", "dst_name": "Core", "src_id": "2", "dst_id": "1"}, + ] + expected = { + "components": [ + { + "name": "Core", + "component_id": "1", + "file_methods": [ + {"file_path": "pkg/a.py", "methods": ["pkg.a.run", "pkg.a.helper"]}, + {"file_path": "pkg/b.py", "methods": ["pkg.b.save", "~Weird"]}, + ], + "components": [ + { + "name": "Run", + "component_id": "1.1", + "file_methods": [{"file_path": "pkg/a.py", "methods": ["pkg.a.run", "pkg.a.helper"]}], + }, + { + "name": "Store", + "component_id": "1.2", + "file_methods": [{"file_path": "pkg/b.py", "methods": ["pkg.b.save", "~Weird"]}], + }, + ], + "components_relations": [ + {"relation": "saves", "src_name": "Run", "dst_name": "Store", "src_id": "1.1", "dst_id": "1.2"} + ], + }, + { + "name": "Also", + "component_id": "2", + "file_methods": [{"file_path": "pkg/c.py", "methods": ["pkg.c.load"]}], + }, + ], + "components_relations": components_relations, + "files": {path: {"content_hash": ""} for path in ("pkg/a.py", "pkg/b.py", "pkg/c.py")}, + "methods_index": { + f"{path}|{name}": {"file_path": path, "content_hash": content_hash} + for path, name, content_hash in ( + ("pkg/a.py", "pkg.a.run", "h1"), + ("pkg/a.py", "pkg.a.helper", "h2"), + ("pkg/b.py", "pkg.b.save", "h3"), + ("pkg/b.py", "~Weird", "h4"), + ("pkg/c.py", "pkg.c.load", "h5"), + ) + }, + } + read = [] + for fixture in sorted((Path(__file__).parent / "fixtures" / "analysis").glob("*.json")): + try: + view = dm.load_analysis(fixture) + except SystemExit as refused: + if not str(refused.code).startswith("::error::Could not parse"): + raise + continue # another release's format + read.append(fixture.name) + with self.subTest(fixture=fixture.name): + self.assertEqual(view, expected) + diff = dm.build_diff(view, view) + self.assertFalse(dm.render_mermaid(diff)[1]["changed"]) + self.assertTrue( + read, "the pinned release reads none of tests/fixtures/analysis; add the analysis as it writes it" + ) + + if __name__ == "__main__": unittest.main()