diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d9f8887..426190d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,6 +42,8 @@ jobs: python -m pip install --disable-pip-version-check "$requirement" - name: Check the provider table against the installed release run: python -m unittest tests.test_provider_table_drift -v + - name: Build side documents with the installed release + run: python -m unittest tests.test_remote_features.InstalledCoreTests -v - name: Install sync artifacts with the installed CodeBoarding release shell: bash env: diff --git a/action.yml b/action.yml index 9009995..3fd9636 100644 --- a/action.yml +++ b/action.yml @@ -736,8 +736,8 @@ runs: # After the artifact holds the analysis and before it is uploaded: each feature is one more # file in it. Uses production by default; an empty URL disables it. It never fails the run. - - name: Run closed features - id: closed_features + - name: Run remote features + id: remote_features if: steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'review' && inputs.review_guide_url != '' continue-on-error: true shell: bash @@ -777,7 +777,7 @@ runs: REQUESTY_BASE_URL: ${{ inputs.requesty_base_url }} VERCEL_API_KEY: ${{ inputs.vercel_api_key }} VERCEL_BASE_URL: ${{ inputs.vercel_base_url }} - run: python3 "$GITHUB_ACTION_PATH/scripts/closed_features.py" + run: python3 "$GITHUB_ACTION_PATH/scripts/remote_features.py" - name: Upload review artifact id: upload_review_artifact_dotcom diff --git a/scripts/closed_features.py b/scripts/closed_features.py deleted file mode 100644 index 29cc3b7..0000000 --- a/scripts/closed_features.py +++ /dev/null @@ -1,438 +0,0 @@ -#!/usr/bin/env python3 -"""Call the review-guide and architecture endpoints independently after the review analysis. - -Both requests carry the same analyses, call graphs, PR metadata and changed files. Only architecture -carries optional source. Store the legacy guide in review_guide.json and the unchanged architecture -envelope in closed/architecture.json, including partial, skipped and failed results. - -Credentials follow the analysis: a run that chose its own provider uses its own key and model for -every feature, and a hosted or licensed run uses CodeBoarding's hosted model. A run whose provider the -service cannot call (a self-hosted endpoint) makes no model call at all: its results report -what they can without one, and it never falls back to the hosted model. Either way the service needs -to know the caller is this repository's run: a fresh GitHub Actions OIDC token (plus the license, the -same bearer the hosted proxy gets), or the run's GITHUB_TOKEN when the workflow cannot mint one. -Nothing here fails the run. - -Environment: CLOSED_FUNCTIONS_URL, REPOSITORY, PR_NUMBER, ARTIFACT_DIR, HEAD_ANALYSIS_PATH, -BASE_ANALYSIS_PATH, CHECKOUT_DIR, GITHUB_TOKEN, GITHUB_API_URL, and optionally LICENSE_KEY, -CLOSED_FEATURES_SOURCE, LLM, MODEL, AGENT_MODEL, _API_KEY, _BASE_URL, -ACTIONS_ID_TOKEN_REQUEST_URL/_TOKEN. -""" - -from __future__ import annotations - -import gzip -import importlib.util -import json -import os -import pickle -import re -import subprocess -import sys -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from urllib.error import HTTPError -from urllib.parse import quote, urlparse -from urllib.request import Request, urlopen - -# A Function URL takes 6 MB, measured after base64, and a gzipped body goes through base64 on the way -# in: 4.5 MB of bundle arrives, and 5 MB is refused with a 413. -MAX_BUNDLE_BYTES = 4_300_000 -# Changed files whose contents are sent at both commits; past this a feature reads diffs only. -MAX_FILES = 300 -# Fixed hosts matching closed_functions/llm.py; never supplied by a remote service. -PROVIDER_HOSTS = { - "ollama": "ollama.com", - "openrouter": "openrouter.ai", - "openai": "api.openai.com", - "anthropic": "api.anthropic.com", - "aws_bedrock": "bedrock-runtime.{region}.amazonaws.com", - "google": "generativelanguage.googleapis.com", - "cerebras": "api.cerebras.ai", - "deepseek": "api.deepseek.com", - "glm": "open.bigmodel.cn", - "kimi": "api.moonshot.cn", - "vercel": "ai-gateway.vercel.sh", - "orcarouter": "api.orcarouter.ai", - "requesty": "router.requesty.ai", - "opper": "api.opper.ai", -} -SOURCE_EXTENSIONS = (".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".cs", ".csproj") -SOURCE_EXCLUDE = re.compile(r"(^|/)(node_modules|dist|build|vendor|__fixtures__|docs|public)/|\.min\.js$|\.d\.ts$") -MAX_SOURCE_FILE_BYTES = 300_000 -MAX_SOURCE_BYTES = 16_000_000 - - -class Skip(Exception): - """A step that did not happen, for a reason worth one line in the log.""" - - -def _github( - path: str, - accept: str = "application/vnd.github+json", - *, - method: str = "GET", - payload=None, - api: str | None = None, - token: str | None = None, - timeout: int = 30, -) -> bytes: - """A GitHub API call: `path` under `api` (default GITHUB_API_URL), or a full URL.""" - api = (api or os.environ.get("GITHUB_API_URL", "https://api.github.com")).rstrip("/") - req = Request( - path if path.startswith("http") else f"{api}{path}", - data=None if payload is None else json.dumps(payload).encode(), - method=method, - headers={ - "Authorization": f"Bearer {token or os.environ['GITHUB_TOKEN']}", - "Accept": accept, - "User-Agent": "codeboarding-action", - }, - ) - with urlopen(req, timeout=timeout) as res: # nosec B310 - GitHub API URL - return res.read() - - -def _error_message(e: HTTPError) -> str: - """The `message` of an error answer's JSON body, or "" when it has none.""" - try: - return str(json.loads(e.read()).get("message") or "") - except (ValueError, AttributeError, OSError): - return "" - - -def _github_json(path: str): - return json.loads(_github(path)) - - -def call_graph(pkl: Path) -> dict: - """A static_analysis.pkl's call graph as the service reads it: nodes and edges with call sites. - The pickle holds CodeBoarding's own classes, which the installed engine provides.""" - with pkl.open("rb") as f: - results = pickle.load(f) # nosec B301 - written by this run's own analysis - nodes, edges = {}, [] - for lang in results.results.values(): - graph = lang.cfg.graph - for name, n in graph.nodes.items(): - nodes[name] = { - "file": n.file_path, - "start": n.line_start, - "end": n.line_end, - "callable": bool(n.is_callable()), - "cb": bool(n.is_callback_or_anonymous()), - } - for e in graph.edges: - sites = [] - for s in vars(e).get("_call_sites") or []: - v = s if isinstance(s, (tuple, list, dict)) else vars(s) if hasattr(s, "__dict__") else str(s) - sites.append(list(v) if isinstance(v, tuple) else v) - edges.append({"s": e.get_source(), "d": e.get_destination(), "sites": sites}) - return {"nodes": nodes, "edges": edges} - - -def _diff(repo: str, base: str, head: str) -> list[dict]: - """The files changed between the two analysed commits, in the shape of a pull request's files - (the compare API returns the same fields), so a feature reads exactly what was analysed.""" - out = _github_json(f"/repos/{repo}/compare/{base}...{head}") - return [ - { - k: f.get(k) - for k in ("filename", "status", "additions", "deletions", "changes", "patch", "previous_filename") - if k in f - } - for f in out.get("files") or [] - ] - - -def _contents(repo: str, sha: str, path: str) -> str | None: - try: - raw = _github(f"/repos/{repo}/contents/{quote(path)}?ref={sha}", accept="application/vnd.github.raw") - except HTTPError: - return None - try: - return raw.decode("utf-8") - except UnicodeDecodeError: - return None - - -def collect(env: dict) -> dict: - """The required inputs shared by both endpoints.""" - repo, pr = env["REPOSITORY"], int(env["PR_NUMBER"]) - artifact = Path(env["ARTIFACT_DIR"]) - head_dir, base_dir = Path(env["HEAD_ANALYSIS_PATH"]).parent, Path(env["BASE_ANALYSIS_PATH"]).parent - meta = json.loads((artifact / "metadata.json").read_text()) - head_sha = meta["head_sha"] - # The analysis compared against the merge base, so that is the base every feature reads too. - base_sha = meta.get("merge_base_sha") or meta.get("base_sha") - for p in (head_dir / "static_analysis.pkl", base_dir / "static_analysis.pkl"): - if not p.is_file(): - raise Skip(f"no call graph at {p}") - - pull = _github_json(f"/repos/{repo}/pulls/{pr}") - commits = _github_json(f"/repos/{repo}/pulls/{pr}/commits?per_page=100") - pr_files = _diff(repo, base_sha, head_sha) - checkout = Path(env["CHECKOUT_DIR"]) - files: dict[str, str] = {} - for f in pr_files[:MAX_FILES]: - if f.get("status") != "removed": - try: - p = _checkout_file(checkout, f["filename"]) - files[f"{head_sha}:{f['filename']}"] = p.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - pass - if f.get("status") != "added": - was = f.get("previous_filename") or f["filename"] - text = _contents(repo, base_sha, was) - if text is not None: - files[f"{base_sha}:{was}"] = text - - bundle_meta = {**meta, "base_sha": base_sha, "head_sha": head_sha, "pr_number": str(pr)} - inputs = { - "analysis.json": Path(env["HEAD_ANALYSIS_PATH"]).read_text(), - "base/analysis.json": Path(env["BASE_ANALYSIS_PATH"]).read_text(), - "metadata.json": json.dumps(bundle_meta), - "pr.json": json.dumps( - { - "title": pull.get("title") or "", - "body": pull.get("body") or "", - "headRefOid": head_sha, - "commits": [(c.get("commit") or {}).get("message", "").split("\n")[0] for c in commits], - } - ), - "pr-files.json": json.dumps(pr_files), - "head-cg.json": json.dumps(call_graph(head_dir / "static_analysis.pkl")), - "base-cg.json": json.dumps(call_graph(base_dir / "static_analysis.pkl")), - "files.json": json.dumps(files), - } - return inputs - - -def _load(name: str, path: Path): - spec = importlib.util.spec_from_file_location(name, path) - assert spec and spec.loader - mod = importlib.util.module_from_spec(spec) - sys.modules.setdefault(spec.name, mod) # its dataclass looks its own module up while it is defined - spec.loader.exec_module(sys.modules[spec.name]) - return sys.modules[spec.name] - - -def _relay(): - return _load("oidc_relay", Path(__file__).with_name("oidc_relay.py")) - - -def _api_key(env: dict, llm: str) -> str: - """The key as the analysis read it, so a pasted `VAR="..."` that worked there works here too.""" - check = _load("credential_check", Path(__file__).parent / "action" / "credential_check.py") - inputs = check.load_table()["providers"].get(llm, {}).get("inputs", {}) - env_var = inputs.get(f"{llm}_api_key", f"{llm.upper()}_API_KEY") - return check._clean_key(env.get(f"{llm.upper()}_API_KEY", ""), env_var) - - -def model_headers(env: dict) -> dict[str, str]: - """What tells the service which model this run uses. Nothing for a hosted or licensed run, which - use CodeBoarding's model and settings as they stand. A run's own provider sends its key and model; one the service cannot reach sends its name alone, which - the service reads as "no model here", so a run that chose its own provider is never sent to ours. - A base URL elsewhere is a provider the service cannot reach: it calls fixed hosts only.""" - llm = env.get("LLM", "").strip().lower() - if llm in ("", "hosted", "license"): - return {} - key = _api_key(env, llm) - base = env.get(f"{llm.upper()}_BASE_URL", "").strip() - host = PROVIDER_HOSTS.get(llm) - if host and llm == "aws_bedrock": - host = host.format(region=env.get("AWS_BEDROCK_REGION", "").strip()) - if not key or not host or (base and urlparse(base).hostname != host): - return {"x-llm-provider": llm} - out = {"x-llm-api-key": key, "x-llm-provider": llm} - if env.get(f"{llm.upper()}_REGION", "").strip(): - out["x-llm-region"] = env[f"{llm.upper()}_REGION"].strip() - # The analysis's own order, so a run's closed features use its analysis model. - model = env.get("AGENT_MODEL", "").strip() or env.get("MODEL", "").strip() - if model: - out["x-llm-model"] = model - return out - - -def _bearer(env: dict) -> dict[str, str]: - url, token = env.get("ACTIONS_ID_TOKEN_REQUEST_URL", ""), env.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "") - if not url or not token: - # Without `id-token: write` the run's own token proves the same thing: it can read this - # repository. It carries no license, so the owner's free bucket pays. - return {"Authorization": f"Bearer {env['GITHUB_TOKEN']}"} - relay = _relay() - license_file = None - if env.get("LICENSE_KEY", "").strip(): - license_file = Path(env.get("RUNNER_TEMP", "/tmp")) / "codeboarding-guide-license.txt" - license_file.write_text(env["LICENSE_KEY"].strip()) - try: - config = relay.RelayConfig( - upstream_base_url="", id_token_request_url=url, id_token_request_token=token, license_file=license_file - ) - return {"Authorization": relay._authorization(config)} - finally: - if license_file is not None: - license_file.unlink(missing_ok=True) - - -def _checkout_file(checkout: Path, name: str) -> Path: - """Reject traversal and symlinks (including parent directories) before reading source.""" - relative = Path(name) - if relative.is_absolute() or ".." in relative.parts: - raise OSError("path outside checkout") - path = checkout - for part in relative.parts: - path = path / part - if path.is_symlink(): - raise OSError("symlink in checkout") - return path - - -def collect_source(env: dict) -> tuple[str | None, str | None]: - """Tracked architecture source, within fixed size and extension limits, or an omission reason.""" - if env.get("CLOSED_FEATURES_SOURCE", "true").strip().lower() == "false": - return None, "turned off by the workflow" - checkout = Path(env["CHECKOUT_DIR"]) - listed = subprocess.run(["git", "-C", str(checkout), "ls-files", "-z"], capture_output=True, check=True).stdout - total = 0 - files: dict[str, str] = {} - for name in listed.decode("utf-8", "replace").split("\0"): - if not name or not name.endswith(SOURCE_EXTENSIONS) or SOURCE_EXCLUDE.search(name): - continue - try: - path = _checkout_file(checkout, name) - size = path.stat().st_size - if size > MAX_SOURCE_FILE_BYTES or not path.is_file(): - continue - files[name] = path.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - continue - total += size - if total > MAX_SOURCE_BYTES: - return None, f"its source is over the {MAX_SOURCE_BYTES / 1e6:.0f} MB limit" - return json.dumps(files), None - - -def _no_architecture(ctx: dict, status: str, reason: str) -> dict: - """Preserve the architecture envelope even when the request cannot complete.""" - return { - "feature": "architecture", - "schema": 1, - "status": status, - "repo": ctx["repo"], - "pr": ctx["pr"], - "headSha": ctx["head"], - "generator": None, - "coverage": None, - "data": None, - "reason": reason, - } - - -def _post(url: str, body: bytes, headers: dict[str, str], timeout: int = 600) -> dict: - req = Request( - url, - data=body, - headers={**headers, "Content-Type": "application/gzip", "User-Agent": "codeboarding-action"}, - method="POST", - ) - try: - with urlopen(req, timeout=timeout) as res: # nosec B310 - service URL from workflow configuration - return json.loads(res.read()) - except HTTPError as e: - reason = f"the service answered {e.code}: {_error_message(e) or e.reason}" - if e.code == 402: - raise Skip(reason) from e - raise RuntimeError(reason) from e - - -def _gzip(body: dict) -> bytes: - return gzip.compress(json.dumps(body).encode(), compresslevel=9) - - -def _pack(ctx: dict, files: dict[str, str], omitted: dict[str, str]) -> bytes: - body = {"version": 1, "repo": ctx["repo"], "pr": ctx["pr"], "files": files} - if omitted: - body["omitted"] = omitted - return _gzip(body) - - -def _check_size(data: bytes, what: str = "bundle") -> None: - if len(data) > MAX_BUNDLE_BYTES: - raise Skip(f"its {what} is {len(data) / 1e6:.1f} MB, over the service's limit") - - -def run_review_guide(base: str, ctx: dict, inputs: dict, headers: dict, artifact: Path) -> None: - try: - if "x-llm-provider" in headers and "x-llm-api-key" not in headers: - raise Skip("the service cannot call this run's provider") - data = _pack(ctx, inputs, {}) - _check_size(data) - answer = _post(f"{base.rstrip('/')}/v1/review-guide", data, headers) - guide = answer.get("guide") - if not isinstance(guide, dict): - raise ValueError("the service returned no guide") - (artifact / "review_guide.json").write_text(json.dumps(guide, ensure_ascii=False, indent=1)) - except Exception as e: - print(f"::notice::review-guide {'skipped' if isinstance(e, Skip) else 'failed'}: {e}") - - -def run_architecture(base: str, ctx: dict, inputs: dict, headers: dict, env: dict) -> dict: - try: - try: - source, why = collect_source(env) - except Exception as e: - source, why = None, f"reading it failed: {type(e).__name__}" - files, omitted = dict(inputs), {} - if source is None: - omitted["source.json"] = why or "the source was not collected" - else: - files["source.json"] = source - data = _pack(ctx, files, omitted) - if "source.json" in files and len(data) > MAX_BUNDLE_BYTES: - del files["source.json"] - omitted["source.json"] = "the repository is too large to send" - data = _pack(ctx, files, omitted) - _check_size(data) - answer = _post(f"{base.rstrip('/')}/v1/architecture", data, headers) - envelope = answer.get("envelope") - if not isinstance(envelope, dict) or envelope.get("status") not in ("ok", "partial", "skipped", "failed"): - raise ValueError("the service returned no architecture result") - return envelope - except Exception as e: - return _no_architecture(ctx, "skipped" if isinstance(e, Skip) else "failed", str(e)) - - -def main(env: dict | None = None) -> int: - env = dict(os.environ if env is None else env) - try: - artifact = Path(env["ARTIFACT_DIR"]) - meta = json.loads((artifact / "metadata.json").read_text()) - ctx = {"repo": env["REPOSITORY"], "pr": int(env["PR_NUMBER"]), "head": meta.get("head_sha")} - base = env["CLOSED_FUNCTIONS_URL"] - if not base.strip(): - return 0 - try: - headers = {**_bearer(env), **model_headers(env)} - inputs = collect(env) - except Exception as e: - reason = f"collecting inputs or credentials failed: {type(e).__name__}: {e}" - print(f"::notice::review-guide failed: {reason}") - architecture = _no_architecture(ctx, "failed", reason) - else: - with ThreadPoolExecutor(max_workers=2) as pool: - guide = pool.submit(run_review_guide, base, ctx, inputs, headers, artifact) - result = pool.submit(run_architecture, base, ctx, inputs, headers, env) - architecture = result.result() - guide.result() - path = artifact / "closed" / "architecture.json" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(architecture, ensure_ascii=False, indent=1)) - if architecture["status"] not in ("ok", "partial"): - print(f"::notice::architecture {architecture['status']}: {architecture.get('reason')}") - except Exception as e: # never fail the review over a closed feature - print(f"::warning::Running the closed features failed: {type(e).__name__}: {e}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/problem_history.py b/scripts/problem_history.py index 96d0503..fea3bc1 100644 --- a/scripts/problem_history.py +++ b/scripts/problem_history.py @@ -31,14 +31,14 @@ from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen sys.path.insert(0, str(Path(__file__).resolve().parent)) -from closed_features import ( - MAX_BUNDLE_BYTES, +from remote_features import ( + MAX_BRIEFING_JSON_BYTES, Skip, + TooLarge, _bearer, - _check_size, + _encode, _error_message, _github, - _gzip, _post, ) # noqa: E402 @@ -71,6 +71,8 @@ # The review artifact's retention-days in action.yml; an unread merge older than this has nothing left to find. REVIEW_RETENTION_DAYS = 14 MAX_BODY_CHARS = 2000 +# Compressions of one request at most: the first, and retries aimed by the ratio the one before measured. +MAX_COMPRESSIONS = 3 SERVICE_TIMEOUT = 300 # A composite action's step cannot set timeout-minutes, so the script stops itself at the step's budget. STEP_LIMIT_SECONDS = 600 @@ -685,17 +687,27 @@ def build_request(env: dict, ledger: dict | None, facts: dict) -> dict: def pack(body: dict) -> tuple[bytes, list[str]]: - """The gzipped request and the pull requests whose findings were left out of it: the largest - envelopes go first while it is over the service's limit, and their merges stay unread for a later run.""" - findings, dropped = dict(body.get("findings") or {}), [] - while True: - data = _gzip({**body, "findings": findings}) - if len(data) <= MAX_BUNDLE_BYTES or not findings: - _check_size(data, "request") - return data, dropped - largest = max(findings, key=lambda pr: len(json.dumps(findings[pr]))) - del findings[largest] - dropped.append(largest) + """The compressed request and the pull requests whose findings were left out of it, largest first; + their merges stay unread for a later run. Findings are left out until the JSON is under the service's + limit, counted per entry rather than serialised again, and then it is compressed once. Only when that + is over the limit on what is sent is it compressed again, down to a little under what the ratio just + measured says fits, at most MAX_COMPRESSIONS times in all.""" + findings = body.get("findings") or {} + # An entry's bytes in the request: `"pr": {...}` and its separator, exact while one entry is left. + cost = {pr: len(json.dumps({pr: envelope})) for pr, envelope in findings.items()} + kept, dropped = sorted(findings, key=cost.get), [] + size, budget = len(json.dumps({**body, "findings": findings})), MAX_BRIEFING_JSON_BYTES + for compressions in range(1, MAX_COMPRESSIONS + 1): + while kept and size > budget: + dropped.append(kept.pop()) + size -= cost[dropped[-1]] + request = {**body, "findings": {pr: findings[pr] for pr in findings if pr not in dropped}} + try: + return _encode(request, MAX_BRIEFING_JSON_BYTES, "request"), dropped + except TooLarge as e: + if not kept or e.fits is None or compressions == MAX_COMPRESSIONS: + raise + budget = e.fits * 9 // 10 def _newest_pr(*lists) -> int | None: diff --git a/scripts/remote_features.py b/scripts/remote_features.py new file mode 100644 index 0000000..fd5fc6e --- /dev/null +++ b/scripts/remote_features.py @@ -0,0 +1,738 @@ +#!/usr/bin/env python3 +"""Ask the CodeBoarding service for a pull request's review guide and architecture checks. + +Runs after the review analysis. The two requests go out in parallel and independently: either one +failing leaves the other alone, and nothing here fails the run. The guide is stored as +review_guide.json and the architecture envelope as closed/architecture.json in the review artifact, +including partial, skipped and failed results. + +What a request sends +-------------------- +One JSON object of named files, compressed with xz. Both requests send the same files, and the same +bytes unless architecture's carries source: + +- head.json and base.json: one side document per commit (see side_document). It holds what the + service reads from that commit's analysis.json and call graph (static_analysis.pkl), each fact once: + the methods and graph nodes as one table per file, the calls, the component tree, and the few + analysis edges that are no call. Analyses are read through the installed engine's parser, never as a + document, so whatever format the pinned release writes is read. +- metadata.json: the review artifact's metadata, with the merge base as the base. +- pr.json: the pull request's title, body, head commit and commit subjects. +- pr-files.json: the files changed between the two analysed commits, as GitHub's compare API lists them. +- files.json: those files' contents at both commits, keyed `:`. + +Architecture also sends source.json, the repository's tracked source, unless the workflow turns it off +or it does not fit. + +Why it is packed this way +------------------------- +On a large repository such as crate, the two analyses and two call graphs are about 680 MB of JSON. +Most of it repeats itself: the analysis edges are the calls again, the analysis and the graph each list +every method, and the head is nearly the base. A side document writes each fact once (crate: 7 MB per +side). xz then sees the base repeat the head, because its window is 64 MB where gzip's is 32 KB: +crate's whole request is 1.4 MB. + +Size limits +----------- +- The service sets two limits on every request, and _encode checks both before anything is sent: the + JSON, measured first because that is cheap, against MAX_FEATURE_JSON_BYTES (MAX_BRIEFING_JSON_BYTES + for the problem history), then the xz once against MAX_COMPRESSED_BYTES. The JSON limit is 62 times + the compressed one, so the cheap check already turns away whatever could only fit at better than 62:1. + These bundles compress about 10:1, but no tighter cut-off holds for every request. +- xz at this preset takes about 700 MB and seconds per 10 MB, so a run compresses one body at a time and + each at most once: the body without source, which both features send, and architecture's with source + when there is source. When that one does not fit, architecture sends the shared one. +- MAX_FILES: how many changed files have their contents sent; past it a feature reads diffs only. +- MAX_FILE_BYTES: a larger file is sent neither in files.json nor in source.json. +- MAX_SOURCE_BYTES: the most source.json holds; past it source is not sent. + +Credentials +----------- +Credentials follow the analysis. A run that chose its own provider uses its own key and model for every +feature, and a hosted or licensed run uses CodeBoarding's hosted model. A run whose provider the service +cannot call (a self-hosted endpoint) makes no model call at all: its results report what they can +without one, and it never falls back to the hosted model. Either way the service needs to know the +caller is this repository's run: a fresh GitHub Actions OIDC token (plus the license, the same bearer the +hosted proxy gets), or the run's GITHUB_TOKEN when the workflow cannot mint one. + +Environment: CLOSED_FUNCTIONS_URL, REPOSITORY, PR_NUMBER, ARTIFACT_DIR, HEAD_ANALYSIS_PATH, +BASE_ANALYSIS_PATH, CHECKOUT_DIR, GITHUB_TOKEN, GITHUB_API_URL, and optionally LICENSE_KEY, +CLOSED_FEATURES_SOURCE, LLM, MODEL, AGENT_MODEL, _API_KEY, _BASE_URL, +ACTIONS_ID_TOKEN_REQUEST_URL/_TOKEN. +""" + +from __future__ import annotations + +import importlib.util +import json +import lzma +import os +import pickle +import re +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from urllib.error import HTTPError +from urllib.parse import quote, urlparse +from urllib.request import Request, urlopen + +# The service's limits on a request, which move with it. A Function URL takes 6 MB, measured after +# base64, and a compressed body goes through base64 on the way in: 4.5 MB of it arrives, and 5 MB is +# refused with a 413. Decompressed, the service refuses a feature request over 256 MiB and a problem +# history request over 32 MiB: MAX_BUNDLE_BYTES and MAX_BRIEFING_BYTES in licensing-aws +# closed_functions/handler.py. +MAX_COMPRESSED_BYTES = 4_300_000 +MAX_FEATURE_JSON_BYTES = 256 * 1024 * 1024 +MAX_BRIEFING_JSON_BYTES = 32 * 1024 * 1024 +# Changed files whose contents are sent at both commits; past this a feature reads diffs only. +MAX_FILES = 300 +# A file larger than this is sent neither as a changed file's contents nor as source. +MAX_FILE_BYTES = 300_000 +# Fixed hosts matching closed_functions/llm.py; never supplied by a remote service. +PROVIDER_HOSTS = { + "ollama": "ollama.com", + "openrouter": "openrouter.ai", + "openai": "api.openai.com", + "anthropic": "api.anthropic.com", + "aws_bedrock": "bedrock-runtime.{region}.amazonaws.com", + "google": "generativelanguage.googleapis.com", + "cerebras": "api.cerebras.ai", + "deepseek": "api.deepseek.com", + "glm": "open.bigmodel.cn", + "kimi": "api.moonshot.cn", + "vercel": "ai-gateway.vercel.sh", + "orcarouter": "api.orcarouter.ai", + "requesty": "router.requesty.ai", + "opper": "api.opper.ai", +} +SOURCE_EXTENSIONS = (".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".cs", ".csproj") +SOURCE_EXCLUDE = re.compile(r"(^|/)(node_modules|dist|build|vendor|__fixtures__|docs|public)/|\.min\.js$|\.d\.ts$") +MAX_SOURCE_BYTES = 16_000_000 + + +class Skip(Exception): + """A step that did not happen, for a reason worth one line in the log.""" + + +class TooLarge(Skip): + """A request over one of the service's limits. Over the compressed one, `fits` is how much JSON the + compression ratio just measured would bring under it.""" + + def __init__(self, message: str, fits: int | None = None): + super().__init__(message) + self.fits = fits + + +# --- GitHub --------------------------------------------------------------------------------------- + + +def _github( + path: str, + accept: str = "application/vnd.github+json", + *, + method: str = "GET", + payload=None, + api: str | None = None, + token: str | None = None, + timeout: int = 30, +) -> bytes: + """A GitHub API call: `path` under `api` (default GITHUB_API_URL), or a full URL.""" + api = (api or os.environ.get("GITHUB_API_URL", "https://api.github.com")).rstrip("/") + request = Request( + path if path.startswith("http") else f"{api}{path}", + data=None if payload is None else json.dumps(payload).encode(), + method=method, + headers={ + "Authorization": f"Bearer {token or os.environ['GITHUB_TOKEN']}", + "Accept": accept, + "User-Agent": "codeboarding-action", + }, + ) + with urlopen(request, timeout=timeout) as response: # nosec B310 - GitHub API URL + return response.read() + + +def _error_message(error: HTTPError) -> str: + """The `message` of an error answer's JSON body, or "" when it has none.""" + try: + return str(json.loads(error.read()).get("message") or "") + except (ValueError, AttributeError, OSError): + return "" + + +def _github_json(path: str): + return json.loads(_github(path)) + + +def _diff(repo: str, base_sha: str, head_sha: str) -> list[dict]: + """The files changed between the two analysed commits, in the shape of a pull request's files + (the compare API returns the same fields), so a feature reads exactly what was analysed.""" + compared = _github_json(f"/repos/{repo}/compare/{base_sha}...{head_sha}") + fields = ("filename", "status", "additions", "deletions", "changes", "patch", "previous_filename") + return [{field: changed[field] for field in fields if field in changed} for changed in compared.get("files") or []] + + +def _contents(repo: str, sha: str, path: str) -> str | None: + """A file's text at a commit, or None when GitHub has no such file or it is not UTF-8.""" + try: + raw = _github(f"/repos/{repo}/contents/{quote(path)}?ref={sha}", accept="application/vnd.github.raw") + except HTTPError: + return None + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + return None + + +# --- Side documents ------------------------------------------------------------------------------- + + +def call_graph(pickle_path: Path) -> dict: + """A static_analysis.pkl's call graph as the service reads it: each node's file, span and whether + it is a callback, and each call's site lines. A site's file is always its caller's and no reader + takes the column, so neither is kept. The pickle holds CodeBoarding's own classes, which the + installed engine provides.""" + with pickle_path.open("rb") as f: + results = pickle.load(f) # nosec B301 - written by this run's own analysis + nodes, edges = {}, [] + for language in results.results.values(): + graph = language.cfg.graph + for name, node in graph.nodes.items(): + nodes[name] = { + "file": node.file_path, + "start": node.line_start, + "end": node.line_end, + "cb": bool(node.is_callback_or_anonymous()), + } + for edge in graph.edges: + edges.append({"s": edge.get_source(), "d": edge.get_destination(), "lines": _site_lines(edge)}) + return {"nodes": nodes, "edges": edges} + + +def _site_lines(edge) -> list: + """The line of each place the call is made; a site may be a dict or an object.""" + sites = vars(edge).get("_call_sites") or [] + return [site.get("line") if isinstance(site, dict) else getattr(site, "line", None) for site in sites] + + +@dataclass(eq=False) +class _Row: + """One row of a side document's file table: a method of the analysis, a call-graph node that is no + method, or a method that is also a node.""" + + name: str + start: int | None + end: int | None + method: tuple | None # (type, content_hash) for a method of the analysis + callback: int = -1 # 1 or 0 once a graph node shares the row; -1 for a method that is no node + number: int = -1 # the row's position across all files, set by _Table.number_rows + + def written(self, path: str) -> list: + name = _short_name(self.name, path) + if self.method is None: + return [name, self.start, self.end, self.callback] + return [name, self.start, self.end, *self.method, self.callback] + + +def _short_name(name: str, path: str) -> str: + """A qualified name as a row writes it: `~.rest` when it is the file's dotted path plus `.rest`, + and a leading `~` doubled so a name of its own that starts with one reads back unchanged.""" + stem = path.rsplit(".", 1)[0].replace("/", ".") + if name.startswith(stem + "."): + return "~" + name[len(stem) :] + return "~" + name if name.startswith("~") else name + + +class _Table: + """The rows of one side, per file: every method of the analysis, then every call-graph node, which + shares the row of a method with its name.""" + + def __init__(self, files: dict): + self.rows_by_file: dict[str, list[_Row]] = {} + self.row_of_key: dict[str, _Row] = {} + self.methods_named: dict[str, list[tuple[str, _Row]]] = {} # qualified name -> (file, row) + self.row_of_node: dict[str, _Row] = {} + self.row_count = 0 + for path, entry in files.items(): + for method in entry.methods: + self._add_method(path, method) + + def _add_method(self, path: str, method) -> None: + name = method.qualified_name + row = _Row(name, start=method.start_line, end=method.end_line, method=(method.node_type, method.content_hash)) + self.rows_by_file.setdefault(path, []).append(row) + self.row_of_key[f"{path}|{name}"] = row + self.methods_named.setdefault(name, []).append((path, row)) + + def add_node(self, name: str, node: dict) -> _Row: + """The row of the method with the node's name, preferring the one in the node's file, or a new + row for a node that is no method.""" + methods = self.methods_named.get(name) or [] + row = next((row for path, row in methods if path == node.get("file")), methods[0][1] if methods else None) + if row is None: + row = _Row(name, start=node.get("start"), end=node.get("end"), method=None) + self.rows_by_file.setdefault(node.get("file") or "", []).append(row) + row.callback = int(node["cb"]) + self.row_of_node[name] = row + return row + + def number_rows(self) -> list: + """Number the rows, files in path order and each file's rows by start line, and return the + files as the document writes them: `[path, [row, ...]]`.""" + written = [] + for path in sorted(self.rows_by_file): + rows = sorted(self.rows_by_file[path], key=lambda row: (row.start is None, row.start or 0, row.name)) + for row in rows: + row.number = self.row_count + self.row_count += 1 + written.append([path, [row.written(path) for row in rows]]) + return written + + def member(self, key: str) -> int | str: + """A method's `path|qualified_name` key as a component member or edge end: its row number, or + the key itself when the analysis lacks the method.""" + row = self.row_of_key.get(key) + return key if row is None else row.number + + +def side_document(root, sub_analyses: dict, graph: dict) -> dict: + """One commit's analysis, as Core parses it (the root and each component's sub-analysis by id), and + its call graph as the service reads them, each fact once. The service's bundle.expand_side turns it + back into both, in every field a feature reads. + + - files: per file, rows for the analysis's methods `[name, start, end, type, content_hash, cb]` + and for call-graph nodes that are no method `[name, start, end, cb]`. A method and the graph node of + the same name share a row; cb is -1 for a method that is no graph node. A name under its file's + dotted path is written `~.rest`. Rows are numbered in order across files; `unplaced` continues the + numbering for call ends that are no node. + - graph_spans: `[row, start, end]` for a node whose span is not its method's. + - calls: per caller, `[callee, line, ...]`, one line per call site. + - components: `[id, name, description, parent, members, relations]` in tree order. Members are rows + (a key where the analysis lacks the method), or null for a parent holding exactly its children's. + - extra_edges: the analysis's edges that are no call. The rest repeat the calls: on crate one of + 149,316 is not a call.""" + table = _Table(root.files) + spans = [] + for name, node in (graph.get("nodes") or {}).items(): + row = table.add_node(name, node) + if (row.start, row.end) != (node.get("start"), node.get("end")): + spans.append((row, node.get("start"), node.get("end"))) + files = table.number_rows() + calls, unplaced = _calls(graph.get("edges") or [], table) + called = {(caller, callee) for caller, callees in calls for callee, *_ in callees} + edges = {(table.member(source), table.member(target)) for source, target in _analysis_edges(root, sub_analyses)} + return { + "version": 1, + "files": files, + "unplaced": unplaced, + "graph_spans": [[row.number, start, end] for row, start, end in spans], + "components": _components(root, sub_analyses, table), + "relations": _relations(root.components_relations), + "calls": calls, + "extra_edges": sorted(edges - called, key=repr), + } + + +def _calls(edges: list[dict], table: _Table) -> tuple[list, list[str]]: + """The calls grouped by caller, `[caller, [[callee, line, ...], ...]]` in row order, and the call + ends that are no node, numbered after the last row.""" + number_of = {name: row.number for name, row in table.row_of_node.items()} + unplaced: list[str] = [] + by_caller: dict[int, list] = {} + for edge in edges: + for name in (edge["s"], edge["d"]): + if name not in number_of: + number_of[name] = table.row_count + len(unplaced) + unplaced.append(name) + by_caller.setdefault(number_of[edge["s"]], []).append([number_of[edge["d"]], *edge["lines"]]) + return [[caller, by_caller[caller]] for caller in sorted(by_caller)], unplaced + + +def _components(root, sub_analyses: dict, table: _Table) -> list: + """The component tree in pre-order, each `[id, name, description, parent, members, relations]`. + A parent's members are null when they are exactly its children's, which the service rebuilds.""" + flat = _flatten(root, sub_analyses) + children: list[list[int]] = [[] for _ in flat] + for position, (_, parent, _) in enumerate(flat): + if parent >= 0: + children[parent].append(position) + members = [ + [ + table.member(f"{group.file_path}|{method.qualified_name}") + for group in component.file_methods + for method in group.methods + ] + for component, _, _ in flat + ] + written = [] + for position, (component, parent, relations) in enumerate(flat): + from_children = set().union(*(members[child] for child in children[position])) + implied = children[position] and set(members[position]) == from_children + written.append( + [ + component.component_id, + component.name, + component.description, + parent, + None if implied else members[position], + _relations(relations), + ] + ) + return written + + +def _flatten(root, sub_analyses: dict) -> list[tuple[object, int, list]]: + """The component tree in pre-order, each component with its parent's position (-1 at the top) and + the relations among its children.""" + flat: list[tuple[object, int, list]] = [] + + def visit(analysis, parent: int) -> None: + for component in analysis.components: + sub_analysis = sub_analyses.get(component.component_id) + flat.append((component, parent, sub_analysis.components_relations if sub_analysis else [])) + if sub_analysis is not None: + visit(sub_analysis, len(flat) - 1) + + visit(root, -1) + return flat + + +def _relations(relations: list) -> list: + """Each relation as `[src_id, dst_id, label]`.""" + return [[relation.src_id, relation.dst_id, relation.relation] for relation in relations] + + +def _analysis_edges(root, sub_analyses: dict): + """Every method edge of every relation in the analysis, top level and nested, as `path|qualified_name` + keys.""" + for analysis in (root, *sub_analyses.values()): + for relation in analysis.components_relations: + for edge in relation.all_edges: + yield _method_key(edge.source), _method_key(edge.target) + + +def _method_key(reference) -> str: + return f"{reference.reference_file or ''}|{reference.qualified_name}" + + +# --- Collecting the inputs ------------------------------------------------------------------------ + + +def collect(env: dict) -> dict: + """The files both requests send, named as the service reads them.""" + repo, pr = env["REPOSITORY"], int(env["PR_NUMBER"]) + head_analysis, base_analysis = Path(env["HEAD_ANALYSIS_PATH"]), Path(env["BASE_ANALYSIS_PATH"]) + metadata = json.loads((Path(env["ARTIFACT_DIR"]) / "metadata.json").read_text()) + head_sha = metadata["head_sha"] + # The analysis compared against the merge base, so that is the base every feature reads too. + base_sha = metadata.get("merge_base_sha") or metadata.get("base_sha") + for analysis_path in (head_analysis, base_analysis): + pickle_path = analysis_path.parent / "static_analysis.pkl" + if not pickle_path.is_file(): + raise Skip(f"no call graph at {pickle_path}") + + pull = _github_json(f"/repos/{repo}/pulls/{pr}") + commits = _github_json(f"/repos/{repo}/pulls/{pr}/commits?per_page=100") + changed = _diff(repo, base_sha, head_sha) + contents = _changed_contents(repo, changed, Path(env["CHECKOUT_DIR"]), head_sha, base_sha) + return { + "head.json": _side(head_analysis), + "base.json": _side(base_analysis), + "metadata.json": json.dumps({**metadata, "base_sha": base_sha, "head_sha": head_sha, "pr_number": str(pr)}), + "pr.json": json.dumps( + { + "title": pull.get("title") or "", + "body": pull.get("body") or "", + "headRefOid": head_sha, + "commits": [(commit.get("commit") or {}).get("message", "").split("\n")[0] for commit in commits], + } + ), + "pr-files.json": json.dumps(changed), + "files.json": json.dumps(contents), + } + + +def _side(analysis_path: Path) -> str: + """The side document of the commit whose analysis this is; its call graph lies beside it.""" + root, sub_analyses = read_analysis(analysis_path) + graph = call_graph(analysis_path.parent / "static_analysis.pkl") + return json.dumps(side_document(root, sub_analyses, graph), separators=(",", ":")) + + +def read_analysis(analysis_path: Path) -> tuple: + """The analysis as the installed engine parses it: the root and each component's sub-analysis by id. + The run analysed both commits with that engine, so both are in the format it reads.""" + from diagram_analysis.analysis_json import parse_unified_analysis + + return parse_unified_analysis(json.loads(analysis_path.read_text())) + + +def _changed_contents(repo: str, changed: list[dict], checkout: Path, head_sha: str, base_sha: str) -> dict: + """The first MAX_FILES changed files at both commits, keyed `:`: the head from the + checkout, the base from GitHub. A file over MAX_FILE_BYTES or not UTF-8 is left out.""" + contents: dict[str, str] = {} + for change in changed[:MAX_FILES]: + if change.get("status") != "removed": + text = _checkout_text(checkout, change["filename"]) + if text is not None: + contents[f"{head_sha}:{change['filename']}"] = text + if change.get("status") != "added": + base_name = change.get("previous_filename") or change["filename"] + text = _contents(repo, base_sha, base_name) + if text is not None and len(text.encode()) <= MAX_FILE_BYTES: + contents[f"{base_sha}:{base_name}"] = text + return contents + + +def _checkout_text(checkout: Path, name: str) -> str | None: + """A checked-out file's text, or None when it is unsafe to read, over MAX_FILE_BYTES or not UTF-8.""" + try: + path = _checkout_file(checkout, name) + if path.stat().st_size <= MAX_FILE_BYTES: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + pass + return None + + +def _checkout_file(checkout: Path, name: str) -> Path: + """Reject traversal and symlinks (including parent directories) before reading source.""" + relative = Path(name) + if relative.is_absolute() or ".." in relative.parts: + raise OSError("path outside checkout") + path = checkout + for part in relative.parts: + path = path / part + if path.is_symlink(): + raise OSError("symlink in checkout") + return path + + +def collect_source(env: dict) -> tuple[str | None, str | None]: + """Tracked architecture source, within fixed size and extension limits, or an omission reason.""" + if env.get("CLOSED_FEATURES_SOURCE", "true").strip().lower() == "false": + return None, "turned off by the workflow" + checkout = Path(env["CHECKOUT_DIR"]) + listed = subprocess.run(["git", "-C", str(checkout), "ls-files", "-z"], capture_output=True, check=True).stdout + total = 0 + files: dict[str, str] = {} + for name in listed.decode("utf-8", "replace").split("\0"): + if not name or not name.endswith(SOURCE_EXTENSIONS) or SOURCE_EXCLUDE.search(name): + continue + try: + path = _checkout_file(checkout, name) + size = path.stat().st_size + if size > MAX_FILE_BYTES or not path.is_file(): + continue + files[name] = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + total += size + if total > MAX_SOURCE_BYTES: + return None, f"its source is over the {MAX_SOURCE_BYTES / 1e6:.0f} MB limit" + return json.dumps(files), None + + +# --- Credentials ---------------------------------------------------------------------------------- + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules.setdefault(spec.name, module) # its dataclass looks its own module up while it is defined + spec.loader.exec_module(sys.modules[spec.name]) + return sys.modules[spec.name] + + +def _relay(): + return _load("oidc_relay", Path(__file__).with_name("oidc_relay.py")) + + +def _api_key(env: dict, llm: str) -> str: + """The key as the analysis read it, so a pasted `VAR="..."` that worked there works here too.""" + check = _load("credential_check", Path(__file__).parent / "action" / "credential_check.py") + inputs = check.load_table()["providers"].get(llm, {}).get("inputs", {}) + env_var = inputs.get(f"{llm}_api_key", f"{llm.upper()}_API_KEY") + return check._clean_key(env.get(f"{llm.upper()}_API_KEY", ""), env_var) + + +def model_headers(env: dict) -> dict[str, str]: + """What tells the service which model this run uses. Nothing for a hosted or licensed run, which + use CodeBoarding's model and settings as they stand. A run's own provider sends its key and model; + one the service cannot reach sends its name alone, which the service reads as "no model here", so a + run that chose its own provider is never sent to ours. A base URL elsewhere is a provider the + service cannot reach: it calls fixed hosts only.""" + llm = env.get("LLM", "").strip().lower() + if llm in ("", "hosted", "license"): + return {} + key = _api_key(env, llm) + base_url = env.get(f"{llm.upper()}_BASE_URL", "").strip() + host = PROVIDER_HOSTS.get(llm) + if host and llm == "aws_bedrock": + host = host.format(region=env.get("AWS_BEDROCK_REGION", "").strip()) + if not key or not host or (base_url and urlparse(base_url).hostname != host): + return {"x-llm-provider": llm} + headers = {"x-llm-api-key": key, "x-llm-provider": llm} + if env.get(f"{llm.upper()}_REGION", "").strip(): + headers["x-llm-region"] = env[f"{llm.upper()}_REGION"].strip() + # The analysis's own order, so a run's remote features use its analysis model. + model = env.get("AGENT_MODEL", "").strip() or env.get("MODEL", "").strip() + if model: + headers["x-llm-model"] = model + return headers + + +def _bearer(env: dict) -> dict[str, str]: + url, token = env.get("ACTIONS_ID_TOKEN_REQUEST_URL", ""), env.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "") + if not url or not token: + # Without `id-token: write` the run's own token proves the same thing: it can read this + # repository. It carries no license, so the owner's free bucket pays. + return {"Authorization": f"Bearer {env['GITHUB_TOKEN']}"} + relay = _relay() + license_file = None + if env.get("LICENSE_KEY", "").strip(): + license_file = Path(env.get("RUNNER_TEMP", "/tmp")) / "codeboarding-guide-license.txt" + license_file.write_text(env["LICENSE_KEY"].strip()) + try: + config = relay.RelayConfig( + upstream_base_url="", id_token_request_url=url, id_token_request_token=token, license_file=license_file + ) + return {"Authorization": relay._authorization(config)} + finally: + if license_file is not None: + license_file.unlink(missing_ok=True) + + +# --- Requests ------------------------------------------------------------------------------------- + + +def _post(url: str, body: bytes, headers: dict[str, str], timeout: int = 600) -> dict: + request = Request( + url, + data=body, + headers={**headers, "Content-Type": "application/x-xz", "User-Agent": "codeboarding-action"}, + method="POST", + ) + try: + with urlopen(request, timeout=timeout) as response: # nosec B310 - service URL from workflow configuration + return json.loads(response.read()) + except HTTPError as error: + reason = f"the service answered {error.code}: {_error_message(error) or error.reason}" + if error.code == 402: + raise Skip(reason) from error + raise RuntimeError(reason) from error + + +def _encode(body: dict, max_json_bytes: int, what: str = "bundle") -> bytes: + """The request as sent, xz-compressed JSON, or TooLarge when it is over either of the service's + limits. The JSON is measured before compressing it, which is the slow part, and it is compressed once.""" + text = json.dumps(body).encode() + if len(text) > max_json_bytes: + raise TooLarge(f"its {what} is {len(text) / 1e6:.0f} MB of JSON, over the service's limit") + data = lzma.compress(text, preset=9 | lzma.PRESET_EXTREME) + if len(data) > MAX_COMPRESSED_BYTES: + message = f"its {what} is {len(data) / 1e6:.1f} MB compressed, over the service's limit" + raise TooLarge(message, fits=len(text) * MAX_COMPRESSED_BYTES // len(data)) + return data + + +def _requests(ctx: dict, inputs: dict[str, str], env: dict) -> tuple[bytes, bytes]: + """The review guide's request and architecture's, compressed one after the other. Both can send the + body without source, so it is compressed once; it says why source is missing, which only architecture + reads. Architecture's body with source is compressed only when there is source, and the shared one + stands in when it does not fit. TooLarge when not even the shared one fits.""" + try: + source, why = collect_source(env) + except Exception as error: + source, why = None, f"reading it failed: {type(error).__name__}" + body = {"version": 1, "repo": ctx["repo"], "pr": ctx["pr"], "files": inputs} + missing = (why or "the source was not collected") if source is None else "the repository is too large to send" + shared = _encode({**body, "omitted": {"source.json": missing}}, MAX_FEATURE_JSON_BYTES) + if source is None: + return shared, shared + try: + return shared, _encode({**body, "files": {**inputs, "source.json": source}}, MAX_FEATURE_JSON_BYTES) + except TooLarge: + return shared, shared + + +def _no_architecture(ctx: dict, status: str, reason: str) -> dict: + """Preserve the architecture envelope even when the request cannot complete.""" + return { + "feature": "architecture", + "schema": 1, + "status": status, + "repo": ctx["repo"], + "pr": ctx["pr"], + "headSha": ctx["head"], + "generator": None, + "coverage": None, + "data": None, + "reason": reason, + } + + +def run_review_guide(service_url: str, data: bytes, headers: dict, artifact: Path) -> None: + try: + if "x-llm-provider" in headers and "x-llm-api-key" not in headers: + raise Skip("the service cannot call this run's provider") + answer = _post(f"{service_url.rstrip('/')}/v1/review-guide", data, headers) + guide = answer.get("guide") + if not isinstance(guide, dict): + raise ValueError("the service returned no guide") + (artifact / "review_guide.json").write_text(json.dumps(guide, ensure_ascii=False, indent=1)) + except Exception as error: + print(f"::notice::review-guide {'skipped' if isinstance(error, Skip) else 'failed'}: {error}") + + +def run_architecture(service_url: str, ctx: dict, data: bytes, headers: dict) -> dict: + try: + answer = _post(f"{service_url.rstrip('/')}/v1/architecture", data, headers) + envelope = answer.get("envelope") + if not isinstance(envelope, dict) or envelope.get("status") not in ("ok", "partial", "skipped", "failed"): + raise ValueError("the service returned no architecture result") + return envelope + except Exception as error: + return _no_architecture(ctx, "skipped" if isinstance(error, Skip) else "failed", str(error)) + + +def main(env: dict | None = None) -> int: + env = dict(os.environ if env is None else env) + try: + artifact = Path(env["ARTIFACT_DIR"]) + metadata = json.loads((artifact / "metadata.json").read_text()) + ctx = {"repo": env["REPOSITORY"], "pr": int(env["PR_NUMBER"]), "head": metadata.get("head_sha")} + service_url = env["CLOSED_FUNCTIONS_URL"] + if not service_url.strip(): + return 0 + try: + headers = {**_bearer(env), **model_headers(env)} + guide_request, architecture_request = _requests(ctx, collect(env), env) + except TooLarge as error: + print(f"::notice::review-guide skipped: {error}") + architecture = _no_architecture(ctx, "skipped", str(error)) + except Exception as error: + reason = f"collecting inputs or credentials failed: {type(error).__name__}: {error}" + print(f"::notice::review-guide failed: {reason}") + architecture = _no_architecture(ctx, "failed", reason) + else: + with ThreadPoolExecutor(max_workers=2) as pool: + guide = pool.submit(run_review_guide, service_url, guide_request, headers, artifact) + result = pool.submit(run_architecture, service_url, ctx, architecture_request, headers) + architecture = result.result() + guide.result() + path = artifact / "closed" / "architecture.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(architecture, ensure_ascii=False, indent=1)) + if architecture["status"] not in ("ok", "partial"): + print(f"::notice::architecture {architecture['status']}: {architecture.get('reason')}") + except Exception as error: # never fail the review over a remote feature + print(f"::warning::Running the remote features failed: {type(error).__name__}: {error}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_problem_history.py b/tests/test_problem_history.py index 95f9eec..26c9c04 100644 --- a/tests/test_problem_history.py +++ b/tests/test_problem_history.py @@ -1,9 +1,9 @@ """Problem history: the facts a sync run collects, the request it sends, and the ledger commit it makes.""" -import gzip import importlib.util import io import json +import lzma import random import re import sys @@ -218,7 +218,7 @@ def _patched(**names): def _service(seen, problems=3): def post(url, data, headers, timeout=None): - body = json.loads(gzip.decompress(data)) + body = json.loads(lzma.decompress(data)) seen.append({"url": url, "body": body, "headers": headers, "timeout": timeout}) merged = (body["ledger"] or {}).get("merged", []) + body["merged"] answer = {"schema": 1, "repo": body["repo"], "merged": merged, "problems": [{}] * problems} @@ -391,8 +391,11 @@ def test_every_body_is_cut_to_2000_characters_and_an_oversized_request_is_refuse gh.issue_pages[(("OPEN",), "DESC")] = [[issue_node(1, body="x" * 5000), issue_node(2)]] self.run_update(gh) self.assertEqual([i["body"] for i in self.body["issues"]], ["x" * 2000, "It breaks when saving."]) - with mock.patch.dict(ph._check_size.__globals__, {"MAX_BUNDLE_BYTES": 10}): - with self.assertRaisesRegex(problem_history.Skip, "its request is"): + with mock.patch.dict(ph._encode.__globals__, {"MAX_COMPRESSED_BYTES": 10}): + with self.assertRaisesRegex(problem_history.Skip, "its request is .* compressed"): + problem_history.pack({"issues": []}) + with mock.patch.object(ph, "MAX_BRIEFING_JSON_BYTES", 10): + with self.assertRaisesRegex(problem_history.Skip, "its request is .* of JSON"): problem_history.pack({"issues": []}) def test_a_cut_counts_utf16_units_and_never_splits_a_surrogate_pair(self): @@ -784,22 +787,49 @@ def test_an_unread_merge_older_than_the_review_artifacts_is_not_asked_again(self retention = re.search(r"retention-days: (\d+)", upload).group(1) self.assertEqual(int(retention), ph.REVIEW_RETENTION_DAYS) - def test_findings_that_would_overflow_the_request_are_left_out(self): + def pack_counting(self, body, **limits): + """pack(body) under `limits` (MAX_BRIEFING_JSON_BYTES, MAX_COMPRESSED_BYTES), and how often it compressed.""" + compress = mock.Mock(wraps=lzma.compress) + with ( + mock.patch.object( + ph, "MAX_BRIEFING_JSON_BYTES", limits.get("MAX_BRIEFING_JSON_BYTES", ph.MAX_BRIEFING_JSON_BYTES) + ), + mock.patch.dict(ph._encode.__globals__, limits), + mock.patch.object(lzma, "compress", compress), + ): + data, dropped = ph.pack(body) + return json.loads(lzma.decompress(data))["findings"], dropped, compress.call_count + + def test_findings_over_the_services_json_limit_are_left_out_before_compressing(self): + # xz takes these to almost nothing, so only their JSON is over the service's limit. + small, big = {**ENVELOPE, "data": {"findings": ["a"]}}, {**ENVELOPE, "data": {"findings": ["b" * 3000]}} + body = {"issues": [], "findings": {"41": small, "42": big, "43": small}} + limit = len(json.dumps({**body, "findings": {"41": small, "43": small}})) + kept, dropped, compressions = self.pack_counting(body, MAX_BRIEFING_JSON_BYTES=limit) + self.assertEqual((kept, dropped, compressions), ({"41": small, "43": small}, ["42"], 1)) + + def test_findings_over_the_compressed_limit_are_left_out_in_a_few_compressions(self): + rnd = random.Random(1) + findings = { + str(pr): {**ENVELOPE, "data": {"findings": [rnd.randbytes(100 + 20 * pr).hex()]}} for pr in range(60) + } + kept, dropped, compressions = self.pack_counting( + {"issues": [], "findings": findings}, MAX_COMPRESSED_BYTES=20_000 + ) + self.assertLessEqual(compressions, ph.MAX_COMPRESSIONS) + self.assertTrue(dropped and kept) + self.assertEqual(list(kept), [pr for pr in findings if pr not in dropped]) + self.assertLess(max(map(int, kept)), min(map(int, dropped))) + small, big = {**ENVELOPE, "data": {"findings": ["a"]}}, { **ENVELOPE, "data": {"findings": [random.Random(1).randbytes(3000).hex()]}, } - body = {"issues": [], "findings": {"41": small, "42": big}} - with mock.patch.dict(ph._check_size.__globals__, {"MAX_BUNDLE_BYTES": 1000}): - with mock.patch.object(ph, "MAX_BUNDLE_BYTES", 1000): - data, dropped = ph.pack(body) - self.assertEqual((json.loads(gzip.decompress(data))["findings"], dropped), ({"41": small}, ["42"])) - gh = FakeGitHub() gh.merged_pages = [[pr_node(41, "2026-11-06T00:00:00Z"), pr_node(42, "2026-11-05T00:00:00Z")]] gh.add_review(41, 41, small) gh.add_review(42, 42, big) - with mock.patch.object(ph, "MAX_BUNDLE_BYTES", 1200): + with mock.patch.dict(ph._encode.__globals__, {"MAX_COMPRESSED_BYTES": 1200}): notice = self.run_update(gh) self.assertEqual(list(self.body["findings"]), ["41"]) self.assertIn("left out the findings of #42 to stay under the request limit", notice) @@ -1007,9 +1037,9 @@ def test_the_step_uses_only_keys_a_composite_run_step_accepts(self): self.assertLessEqual(keys, allowed) self.assertIn("continue-on-error", keys) - def test_shares_the_closed_features_transport(self): - for shared in (ph._bearer, ph._post, ph._github, ph._gzip, ph._check_size, ph._error_message): - self.assertEqual(shared.__module__, "closed_features") + def test_shares_the_remote_features_transport(self): + for shared in (ph._bearer, ph._post, ph._github, ph._encode, ph._error_message): + self.assertEqual(shared.__module__, "remote_features") if __name__ == "__main__": diff --git a/tests/test_closed_features.py b/tests/test_remote_features.py similarity index 51% rename from tests/test_closed_features.py rename to tests/test_remote_features.py index 8ba89b1..2d80da5 100644 --- a/tests/test_closed_features.py +++ b/tests/test_remote_features.py @@ -1,35 +1,27 @@ """Direct endpoint, source and provider contracts for the optional review requests.""" -import gzip import importlib.util import json +import lzma import os import subprocess import sys import tempfile import threading import unittest -from contextlib import nullcontext from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from importlib import metadata from pathlib import Path +from types import SimpleNamespace from unittest import mock -_SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "closed_features.py" -_SPEC = importlib.util.spec_from_file_location("closed_features", _SCRIPT) -closed_features = importlib.util.module_from_spec(_SPEC) -sys.modules[_SPEC.name] = closed_features -_SPEC.loader.exec_module(closed_features) - -INPUTS = { - "analysis.json", - "base/analysis.json", - "metadata.json", - "pr.json", - "pr-files.json", - "head-cg.json", - "base-cg.json", - "files.json", -} +_SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "remote_features.py" +_SPEC = importlib.util.spec_from_file_location("remote_features", _SCRIPT) +remote_features = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = remote_features +_SPEC.loader.exec_module(remote_features) + +INPUTS = {"head.json", "base.json", "metadata.json", "pr.json", "pr-files.json", "files.json"} GUIDE = {"version": 1, "changes": [{"id": "u1"}]} ENVELOPE = { "feature": "architecture", @@ -119,21 +111,38 @@ def _read(env, name): class BundleTests(unittest.TestCase): + def setUp(self): + patch = mock.patch.object(remote_features, "read_analysis", return_value=(_analysis([]), {})) + patch.start() + self.addCleanup(patch.stop) + def test_analysed_commits_and_both_sides(self): with tempfile.TemporaryDirectory() as tmp: env = _workspace(Path(tmp)) with ( - mock.patch.object(closed_features, "_github_json", side_effect=_github_json) as gh, - mock.patch.object(closed_features, "_contents", return_value="base content") as base, - mock.patch.object(closed_features, "call_graph", return_value={"nodes": {}, "edges": []}), + mock.patch.object(remote_features, "_github_json", side_effect=_github_json) as gh, + mock.patch.object(remote_features, "_contents", return_value="base content") as base, + mock.patch.object(remote_features, "call_graph", return_value={"nodes": {}, "edges": []}), ): - bundle = closed_features.collect(env) + bundle = remote_features.collect(env) self.assertIn("/repos/o/r/compare/B...H", [c.args[0] for c in gh.call_args_list]) self.assertEqual(json.loads(bundle["files.json"]), {"H:app.py": "head content", "B:app.py": "base content"}) base.assert_called_once_with("o/r", "B", "app.py") self.assertEqual(json.loads(bundle["metadata.json"])["base_sha"], "B") self.assertEqual(json.loads(bundle["pr.json"])["commits"], ["Return two"]) self.assertEqual(set(bundle), INPUTS) + self.assertEqual(json.loads(bundle["head.json"])["version"], 1) + + def test_a_changed_file_over_the_size_limit_is_left_out_only_at_that_commit(self): + with tempfile.TemporaryDirectory() as tmp: + env = _workspace(Path(tmp)) + with ( + mock.patch.object(remote_features, "_github_json", side_effect=_github_json), + mock.patch.object(remote_features, "_contents", return_value="b" * 13), + mock.patch.object(remote_features, "call_graph", return_value={"nodes": {}, "edges": []}), + mock.patch.object(remote_features, "MAX_FILE_BYTES", 12), + ): + self.assertEqual(json.loads(remote_features.collect(env)["files.json"]), {"H:app.py": "head content"}) def test_changed_file_symlink_is_not_read(self): with tempfile.TemporaryDirectory() as tmp: @@ -142,22 +151,217 @@ def test_changed_file_symlink_is_not_read(self): path.unlink() path.symlink_to(Path(env["ARTIFACT_DIR"]) / "metadata.json") with ( - mock.patch.object(closed_features, "_github_json", side_effect=_github_json), - mock.patch.object(closed_features, "_contents", return_value=None), - mock.patch.object(closed_features, "call_graph", return_value={}), + mock.patch.object(remote_features, "_github_json", side_effect=_github_json), + mock.patch.object(remote_features, "_contents", return_value=None), + mock.patch.object(remote_features, "call_graph", return_value={}), ): - self.assertEqual(json.loads(closed_features.collect(env)["files.json"]), {}) + self.assertEqual(json.loads(remote_features.collect(env)["files.json"]), {}) + + +def _method(name, start, end, content_hash): + return SimpleNamespace( + qualified_name=name, start_line=start, end_line=end, node_type="FUNCTION", content_hash=content_hash + ) + + +def _component(cid, name=None, members=None, description=None): + """A component as Core parses it; members are qualified names by file.""" + groups = [ + SimpleNamespace(file_path=path, methods=[SimpleNamespace(qualified_name=q) for q in names]) + for path, names in (members or {}).items() + ] + return SimpleNamespace(component_id=cid, name=name, description=description, file_methods=groups) + + +def _relation(src, dst, label, *edges): + """A relation as Core parses it; edges are `path|qualified_name` pairs.""" + + def reference(key): + path, name = key.split("|") + return SimpleNamespace(reference_file=path, qualified_name=name) + + return SimpleNamespace( + src_id=src, + dst_id=dst, + relation=label, + all_edges=[SimpleNamespace(source=reference(s), target=reference(t)) for s, t in edges], + ) + + +def _analysis(components, relations=(), files=None): + """A scope as Core parses it: the root carries the file index, a sub-analysis does not need one.""" + return SimpleNamespace( + components=list(components), + components_relations=list(relations), + files={path: SimpleNamespace(methods=methods) for path, methods in (files or {}).items()}, + ) + + +class SideDocumentTests(unittest.TestCase): + """The service's bundle.expand_side reads this exact document back (its tests hold the same one).""" + + ROOT = _analysis( + [ + _component( + "1", + "Core", + {"pkg/a.py": ["pkg.a.run", "pkg.a.helper"], "pkg/b.py": ["pkg.b.save", "~Weird"]}, + description="d1", + ), + _component("2", "Also", {"pkg/b.py": ["pkg.b.save"], "pkg/c.py": ["pkg.c.gone"]}), + ], + [_relation("1", "2", "uses", ("pkg/a.py|pkg.a.run", "pkg/b.py|pkg.b.save"))], + files={ + "pkg/a.py": [_method("pkg.a.run", 1, 5, "h1"), _method("pkg.a.helper", 6, 8, "h2")], + "pkg/b.py": [_method("pkg.b.save", 1, 4, "h3"), _method("~Weird", 10, 12, "h4")], + }, + ) + SUB_ANALYSES = { + "1": _analysis( + [ + _component("1.1", "Run", {"pkg/a.py": ["pkg.a.run", "pkg.a.helper"]}), + _component("1.2", "Store", {"pkg/b.py": ["pkg.b.save", "~Weird"]}), + ], + [ + _relation( + "1.1", + "1.2", + "saves", + ("pkg/a.py|pkg.a.run", "pkg/b.py|pkg.b.save"), + ("pkg/a.py|pkg.a.helper", "pkg/b.py|pkg.b.save"), + ) + ], + ) + } + GRAPH = { + "nodes": { + "pkg.a.run": {"file": "pkg/a.py", "start": 1, "end": 5, "cb": False}, + "pkg.a.helper": {"file": "pkg/a.py", "start": 6, "end": 9, "cb": False}, + "pkg.b.save": {"file": "pkg/b.py", "start": 1, "end": 4, "cb": False}, + "pkg.x.Only": {"file": "pkg/x.py", "start": 1, "end": 2, "cb": True}, + }, + "edges": [ + {"s": "pkg.a.run", "d": "pkg.b.save", "lines": [3, 4]}, + {"s": "pkg.a.run", "d": "pkg.a.helper", "lines": [2]}, + {"s": "pkg.b.save", "d": "pkg.a.run", "lines": [2]}, + {"s": "pkg.x.Only", "d": "ext.lib.call", "lines": []}, + ], + } + DOCUMENT = { + "version": 1, + "files": [ + ["pkg/a.py", [["~.run", 1, 5, "FUNCTION", "h1", 0], ["~.helper", 6, 8, "FUNCTION", "h2", 0]]], + ["pkg/b.py", [["~.save", 1, 4, "FUNCTION", "h3", 0], ["~~Weird", 10, 12, "FUNCTION", "h4", -1]]], + ["pkg/x.py", [["~.Only", 1, 2, 1]]], + ], + "unplaced": ["ext.lib.call"], + "graph_spans": [[1, 6, 9]], + "components": [ + ["1", "Core", "d1", -1, None, [["1.1", "1.2", "saves"]]], + ["1.1", "Run", None, 0, [0, 1], []], + ["1.2", "Store", None, 0, [2, 3], []], + ["2", "Also", None, -1, [2, "pkg/c.py|pkg.c.gone"], []], + ], + "relations": [["1", "2", "uses"]], + "calls": [[0, [[2, 3, 4], [1, 2]]], [2, [[0, 2]]], [4, [[5]]]], + "extra_edges": [[1, 2]], + } + + def test_each_fact_once_and_only_what_the_service_reads(self): + doc = json.loads(json.dumps(remote_features.side_document(self.ROOT, self.SUB_ANALYSES, self.GRAPH))) + self.assertEqual(doc, self.DOCUMENT) + + def test_a_parent_is_left_out_only_when_its_childrens_members_rebuild_it(self): + # The service rebuilds a parent from its children's members, so a parent holding fewer than a + # child does is written out even though its leaves hold exactly its members. + def component(cid, names): + return _component(cid, members={"a.py": list(names)}) + + root = _analysis( + [component("G", "xy"), component("P", "xy")], + files={"a.py": [_method(q, i, i, "h") for i, q in enumerate("xyz", 1)]}, + ) + sub_analyses = { + "G": _analysis([component("M", "xyz")]), + "M": _analysis([component("L1", "x"), component("L2", "y")]), + "P": _analysis([component("C1", "x"), component("C2", "y")]), + } + members = [c[4] for c in remote_features.side_document(root, sub_analyses, {})["components"]] + self.assertEqual(members, [[0, 1], [0, 1, 2], [0], [1], None, [0], [1]]) + + +def _core_installed() -> bool: + try: + metadata.version("codeboarding") + except metadata.PackageNotFoundError: + return False + return True + + +@unittest.skipUnless(_core_installed(), "the pinned CodeBoarding release is not installed") +class InstalledCoreTests(unittest.TestCase): + """read_analysis and side_document against the release action.yml pins. It runs in the + `core-compatibility` CI job. + + tests/fixtures/analysis holds one analysis as two Core releases wrote it: core-0.15.5.json and + core-9cf50fb9.json (main after #571). Each release reads only its own, so the pinned one must read at + least one, and every one it reads must give the same document. A pin whose release reads none needs + that analysis written by it, which is the point: the document has not been checked against its parser.""" + + GRAPH = { + "nodes": { + **SideDocumentTests.GRAPH["nodes"], + "pkg.c.load": {"file": "pkg/c.py", "start": 1, "end": 3, "cb": False}, + }, + "edges": [*SideDocumentTests.GRAPH["edges"], {"s": "pkg.c.load", "d": "pkg.b.save", "lines": [2]}], + } + DOCUMENT = { + "version": 1, + "files": [ + ["pkg/a.py", [["~.run", 1, 5, "FUNCTION", "h1", 0], ["~.helper", 6, 8, "FUNCTION", "h2", 0]]], + ["pkg/b.py", [["~.save", 1, 4, "FUNCTION", "h3", 0], ["~~Weird", 10, 12, "FUNCTION", "h4", -1]]], + ["pkg/c.py", [["~.load", 1, 3, "FUNCTION", "h5", 0]]], + ["pkg/x.py", [["~.Only", 1, 2, 1]]], + ], + "unplaced": ["ext.lib.call"], + "graph_spans": [[1, 6, 9]], + "components": [ + ["1", "Core", "d1", -1, None, [["1.1", "1.2", "saves"]]], + ["1.1", "Run", "", 0, [0, 1], []], + ["1.2", "Store", "", 0, [2, 3], []], + ["2", "Also", "", -1, [4], []], + ], + "relations": [["1.1", "1.2", "saves"], ["1.1", "2", "uses"], ["1.2", "2", "loads"], ["2", "1.2", "reads"]], + "calls": [[0, [[2, 3, 4], [1, 2]]], [2, [[0, 2]]], [4, [[2, 2]]], [5, [[6]]]], + # Of the analysis's edges, the calls are left to the graph: load -> save is one. + "extra_edges": [[0, 4], [1, 2], [3, 4]], + } + + def test_the_pinned_release_reads_an_analysis_it_wrote_into_the_side_document(self): + read = [] + for fixture in sorted((Path(__file__).parent / "fixtures" / "analysis").glob("*.json")): + try: + root, sub_analyses = remote_features.read_analysis(fixture) + except (TypeError, ValueError): # another release's format + continue + read.append(fixture.name) + with self.subTest(fixture=fixture.name): + doc = json.loads(json.dumps(remote_features.side_document(root, sub_analyses, self.GRAPH))) + self.assertEqual(doc, self.DOCUMENT) + self.assertTrue( + read, "the pinned release reads none of tests/fixtures/analysis; add the analysis as it writes it" + ) class CredentialTests(unittest.TestCase): def test_without_oidc_the_runs_own_token_proves_the_repository(self): - self.assertEqual(closed_features._bearer({"GITHUB_TOKEN": "ghs_run"}), {"Authorization": "Bearer ghs_run"}) + self.assertEqual(remote_features._bearer({"GITHUB_TOKEN": "ghs_run"}), {"Authorization": "Bearer ghs_run"}) def test_oidc_bearer_includes_license_and_removes_temporary_file(self): mint = _Server(lambda p, b: (200, {"value": "jwt.token.sig"})) self.addCleanup(mint.close) with tempfile.TemporaryDirectory() as tmp: - headers = closed_features._bearer( + headers = remote_features._bearer( { "ACTIONS_ID_TOKEN_REQUEST_URL": f"{mint.url}/token?x=1", "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "req", @@ -171,10 +375,10 @@ def test_oidc_bearer_includes_license_and_removes_temporary_file(self): def test_hosted_and_license_send_no_model(self): for llm in ("", "hosted", "license"): - self.assertEqual(closed_features.model_headers({"LLM": llm, "OPENAI_API_KEY": "leftover"}), {}) + self.assertEqual(remote_features.model_headers({"LLM": llm, "OPENAI_API_KEY": "leftover"}), {}) def test_each_fixed_provider_and_analysis_model(self): - for provider, host in closed_features.PROVIDER_HOSTS.items(): + for provider, host in remote_features.PROVIDER_HOSTS.items(): with self.subTest(provider=provider): env = { "LLM": provider, @@ -186,7 +390,7 @@ def test_each_fixed_provider_and_analysis_model(self): env["AWS_BEDROCK_REGION"] = "eu-west-1" host = host.format(region="eu-west-1") env[f"{provider.upper()}_BASE_URL"] = f"https://{host}/v1" - headers = closed_features.model_headers(env) + headers = remote_features.model_headers(env) self.assertEqual(headers["x-llm-api-key"], "k") self.assertEqual(headers["x-llm-model"], "analysis") if provider == "aws_bedrock": @@ -200,7 +404,7 @@ def test_key_is_unwrapped_the_way_the_analysis_reads_it(self): ({**bedrock, "AWS_BEDROCK_API_KEY": "AWS_BEARER_TOKEN_BEDROCK=abc="}, "abc="), ): with self.subTest(llm=env["LLM"]): - self.assertEqual(closed_features.model_headers(env)["x-llm-api-key"], want) + self.assertEqual(remote_features.model_headers(env)["x-llm-api-key"], want) def test_unsupported_self_hosted_and_missing_key_never_fall_back(self): for env in ( @@ -209,7 +413,7 @@ def test_unsupported_self_hosted_and_missing_key_never_fall_back(self): {"LLM": "openai", "OPENAI_API_KEY": "k", "OPENAI_BASE_URL": "https://gateway.corp/v1"}, {"LLM": "anthropic"}, ): - self.assertEqual(closed_features.model_headers(env), {"x-llm-provider": env["LLM"]}) + self.assertEqual(remote_features.model_headers(env), {"x-llm-provider": env["LLM"]}) class EndpointTests(unittest.TestCase): @@ -221,8 +425,9 @@ def setUp(self): ("_github_json", {"side_effect": _github_json}), ("_contents", {"return_value": "base"}), ("call_graph", {"return_value": {"nodes": {}, "edges": []}}), + ("read_analysis", {"return_value": (_analysis([]), {})}), ): - patch = mock.patch.object(closed_features, name, **kwargs) + patch = mock.patch.object(remote_features, name, **kwargs) patch.start() self.addCleanup(patch.stop) @@ -243,7 +448,7 @@ def run_service(self, respond=None): self.addCleanup(service.close) self.env["CLOSED_FUNCTIONS_URL"] = service.url + "/" with mock.patch("builtins.print"): - self.assertEqual(closed_features.main(self.env), 0) + self.assertEqual(remote_features.main(self.env), 0) return service def test_exactly_two_posts_shared_inputs_and_fixed_outputs(self): @@ -258,7 +463,7 @@ def test_exactly_two_posts_shared_inputs_and_fixed_outputs(self): for request in service.requests: self.assertEqual(request["method"], "POST") self.assertEqual(request["headers"]["Authorization"], "Bearer ghs_run") - bodies[request["path"]] = json.loads(gzip.decompress(request["body"])) + bodies[request["path"]] = json.loads(lzma.decompress(request["body"])) guide, arch = bodies["/v1/review-guide"], bodies["/v1/architecture"] self.assertEqual(set(guide["files"]), INPUTS) self.assertEqual(set(arch["files"]), INPUTS | {"source.json"}) @@ -318,38 +523,54 @@ def respond(path, body): def test_empty_url_disables_both(self): self.env["CLOSED_FUNCTIONS_URL"] = "" - with mock.patch.object(closed_features, "_post") as post: - self.assertEqual(closed_features.main(self.env), 0) + with mock.patch.object(remote_features, "_post") as post: + self.assertEqual(remote_features.main(self.env), 0) post.assert_not_called() self.assertFalse((Path(self.env["ARTIFACT_DIR"]) / "closed").exists()) - def test_source_toggle_and_read_failure_only_omit_architecture_source(self): + def run_counting(self, **patches): + """run_service with `patches` on remote_features, and how often it compressed.""" + compress = mock.Mock(wraps=lzma.compress) + with mock.patch.object(lzma, "compress", compress), mock.patch.dict(remote_features.__dict__, patches): + service = self.run_service() + return {r["path"]: r["body"] for r in service.requests}, compress.call_count + + def test_without_source_both_send_one_body_compressed_once(self): for failure in (False, True): self.env["CLOSED_FEATURES_SOURCE"] = "false" - with ( - mock.patch.object(closed_features, "collect_source", side_effect=OSError()) - if failure - else nullcontext() - ): - service = self.run_service() - for request in service.requests: - body = json.loads(gzip.decompress(request["body"])) - self.assertNotIn("source.json", body["files"]) - self.assertEqual("omitted" in body, request["path"] == "/v1/architecture") - - def test_bundle_limit_omits_source_then_skips_if_required_inputs_too_large(self): - with ( - mock.patch.object(closed_features, "MAX_BUNDLE_BYTES", 1000), - mock.patch.object(closed_features, "collect_source", return_value=(os.urandom(2000).hex(), None)), + patches = {"collect_source": mock.Mock(side_effect=OSError())} if failure else {} + bodies, compressions = self.run_counting(**patches) + self.assertEqual((compressions, len(set(bodies.values()))), (1, 1)) + body = json.loads(lzma.decompress(bodies["/v1/architecture"])) + self.assertNotIn("source.json", body["files"]) + self.assertIn("reading it failed" if failure else "turned off", body["omitted"]["source.json"]) + + def test_source_is_compressed_once_more_and_only_for_architecture(self): + bodies, compressions = self.run_counting(collect_source=mock.Mock(return_value=("{}", None))) + self.assertEqual(compressions, 2) + self.assertNotEqual(bodies["/v1/architecture"], bodies["/v1/review-guide"]) + + def test_either_limit_omits_source_then_skips_if_required_inputs_too_large(self): + with mock.patch.object(remote_features, "collect_source", return_value=(None, "off")): + without_source = max(len(lzma.decompress(r["body"])) for r in self.run_service().requests) + # Random source is over the compressed limit; repeated source compresses to almost nothing and is + # over the JSON limit only. + for limit, room, source in ( + ("MAX_COMPRESSED_BYTES", 1000, os.urandom(2000).hex()), + ("MAX_FEATURE_JSON_BYTES", without_source + 1000, "x" * 100_000), ): - service = self.run_service() - arch = next(r for r in service.requests if r["path"] == "/v1/architecture") - body = json.loads(gzip.decompress(arch["body"])) - self.assertNotIn("source.json", body["files"]) - self.assertIn("too large", body["omitted"]["source.json"]) - with mock.patch.object(closed_features, "MAX_BUNDLE_BYTES", 1): - self.assertEqual(self.run_service().requests, []) - self.assertEqual(_read(self.env, "closed/architecture.json")["status"], "skipped") + bodies, compressions = self.run_counting( + **{limit: room, "collect_source": mock.Mock(return_value=(source, None))} + ) + # Architecture sends the guide's body, which already says why source is missing. + self.assertEqual(bodies["/v1/architecture"], bodies["/v1/review-guide"]) + self.assertEqual(compressions, 2 if limit == "MAX_COMPRESSED_BYTES" else 1) + body = json.loads(lzma.decompress(bodies["/v1/architecture"])) + self.assertNotIn("source.json", body["files"]) + self.assertIn("too large", body["omitted"]["source.json"]) + with mock.patch.object(remote_features, limit, 1): + self.assertEqual(self.run_service().requests, []) + self.assertEqual(_read(self.env, "closed/architecture.json")["status"], "skipped") class SourceTests(unittest.TestCase): @@ -384,11 +605,11 @@ def test_tracked_extensions_exclusions_limits_and_symlinks(self): (root / "link.py").symlink_to(root / "a.py") subprocess.run(["git", "-C", tmp, "add", "."], check=True) (root / "untracked.py").write_text("not tracked") - text, why = closed_features.collect_source({"CHECKOUT_DIR": tmp}) + text, why = remote_features.collect_source({"CHECKOUT_DIR": tmp}) self.assertEqual(set(json.loads(text)), set(names[:9])) self.assertIsNone(why) - with mock.patch.object(closed_features, "MAX_SOURCE_BYTES", 1): - text, why = closed_features.collect_source({"CHECKOUT_DIR": tmp}) + with mock.patch.object(remote_features, "MAX_SOURCE_BYTES", 1): + text, why = remote_features.collect_source({"CHECKOUT_DIR": tmp}) self.assertIsNone(text) self.assertIn("limit", why) @@ -398,11 +619,11 @@ def test_symlink_parent_and_traversal_rejected(self): (root / "link").symlink_to(root, target_is_directory=True) for name in ("link/a.py", "../a.py", "/tmp/a.py"): with self.assertRaises(OSError): - closed_features._checkout_file(root, name) + remote_features._checkout_file(root, name) def test_disabled_source_does_not_read_checkout(self): self.assertEqual( - closed_features.collect_source({"CLOSED_FEATURES_SOURCE": "false"}), (None, "turned off by the workflow") + remote_features.collect_source({"CLOSED_FEATURES_SOURCE": "false"}), (None, "turned off by the workflow") )