diff --git a/scripts/test_validate.py b/scripts/test_validate.py index b66b7d166..7890b37df 100644 --- a/scripts/test_validate.py +++ b/scripts/test_validate.py @@ -355,6 +355,62 @@ def test_ci_skipped_checks_count_as_success(self): success, message = check_ci_status(SAMPLE_COMMIT_SHA) assert success is True + def test_ci_ignored_cla_status_does_not_block(self): + """cla-bot's verification/cla-signed status is permanently red on + external-contributor backport branches; filtering it out should let + the build proceed when it's the only failure.""" + with patch("validate.gh_api") as mock_api: + mock_api.side_effect = [ + { + "state": "failure", + "total_count": 2, + "statuses": [ + {"context": "verification/cla-signed", "state": "error"}, + {"context": "buildkite/firecracker", "state": "success"}, + ], + }, + {"total_count": 0, "check_runs": []}, + ] + success, message = check_ci_status(SAMPLE_COMMIT_SHA) + assert success is True + assert "passed" in message + + def test_ci_ignored_cla_status_alone_falls_through(self): + """If the CLA status is the only one and we filter it out, the + rollup is empty → fall back to the no-checks 'proceed anyway' path.""" + with patch("validate.gh_api") as mock_api: + mock_api.side_effect = [ + { + "state": "failure", + "total_count": 1, + "statuses": [ + {"context": "verification/cla-signed", "state": "error"}, + ], + }, + {"total_count": 0, "check_runs": []}, + ] + success, message = check_ci_status(SAMPLE_COMMIT_SHA) + assert success is True + + def test_ci_other_failure_still_blocks_when_cla_also_failed(self): + """Real CI failure must still block even when the CLA status is also + failing — the filter must not mask non-ignored failures.""" + with patch("validate.gh_api") as mock_api: + mock_api.side_effect = [ + { + "state": "failure", + "total_count": 2, + "statuses": [ + {"context": "verification/cla-signed", "state": "error"}, + {"context": "buildkite/firecracker", "state": "failure"}, + ], + }, + {"total_count": 0, "check_runs": []}, + ] + success, message = check_ci_status(SAMPLE_COMMIT_SHA) + assert success is False + assert "failed" in message + class TestGenerateBuildMatrix: """Tests for generate_build_matrix function.""" diff --git a/scripts/validate.py b/scripts/validate.py index e67568d3c..dc60de6f0 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -154,27 +154,88 @@ def resolve_tag_and_commit( return "", "", "Either tag or commit_hash must be provided" +# IGNORED_STATUS_CONTEXTS lists legacy commit-status contexts that should not +# block a release build even when failing. Keep the set tiny and well-justified. +# +# verification/cla-signed: cla-bot fails on the upstream firecracker fork +# whenever a backport branch carries commits authored by upstream maintainers +# we don't have a CLA for (e.g. ilstam, ShadowCurse, JackThomson2). Those +# contributors won't ever sign our CLA, so the status is permanently red on +# every direct-mem / hint backport branch — we still want to ship those builds. +IGNORED_STATUS_CONTEXTS = frozenset({"verification/cla-signed"}) + +# IGNORED_CHECK_NAMES is the equivalent for the Checks API (apps that file a +# check-run rather than a legacy status). Empty today; mirror IGNORED_STATUS_CONTEXTS +# if a check-run-based bot ever ends up in the same situation. +IGNORED_CHECK_NAMES = frozenset() + + +def _rollup_status(statuses: list[dict]) -> tuple[str, int]: + """Compute (state, count) over the statuses list, mirroring how GitHub's + combined-status endpoint rolls up: any failure → failure, else any pending + → pending, else any success → success, else unknown. + """ + if not statuses: + return "unknown", 0 + states = {s.get("state") for s in statuses} + if "failure" in states or "error" in states: + return "failure", len(statuses) + if "pending" in states: + return "pending", len(statuses) + if "success" in states: + return "success", len(statuses) + return "unknown", len(statuses) + + def check_ci_status(commit_hash: str, repo: str = "e2b-dev/firecracker") -> tuple[bool, str]: """ Check CI status for a commit. Returns (success, message). """ - # Check commit status API + # Check commit status API. Filter out IGNORED_STATUS_CONTEXTS and recompute + # the rollup so a single permanently-red status (e.g. cla-bot on + # external-contributor backport branches) doesn't block release builds. status_response = gh_api(f"/repos/{repo}/commits/{commit_hash}/status") if not status_response: - status_response = {"state": "unknown", "total_count": 0} - - status = status_response.get("state", "unknown") - status_count = status_response.get("total_count", 0) + status_response = {"state": "unknown", "total_count": 0, "statuses": []} + + raw_statuses = status_response.get("statuses", []) or [] + ignored_status_contexts = [ + s.get("context") for s in raw_statuses + if s.get("context") in IGNORED_STATUS_CONTEXTS + ] + filtered_statuses = [ + s for s in raw_statuses + if s.get("context") not in IGNORED_STATUS_CONTEXTS + ] + if ignored_status_contexts: + status, status_count = _rollup_status(filtered_statuses) + print( + f"Status API: ignoring contexts {sorted(set(ignored_status_contexts))} " + f"→ rollup state={status}, count={status_count}", + file=sys.stderr, + ) + else: + status = status_response.get("state", "unknown") + status_count = status_response.get("total_count", 0) + print(f"Status API: state={status}, count={status_count}", file=sys.stderr) - # Check check-runs API + # Check check-runs API. Same filter for IGNORED_CHECK_NAMES. check_response = gh_api(f"/repos/{repo}/commits/{commit_hash}/check-runs") if not check_response: check_response = {"total_count": 0, "check_runs": []} - check_count = check_response.get("total_count", 0) - check_runs = check_response.get("check_runs", []) + raw_check_runs = check_response.get("check_runs", []) or [] + ignored_check_names = [ + cr.get("name") for cr in raw_check_runs + if cr.get("name") in IGNORED_CHECK_NAMES + ] + check_runs = [ + cr for cr in raw_check_runs + if cr.get("name") not in IGNORED_CHECK_NAMES + ] + check_count = len(check_runs) # Determine check conclusion if check_count == 0: @@ -188,8 +249,14 @@ def check_ci_status(commit_hash: str, repo: str = "e2b-dev/firecracker") -> tupl else: check_conclusion = "unknown" - print(f"Status API: state={status}, count={status_count}", file=sys.stderr) - print(f"Check-runs API: conclusion={check_conclusion}, count={check_count}", file=sys.stderr) + if ignored_check_names: + print( + f"Check-runs API: ignoring {sorted(set(ignored_check_names))} " + f"→ conclusion={check_conclusion}, count={check_count}", + file=sys.stderr, + ) + else: + print(f"Check-runs API: conclusion={check_conclusion}, count={check_count}", file=sys.stderr) if status == "failure" or check_conclusion == "failure": return False, f"CI failed for commit {commit_hash} - refusing to build"