diff --git a/.agents/skills/buildkite-retry-job/SKILL.md b/.agents/skills/buildkite-retry-job/SKILL.md index 4b822e6222..d01707cb01 100644 --- a/.agents/skills/buildkite-retry-job/SKILL.md +++ b/.agents/skills/buildkite-retry-job/SKILL.md @@ -1,27 +1,42 @@ --- name: buildkite-retry-job -description: Retry failed Buildkite jobs individually or rebuild full builds +description: Retry failed Buildkite jobs, unblock manual approval steps, or + rebuild full builds --- -Use `scripts/retry_buildkite_jobs.py` to retry individual jobs or rebuild full +Use `scripts/retry_buildkite_jobs.py` to retry individual jobs, unblock manual +approval steps (such as `.bazelci/presubmit.yml` changes), or rebuild full Buildkite builds. -### Retrying Modes +### Retrying & Unblocking Modes -1. **Job-Level Retry (Default for failed jobs)**: - Retries only specific failing jobs via `bk job retry `: +1. **Default Retry & Unblock**: + Unblocks any blocked manual approval steps (`bk job unblock + --no-input`) and retries specific failing jobs (`bk job retry `): ```bash - ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py + ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ + ``` -2. **Retry Specific Jobs by Name or ID**: +2. **Unblock Manual Approval Steps (e.g. `.bazelci/presubmit.yml` changes)**: + Approves and unblocks blocked manual approval steps without retrying or + rebuilding other jobs: ```bash - ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py --jobs "compile_pip_requirements" - ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py --job-id + ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ + --unblock ``` -3. **Rebuild Full Build**: +3. **Retry Specific Jobs by Name or ID**: + ```bash + ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ + --jobs "compile_pip_requirements" + ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ + --job-id + ``` + +4. **Rebuild Full Build**: Re-runs the entire pipeline build from scratch via `bk build rebuild`: ```bash - ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py --rebuild + ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ + --rebuild ``` diff --git a/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py b/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py index b0a154ac0c..97e5fb5a8c 100755 --- a/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py +++ b/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py @@ -61,6 +61,20 @@ def retry_job_by_uuid(job_id): return subprocess.run(["bk", "job", "retry", job_id]) +def unblock_manual_jobs(jobs): + unblocked = [] + for j in jobs: + jstate = j.get("state", "") + is_manual_block = j.get("type") == "manual" or j.get("unblockable") is True + jid = j.get("id") + if jid and jstate in ("blocked", "blocked_failed") and is_manual_block: + jname = j.get("name") or j.get("label") or jid + print(f"🔓 Unblocking manual approval job '{jname}' (ID: {jid})...") + subprocess.run(["bk", "job", "unblock", jid, "--no-input"]) + unblocked.append(j) + return unblocked + + def retry_jobs_by_name(pipeline, build_number, job_pattern): check_cli("bk", "https://github.com/buildkite/cli") cmd = ["bk", "build", "view", str(build_number), "-p", pipeline, "--json"] @@ -119,6 +133,11 @@ def main(): action="store_true", help="Rebuild the entire build afresh via 'bk build rebuild'", ) + parser.add_argument( + "--unblock", + action="store_true", + help="Unblock manual approval steps (e.g. .bazelci/presubmit.yml changes) via 'bk job unblock'", + ) args = parser.parse_args() check_cli("bk", "https://github.com/buildkite/cli") @@ -146,6 +165,24 @@ def main(): if args.job_name: retry_jobs_by_name(pipeline, build_number, args.job_name) + elif args.unblock: + cmd = [ + "bk", + "build", + "view", + str(build_number), + "-p", + pipeline, + "--json", + ] + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + data = json.loads(res.stdout) + unblocked = unblock_manual_jobs(data.get("jobs", [])) + if not unblocked: + print( + f"ℹī¸ No blocked manual approval jobs found in build #{build_number}.", + file=sys.stderr, + ) elif args.rebuild: print(f"🚀 Rebuilding entire build #{build_number} for pipeline: {pipeline}") res = subprocess.run( @@ -153,7 +190,7 @@ def main(): ) sys.exit(res.returncode) else: - # Default behavior: try job-level retries of failed jobs if any, otherwise rebuild + # Default behavior: unblock manual approval steps and retry failed jobs if any, otherwise rebuild cmd = [ "bk", "build", @@ -166,9 +203,11 @@ def main(): try: res = subprocess.run(cmd, capture_output=True, text=True, check=True) data = json.loads(res.stdout) + jobs = data.get("jobs", []) + unblocked_jobs = unblock_manual_jobs(jobs) failed_jobs = [ j - for j in data.get("jobs", []) + for j in jobs if j.get("state") in ("failed", "broken", "timed_out") and j.get("id") ] if failed_jobs: @@ -177,7 +216,7 @@ def main(): jname = j.get("name") or j.get("label") or jid print(f"🚀 Retrying failed job '{jname}' (ID: {jid})...") subprocess.run(["bk", "job", "retry", jid]) - else: + elif not unblocked_jobs: print( f"🚀 Rebuilding entire build #{build_number} for pipeline: {pipeline}" ) diff --git a/.agents/skills/merge-pr/SKILL.md b/.agents/skills/merge-pr/SKILL.md index e611e64aa0..f05a760d31 100644 --- a/.agents/skills/merge-pr/SKILL.md +++ b/.agents/skills/merge-pr/SKILL.md @@ -23,7 +23,13 @@ this PR", or monitor its merge): - Have the subagent use the **`buildkite-retry-job`** skill (`retry_buildkite_jobs.py `) to automatically retry any transient network flakes (e.g., HTTP 504 gateway timeouts, downloader - errors). + errors) and unblock manual approval steps. + - **Blocked Builds (`.bazelci/presubmit.yml` Approval)**: When + `.bazelci/presubmit.yml` is modified, Buildkite blocks the build until + manually approved. Approve and unblock it immediately via + `./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py + --unblock` (or `bk job unblock --no-input`) so CI does + not sit blocked indefinitely. - **Soft-Failing Jobs**: Experimental Buildkite jobs (e.g. `*rolling*` Bazel) are non-blocking soft failures; do not treat them as merge blockers. - When the PR is queued, actively discover the merge queue branch via diff --git a/.agents/skills/monitor-ci-results/SKILL.md b/.agents/skills/monitor-ci-results/SKILL.md index 2e1190d12b..2c30533f90 100644 --- a/.agents/skills/monitor-ci-results/SKILL.md +++ b/.agents/skills/monitor-ci-results/SKILL.md @@ -30,9 +30,14 @@ or when monitoring CI after PR updates: 1. **Background Polling**: `monitor_remote_ci.py` continuously polls both GitHub PR checks and Buildkite workflow executions in the background across new commits and CI re-runs. -2. **Blocked Jobs**: When a Buildkite job or GitHub check is in a blocked state - waiting for user confirmation, it dispatches a notification via `agentapi - send-message` so the user is alerted to confirm running the job. +2. **Blocked Jobs (`.bazelci/presubmit.yml` Approval)**: When a Buildkite build + is blocked on a manual approval step (e.g., when `.bazelci/presubmit.yml` is + modified), `monitor_remote_ci.py` automatically approves and unblocks it via + `bk job unblock --no-input` so CI does not sit blocked. If + automatic unblocking fails, it notifies the conversation via + `agentapi send-message`; unblock Buildkite jobs using the + `buildkite-retry-job` skill (`retry_buildkite_jobs.py + --unblock`). 3. **Failure Reporting**: When any GitHub check or Buildkite job completes with errors, `monitor_remote_ci.py` dispatches a high-priority notification message reporting the failed check back to your conversation. diff --git a/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py b/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py index 57fde0ce8d..19a7f119c9 100755 --- a/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py +++ b/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py @@ -4,6 +4,7 @@ import fcntl import json import os +import re import subprocess import sys import time @@ -43,6 +44,25 @@ def get_buildkite_jobs(build_url): if base_url.endswith(".json"): base_url = base_url[:-5] + m = re.search(r"buildkite\.com/([^/]+/[^/]+)/builds/(\d+)", base_url) + if m and check_cli("bk"): + pipeline, build_number = m.group(1), m.group(2) + try: + res = subprocess.run( + ["bk", "build", "view", build_number, "-p", pipeline, "--json"], + capture_output=True, + text=True, + check=True, + ) + data = json.loads(res.stdout) + if isinstance(data, dict) and "jobs" in data: + return data["jobs"] + except Exception as e: + print( + f"⚠ī¸ Could not fetch Buildkite jobs via bk CLI for {pipeline}#{build_number}: {e}", + file=sys.stderr, + ) + jobs_url = f"{base_url}/data/jobs.json" req = urllib.request.Request(jobs_url, headers={"User-Agent": "ci-monitor"}) try: @@ -198,14 +218,18 @@ def main(): for job in jobs: if job.get("retried_in_job_uuid") or job.get("retried_at"): continue - jname = job.get("name", "unknown_job") + jname = job.get("name") or job.get("label") or "unknown_job" jstate = job.get("state", "unknown") jid = job.get("id", "") jkey = f"bk_{jid}" + log_url = job.get("web_url") or job.get("log_url") or link exit_status = job.get("exit_status") is_soft_failed = job.get("soft_failed") is True is_blocked = jstate in ["blocked", "blocked_failed"] + is_manual_block = ( + job.get("type") == "manual" or job.get("unblockable") is True + ) is_failed = ( ( jstate in ["failed", "failing"] @@ -215,27 +239,38 @@ def main(): and not is_blocked ) - if is_blocked: + if is_blocked and is_manual_block: jkey_blocked = f"bk_blocked_{jid}" if jkey_blocked not in monitored: - print( - f"⏸ī¸ Notifying blocked state for Buildkite job '{jname}' (ID: {jid})..." - ) - msg = ( - f"⚠ī¸ Remote CI Buildkite Job '{jname}' is blocked!\n\n" - f"Build ID: {build_id} | Job ID: {jid}\n" - f"Log URL: {job.get('log_url', link)}\n\n" - f"The user must confirm whether to run the CI jobs." - ) - subprocess.run( - [ - "agentapi", - "send-message", - "--title=CI Job Blocked", - args.conv_id, - msg, - ] - ) + unblocked = False + if jid and check_cli("bk"): + print( + f"🔓 Automatically unblocking Buildkite approval step '{jname}' (ID: {jid})..." + ) + res = subprocess.run( + ["bk", "job", "unblock", jid, "--no-input"] + ) + unblocked = res.returncode == 0 + + if not unblocked: + print( + f"⏸ī¸ Notifying blocked state for Buildkite job '{jname}' (ID: {jid})..." + ) + msg = ( + f"⚠ī¸ Remote CI Buildkite Job '{jname}' is blocked!\n\n" + f"Build ID: {build_id} | Job ID: {jid}\n" + f"Log URL: {log_url}\n\n" + f"Action required: Unblock the job using the buildkite-retry-job skill (`retry_buildkite_jobs.py {args.pr} --unblock`)." + ) + subprocess.run( + [ + "agentapi", + "send-message", + "--title=CI Job Blocked", + args.conv_id, + msg, + ] + ) monitored[jkey_blocked] = time.time() with open(state_file, "w") as f: json.dump(monitored, f) @@ -247,7 +282,7 @@ def main(): msg = ( f"⚠ī¸ Remote CI Buildkite Job '{jname}' completed with errors!\n\n" f"Build ID: {build_id} | Job ID: {jid}\n" - f"Log URL: {job.get('log_url', link)}\n\n" + f"Log URL: {log_url}\n\n" f"Action required: Start a subagent to run the analyze-ci-failure skill for '{jname}' with build ID '{build_id}' and job ID '{jid}'." ) subprocess.run(