agents(ci): automatically unblock Buildkite presubmit approval steps

.agents/skills/buildkite-retry-job/SKILL.md

@@ -1,27 +1,42 @@
 ---
 name: buildkite-retry-job
-description: Retry failed Buildkite jobs individually or rebuild full builds
+description: Retry failed Buildkite jobs, unblock manual approval steps, or
+  rebuild full builds
 ---
 
-Use `scripts/retry_buildkite_jobs.py` to retry individual jobs or rebuild full
+Use `scripts/retry_buildkite_jobs.py` to retry individual jobs, unblock manual
+approval steps (such as `.bazelci/presubmit.yml` changes), or rebuild full
 Buildkite builds.
 
-### Retrying Modes
+### Retrying & Unblocking Modes
 
-1. **Job-Level Retry (Default for failed jobs)**:
-   Retries only specific failing jobs via `bk job retry <job_id>`:
+1. **Default Retry & Unblock**:
+   Unblocks any blocked manual approval steps (`bk job unblock <job_id>
+   --no-input`) and retries specific failing jobs (`bk job retry <job_id>`):
    ```bash
-   ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py <pr_number_or_url>
+   ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \
+     <pr_number_or_url>
    ```
 
-2. **Retry Specific Jobs by Name or ID**:
+2. **Unblock Manual Approval Steps (e.g. `.bazelci/presubmit.yml` changes)**:
+   Approves and unblocks blocked manual approval steps without retrying or
+   rebuilding other jobs:
    ```bash
-   ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py <pr_number> --jobs "compile_pip_requirements"
-   ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py --job-id <job_uuid>
+   ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \
+     <pr_number_or_url> --unblock
    ```
 
-3. **Rebuild Full Build**:
+3. **Retry Specific Jobs by Name or ID**:
+   ```bash
+   ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \
+     <pr_number> --jobs "compile_pip_requirements"
+   ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \
+     --job-id <job_uuid>
+   ```
+
+4. **Rebuild Full Build**:
    Re-runs the entire pipeline build from scratch via `bk build rebuild`:
    ```bash
-   ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py <pr_number> --rebuild
+   ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \
+     <pr_number> --rebuild
    ```

.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py

@@ -61,6 +61,20 @@ def retry_job_by_uuid(job_id):
     return subprocess.run(["bk", "job", "retry", job_id])
 
 
+def unblock_manual_jobs(jobs):
+    unblocked = []
+    for j in jobs:
+        jstate = j.get("state", "")
+        is_manual_block = j.get("type") == "manual" or j.get("unblockable") is True
+        jid = j.get("id")
+        if jid and jstate in ("blocked", "blocked_failed") and is_manual_block:
+            jname = j.get("name") or j.get("label") or jid
+            print(f"🔓 Unblocking manual approval job '{jname}' (ID: {jid})...")
+            subprocess.run(["bk", "job", "unblock", jid, "--no-input"])
+            unblocked.append(j)
+    return unblocked
+
+
 def retry_jobs_by_name(pipeline, build_number, job_pattern):
     check_cli("bk", "https://github.com/buildkite/cli")
     cmd = ["bk", "build", "view", str(build_number), "-p", pipeline, "--json"]
@@ -119,6 +133,11 @@ def main():
         action="store_true",
         help="Rebuild the entire build afresh via 'bk build rebuild'",
     )
+    parser.add_argument(
+        "--unblock",
+        action="store_true",
+        help="Unblock manual approval steps (e.g. .bazelci/presubmit.yml changes) via 'bk job unblock'",
+    )
     args = parser.parse_args()
 
     check_cli("bk", "https://github.com/buildkite/cli")
@@ -146,14 +165,32 @@ def main():
 
     if args.job_name:
         retry_jobs_by_name(pipeline, build_number, args.job_name)
+    elif args.unblock:
+        cmd = [
+            "bk",
+            "build",
+            "view",
+            str(build_number),
+            "-p",
+            pipeline,
+            "--json",
+        ]
+        res = subprocess.run(cmd, capture_output=True, text=True, check=True)
+        data = json.loads(res.stdout)
+        unblocked = unblock_manual_jobs(data.get("jobs", []))
+        if not unblocked:
+            print(
+                f"ℹ️ No blocked manual approval jobs found in build #{build_number}.",
+                file=sys.stderr,
+            )
     elif args.rebuild:
         print(f"🚀 Rebuilding entire build #{build_number} for pipeline: {pipeline}")
         res = subprocess.run(
             ["bk", "build", "rebuild", build_number, "-p", pipeline, "--yes"]
         )
         sys.exit(res.returncode)
     else:
-        # Default behavior: try job-level retries of failed jobs if any, otherwise rebuild
+        # Default behavior: unblock manual approval steps and retry failed jobs if any, otherwise rebuild
         cmd = [
             "bk",
             "build",
@@ -166,9 +203,11 @@ def main():
         try:
             res = subprocess.run(cmd, capture_output=True, text=True, check=True)
             data = json.loads(res.stdout)
+            jobs = data.get("jobs", [])
+            unblocked_jobs = unblock_manual_jobs(jobs)
             failed_jobs = [
                 j
-                for j in data.get("jobs", [])
+                for j in jobs
                 if j.get("state") in ("failed", "broken", "timed_out") and j.get("id")
             ]
             if failed_jobs:
@@ -177,7 +216,7 @@ def main():
                     jname = j.get("name") or j.get("label") or jid
                     print(f"🚀 Retrying failed job '{jname}' (ID: {jid})...")
                     subprocess.run(["bk", "job", "retry", jid])
-            else:
+            elif not unblocked_jobs:
                 print(
                     f"🚀 Rebuilding entire build #{build_number} for pipeline: {pipeline}"
                 )

.agents/skills/merge-pr/SKILL.md

@@ -23,7 +23,13 @@ this PR", or monitor its merge):
    - Have the subagent use the **`buildkite-retry-job`** skill
      (`retry_buildkite_jobs.py <pr_number>`) to automatically retry any
      transient network flakes (e.g., HTTP 504 gateway timeouts, downloader
-     errors).
+     errors) and unblock manual approval steps.
+   - **Blocked Builds (`.bazelci/presubmit.yml` Approval)**: When
+     `.bazelci/presubmit.yml` is modified, Buildkite blocks the build until
+     manually approved. Approve and unblock it immediately via
+     `./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py
+     <pr_number> --unblock` (or `bk job unblock <job_id> --no-input`) so CI does
+     not sit blocked indefinitely.
    - **Soft-Failing Jobs**: Experimental Buildkite jobs (e.g. `*rolling*`
      Bazel) are non-blocking soft failures; do not treat them as merge blockers.
    - When the PR is queued, actively discover the merge queue branch via

.agents/skills/monitor-ci-results/SKILL.md

@@ -30,9 +30,14 @@ or when monitoring CI after PR updates:
 1. **Background Polling**: `monitor_remote_ci.py` continuously polls both
    GitHub PR checks and Buildkite workflow executions in the background across
    new commits and CI re-runs.
-2. **Blocked Jobs**: When a Buildkite job or GitHub check is in a blocked state
-   waiting for user confirmation, it dispatches a notification via `agentapi
-   send-message` so the user is alerted to confirm running the job.
+2. **Blocked Jobs (`.bazelci/presubmit.yml` Approval)**: When a Buildkite build
+   is blocked on a manual approval step (e.g., when `.bazelci/presubmit.yml` is
+   modified), `monitor_remote_ci.py` automatically approves and unblocks it via
+   `bk job unblock <job_id> --no-input` so CI does not sit blocked. If
+   automatic unblocking fails, it notifies the conversation via
+   `agentapi send-message`; unblock Buildkite jobs using the
+   `buildkite-retry-job` skill (`retry_buildkite_jobs.py <pr_number>
+   --unblock`).
 3. **Failure Reporting**: When any GitHub check or Buildkite job completes
    with errors, `monitor_remote_ci.py` dispatches a high-priority notification
    message reporting the failed check back to your conversation.

.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py

@@ -4,6 +4,7 @@
 import fcntl
 import json
 import os
+import re
 import subprocess
 import sys
 import time
@@ -43,6 +44,25 @@ def get_buildkite_jobs(build_url):
     if base_url.endswith(".json"):
         base_url = base_url[:-5]
 
+    m = re.search(r"buildkite\.com/([^/]+/[^/]+)/builds/(\d+)", base_url)
+    if m and check_cli("bk"):
+        pipeline, build_number = m.group(1), m.group(2)
+        try:
+            res = subprocess.run(
+                ["bk", "build", "view", build_number, "-p", pipeline, "--json"],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+            data = json.loads(res.stdout)
+            if isinstance(data, dict) and "jobs" in data:
+                return data["jobs"]
+        except Exception as e:
+            print(
+                f"⚠️ Could not fetch Buildkite jobs via bk CLI for {pipeline}#{build_number}: {e}",
+                file=sys.stderr,
+            )
+
     jobs_url = f"{base_url}/data/jobs.json"
     req = urllib.request.Request(jobs_url, headers={"User-Agent": "ci-monitor"})
     try:
@@ -198,14 +218,18 @@ def main():
                 for job in jobs:
                     if job.get("retried_in_job_uuid") or job.get("retried_at"):
                         continue
-                    jname = job.get("name", "unknown_job")
+                    jname = job.get("name") or job.get("label") or "unknown_job"
                     jstate = job.get("state", "unknown")
                     jid = job.get("id", "")
                     jkey = f"bk_{jid}"
+                    log_url = job.get("web_url") or job.get("log_url") or link
 
                     exit_status = job.get("exit_status")
                     is_soft_failed = job.get("soft_failed") is True
                     is_blocked = jstate in ["blocked", "blocked_failed"]
+                    is_manual_block = (
+                        job.get("type") == "manual" or job.get("unblockable") is True
+                    )
                     is_failed = (
                         (
                             jstate in ["failed", "failing"]
@@ -215,27 +239,38 @@ def main():
                         and not is_blocked
                     )
 
-                    if is_blocked:
+                    if is_blocked and is_manual_block:
                         jkey_blocked = f"bk_blocked_{jid}"
                         if jkey_blocked not in monitored:
-                            print(
-                                f"⏸️ Notifying blocked state for Buildkite job '{jname}' (ID: {jid})..."
-                            )
-                            msg = (
-                                f"⚠️ Remote CI Buildkite Job '{jname}' is blocked!\n\n"
-                                f"Build ID: {build_id} | Job ID: {jid}\n"
-                                f"Log URL: {job.get('log_url', link)}\n\n"
-                                f"The user must confirm whether to run the CI jobs."
-                            )
-                            subprocess.run(
-                                [
-                                    "agentapi",
-                                    "send-message",
-                                    "--title=CI Job Blocked",
-                                    args.conv_id,
-                                    msg,
-                                ]
-                            )
+                            unblocked = False
+                            if jid and check_cli("bk"):
+                                print(
+                                    f"🔓 Automatically unblocking Buildkite approval step '{jname}' (ID: {jid})..."
+                                )
+                                res = subprocess.run(
+                                    ["bk", "job", "unblock", jid, "--no-input"]
+                                )
+                                unblocked = res.returncode == 0
+
+                            if not unblocked:
+                                print(
+                                    f"⏸️ Notifying blocked state for Buildkite job '{jname}' (ID: {jid})..."
+                                )
+                                msg = (
+                                    f"⚠️ Remote CI Buildkite Job '{jname}' is blocked!\n\n"
+                                    f"Build ID: {build_id} | Job ID: {jid}\n"
+                                    f"Log URL: {log_url}\n\n"
+                                    f"Action required: Unblock the job using the buildkite-retry-job skill (`retry_buildkite_jobs.py {args.pr} --unblock`)."
+                                )
+                                subprocess.run(
+                                    [
+                                        "agentapi",
+                                        "send-message",
+                                        "--title=CI Job Blocked",
+                                        args.conv_id,
+                                        msg,
+                                    ]
+                                )
                             monitored[jkey_blocked] = time.time()
                             with open(state_file, "w") as f:
                                 json.dump(monitored, f)
@@ -247,7 +282,7 @@ def main():
                         msg = (
                             f"⚠️ Remote CI Buildkite Job '{jname}' completed with errors!\n\n"
                             f"Build ID: {build_id} | Job ID: {jid}\n"
-                            f"Log URL: {job.get('log_url', link)}\n\n"
+                            f"Log URL: {log_url}\n\n"
                             f"Action required: Start a subagent to run the analyze-ci-failure skill for '{jname}' with build ID '{build_id}' and job ID '{jid}'."
                         )
                         subprocess.run(