fix(ci): recover a release whose release-PR merge run failed

.github/workflows/main.yaml

@@ -10,6 +10,9 @@ on:  # yamllint disable-line rule:truthy
   workflow_dispatch:
 
 permissions:
+  # actions: write lets manage-release-pr dispatch a release run whose
+  # release-PR merge run failed (ci/recover_release.sh).
+  actions: write
   contents: write
   packages: write
   pull-requests: write
@@ -50,6 +53,15 @@ jobs:
         uses: actions/checkout@v7
         with:
           fetch-tags: true
+      -
+        # Only the release-PR merge run builds and publishes a release. If
+        # that run fails (as 2.10.0 and 2.11.0 did, on validation), the
+        # manifest version is never tagged and release-please below aborts
+        # on every push. This dispatches the release run for that version.
+        name: Recover a release whose run failed
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: ./ci/recover_release.sh
       -
         # skip-github-release: this only ever manages the version-bump PR
         # (manifest/CHANGELOG.md/mix.exs) - it never creates a tag or
@@ -338,14 +350,32 @@ jobs:
         # keeps finding this PR stuck "pending" forever and refuses to
         # open any future release PR ("There are untagged, merged release
         # PRs outstanding - aborting").
+        # A dispatched run (a manual or recovered release) has no PR in its
+        # event, so it relabels every merged release PR still marked pending.
         name: Mark the release PR as tagged
-        if: github.event_name == 'pull_request'
         env:
           GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
         run: |
-          gh pr edit "${{ github.event.pull_request.number }}" \
-              --remove-label "autorelease: pending" \
-              --add-label "autorelease: tagged"
+          if [ -n "$PR_NUMBER" ]
+          then
+              numbers="$PR_NUMBER"
+          elif ! numbers=$(gh pr list --state merged --label "autorelease: pending" \
+              --head release-please--branches--main --json number --jq '.[].number')
+          then
+              printf 'ERROR: unable to list pending release PRs\n' >&2
+              exit 1
+          fi
+          for number in $numbers
+          do
+              if ! gh pr edit "$number" \
+                  --remove-label "autorelease: pending" \
+                  --add-label "autorelease: tagged"
+              then
+                  printf 'ERROR: unable to relabel release PR #%s\n' "$number" >&2
+                  exit 1
+              fi
+          done
         working-directory: .
 
   container:

ci/recover_release.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Re-runs a release whose run failed. Called by the manage-release-pr job on
+# every push to main.
+#
+# Only the release-PR merge event builds and publishes a release. If that one
+# run fails, the manifest version is never tagged, and release-please refuses
+# to open any later release PR ("There are untagged, merged release PRs
+# outstanding"). This script finds that state and dispatches the release run.
+#
+# Environment:
+#   GH_TOKEN          token for gh; needs actions: write to dispatch
+#   RELEASE_WORKFLOW  workflow file to dispatch (default: main.yaml)
+#   RELEASE_MANIFEST  release-please manifest (default: .release-please-manifest.json)
+
+workflow="${RELEASE_WORKFLOW:-main.yaml}"
+manifest="${RELEASE_MANIFEST:-.release-please-manifest.json}"
+
+if ! version=$(jq -er '."."' "$manifest")
+then
+    printf 'ERROR: unable to read the version from %s\n' "$manifest" >&2
+    exit 1
+fi
+tag="v$version"
+
+if gh release view "$tag" --json tagName >/dev/null 2>&1
+then
+    printf 'Release %s exists; nothing to recover\n' "$tag"
+    exit 0
+fi
+
+# A release-PR merge run (pull_request) or an earlier recovery
+# (workflow_dispatch) may still be building this release. Leave it alone; the
+# next push to main checks again.
+if ! in_flight=$(gh run list --workflow "$workflow" --limit 50 \
+    --json event,status \
+    --jq '[.[] | select(.status != "completed")
+                | select(.event == "pull_request" or .event == "workflow_dispatch")]
+           | length')
+then
+    printf 'ERROR: unable to list %s runs\n' "$workflow" >&2
+    exit 1
+fi
+
+if [ "$in_flight" != "0" ]
+then
+    printf 'Release %s is missing, but %s release-capable run(s) are in progress; not dispatching\n' \
+        "$tag" "$in_flight"
+    exit 0
+fi
+
+if ! gh workflow run "$workflow" --ref main
+then
+    printf 'ERROR: release %s is missing and dispatching %s failed\n' "$tag" "$workflow" >&2
+    exit 1
+fi
+
+printf '::warning::Release %s was missing; dispatched %s to build and publish it\n' "$tag" "$workflow"
+exit 0

test/recover_release_test.exs

@@ -0,0 +1,103 @@
+defmodule RecoverReleaseTest do
+  use ExUnit.Case, async: true
+
+  # ci/recover_release.sh re-dispatches a release whose release-PR merge run
+  # failed. 2.10.0 and 2.11.0 both failed validation on that run, which left
+  # release-please aborting on every later push until the release was
+  # dispatched by hand.
+
+  @script Path.expand("../ci/recover_release.sh", __DIR__)
+
+  @fake_gh """
+  #!/bin/sh
+  printf '%s\\n' "$*" >> "$FAKE_GH_LOG"
+  case "$1 $2" in
+      "release view")
+          [ "$FAKE_RELEASE_EXISTS" = "yes" ] && exit 0
+          exit 1
+          ;;
+      "run list")
+          [ -n "$FAKE_RUN_LIST_FAILS" ] && exit 1
+          printf '%s\\n' "${FAKE_IN_FLIGHT:-0}"
+          ;;
+      "workflow run")
+          [ -n "$FAKE_DISPATCH_FAILS" ] && exit 1
+          exit 0
+          ;;
+  esac
+  """
+
+  setup do
+    dir = tmp_dir!()
+    bin = Path.join(dir, "bin")
+    File.mkdir_p!(bin)
+    gh = Path.join(bin, "gh")
+    File.write!(gh, @fake_gh)
+    File.chmod!(gh, 0o755)
+
+    manifest = Path.join(dir, "manifest.json")
+    File.write!(manifest, ~s({".": "2.11.0"}\n))
+
+    %{dir: dir, bin: bin, manifest: manifest, log: Path.join(dir, "gh.log")}
+  end
+
+  test "does nothing when the manifest version is released", ctx do
+    assert {output, 0} = run(ctx, [{"FAKE_RELEASE_EXISTS", "yes"}])
+    assert output =~ "Release v2.11.0 exists; nothing to recover"
+    assert gh_calls(ctx) == ["release view v2.11.0 --json tagName"]
+  end
+
+  test "dispatches the release run when the version was never released", ctx do
+    assert {output, 0} = run(ctx)
+    assert output =~ "::warning::Release v2.11.0 was missing; dispatched main.yaml"
+    assert List.last(gh_calls(ctx)) == "workflow run main.yaml --ref main"
+  end
+
+  test "waits while a release-capable run is still in progress", ctx do
+    assert {output, 0} = run(ctx, [{"FAKE_IN_FLIGHT", "1"}])
+    assert output =~ "1 release-capable run(s) are in progress; not dispatching"
+    refute Enum.any?(gh_calls(ctx), &String.starts_with?(&1, "workflow run"))
+  end
+
+  test "fails when the dispatch fails", ctx do
+    assert {output, 1} = run(ctx, [{"FAKE_DISPATCH_FAILS", "1"}])
+    assert output =~ "ERROR: release v2.11.0 is missing and dispatching main.yaml failed"
+  end
+
+  test "fails when runs cannot be listed", ctx do
+    assert {output, 1} = run(ctx, [{"FAKE_RUN_LIST_FAILS", "1"}])
+    assert output =~ "ERROR: unable to list main.yaml runs"
+  end
+
+  test "fails when the manifest has no version", ctx do
+    File.write!(ctx.manifest, "{}\n")
+
+    assert {output, 1} = run(ctx)
+    assert output =~ "ERROR: unable to read the version from"
+  end
+
+  defp run(ctx, env \\ []) do
+    base = [
+      {"PATH", "#{ctx.bin}:#{System.get_env("PATH")}"},
+      {"FAKE_GH_LOG", ctx.log},
+      {"RELEASE_MANIFEST", ctx.manifest}
+    ]
+
+    System.cmd(@script, [], env: base ++ env, stderr_to_stdout: true, cd: ctx.dir)
+  end
+
+  defp gh_calls(ctx) do
+    case File.read(ctx.log) do
+      {:ok, log} -> String.split(log, "\n", trim: true)
+      {:error, :enoent} -> []
+    end
+  end
+
+  defp tmp_dir! do
+    nonce = :crypto.strong_rand_bytes(16) |> Base.url_encode64(padding: false)
+    path = Path.join(System.tmp_dir!(), "linear_cli_recover_release_#{nonce}")
+    File.mkdir!(path)
+    on_exit(fn -> File.rm_rf!(path) end)
+    path
+  end
+end