From 345e69ad3602f991f25197774c547dd098331a90 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Mon, 10 Aug 2026 17:43:01 +0300
Subject: [PATCH 01/23] feat(ci): report clusteralerts fired during release
 rollover

The release e2e pipeline had no visibility into alerts of the module
itself: nothing in the repo ever looked at ClusterAlert objects, and the
rules in monitoring/prometheus-rules were only covered by promtool unit
tests.

A dedicated job now watches the nested cluster for the whole test and
upgrade sequence and collects every firing D8Virtualization* alert. A
ClusterAlert object exists only while the alert fires, so the watch polls
instead of taking a single snapshot at the end. It stops on a ConfigMap
marker placed in the nested cluster by a separate always-running job:
runners share no filesystem, and the marker must also appear when
test-new-release never started.

The report job renders a table into the job summary, emits a warning
annotation per alert and exits non-zero, which paints it red in the UI.
Being continue-on-error, it leaves the workflow conclusion successful.
Alerts are attributed to pre-upgrade, upgrade or post-upgrade by
comparing the observation time with the upgrade window, so an alert that
was already firing before the rollover is not mistaken for its result.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../scripts/bash/e2e/report-clusteralerts.sh  | 111 ++++++++++++++++
 .../e2e/signal-clusteralerts-watch-stop.sh    |  39 ++++++
 .../scripts/bash/e2e/watch-clusteralerts.sh   |  91 +++++++++++++
 .../e2e-test-releases-reusable-pipeline.yml   | 122 +++++++++++++++++-
 4 files changed, 358 insertions(+), 5 deletions(-)
 create mode 100644 .github/scripts/bash/e2e/report-clusteralerts.sh
 create mode 100644 .github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
 create mode 100644 .github/scripts/bash/e2e/watch-clusteralerts.sh

diff --git a/.github/scripts/bash/e2e/report-clusteralerts.sh b/.github/scripts/bash/e2e/report-clusteralerts.sh
new file mode 100644
index 0000000000..e0b9d2ac27
--- /dev/null
+++ b/.github/scripts/bash/e2e/report-clusteralerts.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/bash/e2e/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+require_env CLUSTERALERTS_DIR
+
+alerts_dir="${CLUSTERALERTS_DIR:-}"
+alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
+fail_on_alerts="${FAIL_ON_ALERTS:-true}"
+summary_file="${GITHUB_STEP_SUMMARY:-/dev/stdout}"
+
+# The watch runs as a single job and does not know which pipeline phase it is
+# observing, so the phase is derived here from the upgrade timestamps. A zero
+# means the corresponding job never reported one.
+started="${RELEASE_UPGRADE_STARTED_AT:-0}"
+finished="${RELEASE_UPGRADE_FINISHED_AT:-0}"
+[[ "${started}" =~ ^[0-9]+$ ]] || started=0
+[[ "${finished}" =~ ^[0-9]+$ ]] || finished=0
+
+# shellcheck disable=SC2016 # $started and $finished are jq variables, passed in via --argjson
+phase_program='
+def phase_of(upgrade_started; upgrade_finished):
+  if upgrade_started == 0 or .observedAt < upgrade_started
+  then { phase: "pre-upgrade", order: 0 }
+  elif upgrade_finished == 0 or .observedAt < upgrade_finished
+  then { phase: "upgrade", order: 1 }
+  else { phase: "post-upgrade", order: 2 }
+  end;
+map(. + phase_of($started; $finished))
+| unique_by([.phase, .name, .id])
+| sort_by([.order, .name])
+'
+
+shopt -s nullglob
+logs=("${alerts_dir}"/*.jsonl)
+shopt -u nullglob
+
+if [ "${#logs[@]}" -eq 0 ]; then
+  echo "[WARN] No ClusterAlerts logs found in ${alerts_dir}"
+  alerts='[]'
+else
+  echo "[INFO] Reading collected ClusterAlerts from: ${logs[*]}"
+  echo "[INFO] Upgrade window: started_at=${started}, finished_at=${finished}"
+  alerts="$(jq -s \
+    --argjson started "${started}" \
+    --argjson finished "${finished}" \
+    "${phase_program}" "${logs[@]}")"
+fi
+
+count="$(jq 'length' <<< "${alerts}")"
+
+# Alert summaries are markdown ending with a newline; flatten them for the
+# table and for annotations.
+oneline='def oneline: gsub("\\s+"; " ") | sub("^ "; "") | sub(" $"; "");'
+
+{
+  echo "## ClusterAlerts in the nested cluster"
+  echo
+} >> "${summary_file}"
+
+if [ "${count}" -eq 0 ]; then
+  echo "No \`${alert_prefix}*\` alerts were firing during the release rollover." >> "${summary_file}"
+  echo "[INFO] No ${alert_prefix}* alerts were firing during the release rollover"
+  exit 0
+fi
+
+{
+  echo "| Phase | Alert | Severity | First seen | Summary |"
+  echo "|---|---|---|---|---|"
+  jq -r "${oneline}"' .[] | "| \(.phase) | \(.name) | \(.severityLevel) | \(.firstSeen) | \(.summary | oneline | gsub("\\|"; "\\|")) |"' <<< "${alerts}"
+  echo
+  echo "<details><summary>Alert details</summary>"
+  echo
+  jq -r '.[] | "#### \(.name) — \(.phase)\n\n- severity level: \(.severityLevel)\n- labels: `\(.labels | tojson)`\n\n\(.description)\n"' <<< "${alerts}"
+  echo "</details>"
+} >> "${summary_file}"
+
+# Annotations put the alerts on top of the run page, not only in the summary.
+jq -r "${oneline}"' .[] | "::warning title=ClusterAlert \(.name)::[\(.phase)] \(.summary | oneline)"' <<< "${alerts}"
+
+echo "[INFO] Firing alerts:"
+jq -r '.[] | "  [\(.phase)] \(.name) (severity \(.severityLevel))"' <<< "${alerts}"
+
+if [ "${fail_on_alerts}" != "true" ]; then
+  echo "[INFO] FAIL_ON_ALERTS is not 'true', not failing the job"
+  exit 0
+fi
+
+# Failing here is what paints this job red; the job itself is
+# continue-on-error, so the workflow conclusion stays successful.
+echo "[ERROR] ${count} ClusterAlert(s) were firing in the nested cluster, see the job summary" >&2
+trap - ERR
+exit 1
diff --git a/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh b/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
new file mode 100644
index 0000000000..05029e67a9
--- /dev/null
+++ b/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Tells the ClusterAlerts watch that the pipeline is over. The marker is a
+# ConfigMap in the nested cluster because the watch runs on its own runner and
+# runners share no filesystem.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/bash/e2e/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
+stop_configmap="${WATCH_STOP_CONFIGMAP:-e2e-clusteralerts-watch-stop}"
+
+echo "[INFO] Creating stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
+
+# Never fail the pipeline over the marker: if it cannot be created, the watch
+# ends on its own timeout instead.
+if kubectl -n "${stop_namespace}" create configmap "${stop_configmap}" \
+  --from-literal=run_id="${GITHUB_RUN_ID:-unknown}"; then
+  echo "[INFO] Stop marker created"
+else
+  echo "[WARN] Failed to create the stop marker, the watch will end on its own timeout"
+fi
diff --git a/.github/scripts/bash/e2e/watch-clusteralerts.sh b/.github/scripts/bash/e2e/watch-clusteralerts.sh
new file mode 100644
index 0000000000..50cfc86b89
--- /dev/null
+++ b/.github/scripts/bash/e2e/watch-clusteralerts.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Collects firing ClusterAlerts of the virtualization module from the cluster
+# the current kubeconfig points at, until the stop marker appears in that same
+# cluster (see signal-clusteralerts-watch-stop.sh) or the timeout is reached.
+#
+# The marker lives in the cluster rather than on disk on purpose: the watch runs
+# on its own runner, and runners share no filesystem.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/bash/e2e/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+require_env CLUSTERALERTS_LOG
+
+alerts_log="${CLUSTERALERTS_LOG:-}"
+alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
+poll_interval="${POLL_INTERVAL:-15}"
+# Keep this below the job timeout, otherwise the job is cancelled before the
+# collected alerts can be uploaded.
+timeout_seconds="${TIMEOUT_SECONDS:-17400}"
+stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
+stop_configmap="${WATCH_STOP_CONFIGMAP:-e2e-clusteralerts-watch-stop}"
+
+mkdir -p "$(dirname -- "${alerts_log}")"
+: > "${alerts_log}"
+
+deadline=$(( $(date +%s) + timeout_seconds ))
+
+# A marker left over from a previous run against the same cluster (a re-run of
+# failed jobs, for example) would stop the watch immediately.
+kubectl -n "${stop_namespace}" delete configmap "${stop_configmap}" --ignore-not-found
+
+echo "[INFO] Watching ClusterAlerts matching '${alert_prefix}*'"
+echo "[INFO] Poll interval: ${poll_interval}s, watch timeout: ${timeout_seconds}s, log: ${alerts_log}"
+echo "[INFO] Stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
+
+# A ClusterAlert object exists only while the alert is firing, so the log
+# accumulates one line per poll per firing alert. Deduplication and the split
+# into pipeline phases happen in report-clusteralerts.sh.
+while [ "$(date +%s)" -lt "${deadline}" ]; do
+  if kubectl -n "${stop_namespace}" get configmap "${stop_configmap}" >/dev/null 2>&1; then
+    echo "[INFO] Stop marker found, ending the watch"
+    exit 0
+  fi
+
+  # The nested API server can blink while the module is being rolled over,
+  # so a failed poll must never end the watch.
+  if ! snapshot="$(kubectl get clusteralerts -o json 2>&1)"; then
+    echo "[WARN] Failed to read ClusterAlerts, retrying in ${poll_interval}s: ${snapshot}"
+    sleep "${poll_interval}"
+    continue
+  fi
+
+  printf '%s' "${snapshot}" | jq -c \
+    --arg prefix "${alert_prefix}" \
+    --argjson observedAt "$(date +%s)" \
+    '.items[]
+     | select((.alert.name // "") | startswith($prefix))
+     | {
+         observedAt: $observedAt,
+         name: .alert.name,
+         severityLevel: (.alert.severityLevel // ""),
+         summary: (.alert.summary // ""),
+         description: (.alert.description // ""),
+         labels: (.alert.labels // {}),
+         id: .metadata.name,
+         firstSeen: (.status.startsAt // .metadata.creationTimestamp // "")
+       }' >> "${alerts_log}" \
+    || echo "[WARN] Failed to parse the ClusterAlerts snapshot, skipping this poll"
+
+  sleep "${poll_interval}"
+done
+
+echo "[WARN] Watch timeout of ${timeout_seconds}s reached before the stop marker appeared"
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index 649b84c161..ce314c0066 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -145,6 +145,7 @@ env:
   K8S_VERSION: ${{ inputs.cluster_config_k8s_version }}
   STORAGE_TYPE: ${{ inputs.storage_type }}
   E2E_START_TIME: ${{ inputs.date_start }}
+  CLUSTERALERTS_DIR: ${{ github.workspace }}/clusteralerts
 
 defaults:
   run:
@@ -536,6 +537,7 @@ jobs:
         with:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
           check-api: "false"
+
       - name: Authenticate go module fetches to the fox kubevirt mirror
         env:
           FOX_TOKEN: ${{ secrets.FOX_TOKEN }}
@@ -600,6 +602,7 @@ jobs:
       - test-current-release
     outputs:
       upgrade_started_at: ${{ steps.patch-modulepulloverride.outputs.upgrade_started_at }}
+      upgrade_finished_at: ${{ steps.upgrade-finished.outputs.upgrade_finished_at }}
     steps:
       - uses: actions/checkout@v6
 
@@ -614,6 +617,7 @@ jobs:
         with:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
           check-api: "false"
+
       - name: Show current MPO state
         run: |
           echo "[INFO] Current ModulePullOverride before patching:"
@@ -655,6 +659,13 @@ jobs:
           RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }}
         run: bash "${E2E_SCRIPT_DIR}/wait-vmops-migration-terminal.sh"
 
+      # Closes the upgrade window for the ClusterAlerts report; runs even on
+      # failure so alerts are still attributed to the right phase.
+      - name: Mark the end of the upgrade window
+        id: upgrade-finished
+        if: always()
+        run: echo "upgrade_finished_at=$(date +%s)" >> "$GITHUB_OUTPUT"
+
   test-new-release:
     name: "E2E test (new-release: ${{ inputs.new_release }})"
     runs-on: ubuntu-latest
@@ -671,6 +682,12 @@ jobs:
           checkout: "false"
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Setup kubeconfig
+        uses: ./.github/actions/use-nested-kubeconfig
+        with:
+          kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
+          check-api: "false"
+
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
@@ -700,11 +717,6 @@ jobs:
           echo "Download dependencies"
           go mod download
 
-      - name: Setup kubeconfig
-        uses: ./.github/actions/use-nested-kubeconfig
-        with:
-          kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
-          check-api: "false"
       - name: "Run E2E tests on new-release"
         env:
           NEW_RELEASE: ${{ env.NEW_RELEASE }}
@@ -715,3 +727,103 @@ jobs:
           RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }}
           RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
         run: bash "${E2E_SCRIPT_DIR}/run-release-e2e.sh"
+
+  # Runs alongside the whole test and upgrade sequence: starts together with
+  # test-current-release and ends when signal-watch-stop places the marker in
+  # the nested cluster.
+  watch-clusteralerts:
+    name: Watch ClusterAlerts in nested cluster
+    runs-on: ubuntu-latest
+    needs:
+      - bootstrap
+      - configure-virtualization
+    # Safety net only: the watch itself stops earlier, on its own timeout, so
+    # that the collected alerts are still uploaded.
+    timeout-minutes: 300
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup E2E toolchain
+        uses: ./.github/actions/setup-e2e-toolchain
+        with:
+          checkout: "false"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup kubeconfig
+        uses: ./.github/actions/use-nested-kubeconfig
+        with:
+          kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
+          check-api: "false"
+
+      - name: Watch ClusterAlerts
+        env:
+          CLUSTERALERTS_LOG: ${{ env.CLUSTERALERTS_DIR }}/clusteralerts.jsonl
+        run: bash "${E2E_SCRIPT_DIR}/watch-clusteralerts.sh"
+
+      - name: Upload collected ClusterAlerts
+        uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: clusteralerts-${{ github.run_id }}
+          path: ${{ env.CLUSTERALERTS_DIR }}/*.jsonl
+          if-no-files-found: ignore
+          retention-days: 3
+
+  # Separate job rather than a final step of test-new-release: it must also run
+  # when that job never started because an earlier one failed, otherwise the
+  # watch would keep polling until its timeout.
+  signal-watch-stop:
+    name: Stop the ClusterAlerts watch
+    runs-on: ubuntu-latest
+    needs:
+      - bootstrap
+      - test-new-release
+    # Without a bootstrapped cluster there is no watch to stop and no
+    # kubeconfig to reach it with.
+    if: always() && needs.bootstrap.result == 'success'
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup E2E toolchain
+        uses: ./.github/actions/setup-e2e-toolchain
+        with:
+          checkout: "false"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup kubeconfig
+        uses: ./.github/actions/use-nested-kubeconfig
+        with:
+          kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
+          check-api: "false"
+
+      - name: Signal the ClusterAlerts watch to stop
+        run: bash "${E2E_SCRIPT_DIR}/signal-clusteralerts-watch-stop.sh"
+
+  report-clusteralerts:
+    name: ClusterAlerts in nested cluster
+    runs-on: ubuntu-latest
+    needs:
+      - patch-modulepulloverride
+      - watch-clusteralerts
+    if: always()
+    # A firing alert must be visible but must not fail the pipeline: this job
+    # exits non-zero (turning red in the UI) while the workflow conclusion
+    # stays successful.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Download collected ClusterAlerts
+        uses: actions/download-artifact@v8
+        # Nothing to download when the pipeline failed before the watch ran;
+        # that must not be mistaken for a firing alert.
+        continue-on-error: true
+        with:
+          name: clusteralerts-${{ github.run_id }}
+          path: ${{ env.CLUSTERALERTS_DIR }}
+
+      - name: Report ClusterAlerts
+        env:
+          RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
+          RELEASE_UPGRADE_FINISHED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_finished_at }}
+        run: bash "${E2E_SCRIPT_DIR}/report-clusteralerts.sh"

From 3a59b3a8aa60229313028d73877682033d19146a Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Mon, 10 Aug 2026 18:40:49 +0300
Subject: [PATCH 02/23] fix(ci): wait for the whole rollover before reporting
 clusteralerts

The report job only depended on the upgrade job and on the watch, so in
the run graph it sat next to the new-release tests instead of after them.
Worse, if the watch job died early - on the kubeconfig step, say - the
report started while the new-release tests were still running and
declared that no alerts had been firing.

It now depends on the test jobs as well, and an empty result is reported
as "not monitored" instead of "nothing was firing" when the watch job did
not succeed.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/report-clusteralerts.sh          | 8 ++++++++
 .github/workflows/e2e-test-releases-reusable-pipeline.yml | 6 ++++++
 2 files changed, 14 insertions(+)

diff --git a/.github/scripts/bash/e2e/report-clusteralerts.sh b/.github/scripts/bash/e2e/report-clusteralerts.sh
index e0b9d2ac27..412ae8759e 100644
--- a/.github/scripts/bash/e2e/report-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/report-clusteralerts.sh
@@ -25,6 +25,7 @@ require_env CLUSTERALERTS_DIR
 alerts_dir="${CLUSTERALERTS_DIR:-}"
 alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
 fail_on_alerts="${FAIL_ON_ALERTS:-true}"
+watch_result="${WATCH_RESULT:-}"
 summary_file="${GITHUB_STEP_SUMMARY:-/dev/stdout}"
 
 # The watch runs as a single job and does not know which pipeline phase it is
@@ -77,6 +78,13 @@ oneline='def oneline: gsub("\\s+"; " ") | sub("^ "; "") | sub(" $"; "");'
 } >> "${summary_file}"
 
 if [ "${count}" -eq 0 ]; then
+  # An empty report means "nothing was firing" only if the watch actually ran.
+  if [ -n "${watch_result}" ] && [ "${watch_result}" != "success" ]; then
+    echo "The watch job did not complete (result: \`${watch_result}\`), so alerts were **not** monitored." >> "${summary_file}"
+    echo "::warning title=ClusterAlerts were not monitored::The watch job result is '${watch_result}'"
+    exit 0
+  fi
+
   echo "No \`${alert_prefix}*\` alerts were firing during the release rollover." >> "${summary_file}"
   echo "[INFO] No ${alert_prefix}* alerts were firing during the release rollover"
   exit 0
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index ce314c0066..2d88a44cf6 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -802,8 +802,13 @@ jobs:
   report-clusteralerts:
     name: ClusterAlerts in nested cluster
     runs-on: ubuntu-latest
+    # Depends on the test jobs as well, not only on the watch: should the watch
+    # job die early, the report must still wait for the whole rollover instead
+    # of declaring "no alerts" while the tests are still running.
     needs:
+      - test-current-release
       - patch-modulepulloverride
+      - test-new-release
       - watch-clusteralerts
     if: always()
     # A firing alert must be visible but must not fail the pipeline: this job
@@ -826,4 +831,5 @@ jobs:
         env:
           RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
           RELEASE_UPGRADE_FINISHED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_finished_at }}
+          WATCH_RESULT: ${{ needs.watch-clusteralerts.result }}
         run: bash "${E2E_SCRIPT_DIR}/report-clusteralerts.sh"

From 3eeef5f5b885ee8ee056fe209ee9cc54bd2dc55a Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Mon, 10 Aug 2026 19:22:09 +0300
Subject: [PATCH 03/23] chore(ci): temporary tweaks for testing the
 clusteralerts report

Pin the release-upgrade defaults to the versions under test and delete the
nested cluster right after the ClusterAlerts report, so a test run does not
hold the cluster until the nightly cleanup.

Drop this commit before merge.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../e2e-test-releases-reusable-pipeline.yml   | 26 +++++++++++++++++++
 .github/workflows/e2e-test-releases.yml       |  4 +--
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index 2d88a44cf6..c5ccf60c65 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -833,3 +833,29 @@ jobs:
           RELEASE_UPGRADE_FINISHED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_finished_at }}
           WATCH_RESULT: ${{ needs.watch-clusteralerts.result }}
         run: bash "${E2E_SCRIPT_DIR}/report-clusteralerts.sh"
+
+  # TEMPORARY: drop before merge. Frees the nested cluster right after the
+  # ClusterAlerts report instead of waiting for the nightly cleanup.
+  delete-nested-cluster:
+    name: Delete nested cluster (temporary)
+    runs-on: ubuntu-latest
+    # report-clusteralerts already waits for the whole test and upgrade
+    # sequence, so this runs once nothing needs the cluster any more.
+    needs:
+      - bootstrap
+      - report-clusteralerts
+    if: always()
+    steps:
+      - name: Configure kubectl via azure/k8s-set-context@v4
+        uses: azure/k8s-set-context@v4
+        with:
+          method: kubeconfig
+          context: e2e-cluster-nightly-e2e-virt-sa
+          kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }}
+
+      - name: Delete the namespace and the cluster-scoped VirtualMachineClass
+        env:
+          NAMESPACE: ${{ needs.bootstrap.outputs.namespace }}
+        run: |
+          kubectl delete namespace "${NAMESPACE}" --timeout=300s || true
+          kubectl delete vmclass "${NAMESPACE}-cpu" --timeout=300s || true
diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml
index b209a88abd..dc0cdfae03 100644
--- a/.github/workflows/e2e-test-releases.yml
+++ b/.github/workflows/e2e-test-releases.yml
@@ -20,12 +20,12 @@ on:
       current-release:
         description: "Current release tag like v1.4.1, or PR reference like pr2034/2034"
         required: false # before merge to main, set to true
-        default: "v1.6.3-rc.3" # before merge to main, remove
+        default: "v1.9.6" # before merge to main, remove
         type: string
       next-release:
         description: "Next release like v1.5.0, or PR reference like pr2034/2034"
         required: false # before merge to main, set to true
-        default: "v1.7.0" # before merge to main, remove
+        default: "v1.9.7-rc.0" # before merge to main, remove
         type: string
       enableBuild:
         description: "Build release images before E2E tests"

From f3ab7eb46c4300adc816d5a2d6589520552646db Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Mon, 10 Aug 2026 23:30:06 +0300
Subject: [PATCH 04/23] fix(ci): pick virtualization feature gates by release
 version

The release e2e config hardcoded a feature gate that the 1.9 release line
never shipped, so ModulePullOverride validation failed and the module was
never installed: the pipeline only reported that it timed out waiting for
virtualization to become ready.

Gates are now derived from the release under test. The upgrade narrows the
list to what both releases support before switching the image tag, then
enables everything the new release supports once its images are running, so
each release is tested with all the gates it has.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/common.sh            | 22 +++++++
 .../e2e/configure-virtualization-release.sh   | 11 +++-
 .../e2e/patch-virtualization-feature-gates.sh | 58 +++++++++++++++++++
 .../e2e-test-releases-reusable-pipeline.yml   | 12 ++++
 4 files changed, 100 insertions(+), 3 deletions(-)
 create mode 100644 .github/scripts/bash/e2e/patch-virtualization-feature-gates.sh

diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index 245eb8c877..e9189e974e 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -47,6 +47,28 @@ modules_repo_for_registry() {
   fi
 }
 
+# Echoes the virtualization feature gates supported by every given release, one
+# per line. A gate the pulled module does not know fails ModulePullOverride
+# validation and leaves the module uninstalled, so a gate is only listed when
+# none of the releases predate it. In-place resize never shipped in the 1.9
+# line. Anything that is not a release tag (a PR reference, a build off main)
+# carries the current gates.
+# Usage: virtualization_feature_gates [release]...
+virtualization_feature_gates() {
+  local release
+
+  echo "HotplugCPUWithLiveMigration"
+  echo "HotplugMemoryWithLiveMigration"
+
+  for release in "$@"; do
+    if [[ "${release}" =~ ^v([0-9]+)\.([0-9]+)\. ]] && (( BASH_REMATCH[1] == 1 && BASH_REMATCH[2] < 10 )); then
+      return 0
+    fi
+  done
+
+  echo "HotplugCPUAndMemoryWithInPlaceResize"
+}
+
 # Reads a manifest from stdin and applies it with retries.
 # Usage: kubectl_apply_with_retry [count] [delay] [diag_fn]
 # diag_fn is an optional function name invoked on each failed attempt.
diff --git a/.github/scripts/bash/e2e/configure-virtualization-release.sh b/.github/scripts/bash/e2e/configure-virtualization-release.sh
index b39ff8a7ad..d8fc6d86da 100644
--- a/.github/scripts/bash/e2e/configure-virtualization-release.sh
+++ b/.github/scripts/bash/e2e/configure-virtualization-release.sh
@@ -37,6 +37,13 @@ current_release="$(required_env_value CURRENT_RELEASE)"
 
 REGISTRY="$(registry_host_from_docker_cfg "${dev_registry_docker_cfg}")"
 
+# Only the gates this release knows: a gate it does not support fails
+# ModulePullOverride validation and the module never installs. The upgrade
+# revisits the list for the new release (patch-virtualization-feature-gates.sh).
+feature_gates_yaml="$(virtualization_feature_gates "${current_release}" | sed 's/^/      - /')"
+echo "[INFO] Feature gates for ${current_release}:"
+echo "${feature_gates_yaml}"
+
 echo "[INFO] Apply ModuleSource prod config"
 kubectl_apply_with_retry 20 10 show_deckhouse_state <<EOF
 apiVersion: deckhouse.io/v1alpha1
@@ -71,9 +78,7 @@ spec:
     virtualMachineCIDRs:
       - 192.168.10.0/24
     featureGates:
-      - HotplugCPUWithLiveMigration
-      - HotplugMemoryWithLiveMigration
-      - HotplugCPUAndMemoryWithInPlaceResize
+${feature_gates_yaml}
   source: deckhouse-dev
   version: 1
 ---
diff --git a/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh b/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh
new file mode 100644
index 0000000000..bb675a88a2
--- /dev/null
+++ b/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sets the virtualization feature gates of the module config to those supported
+# by every given release, and verifies the result.
+#
+# Usage: patch-virtualization-feature-gates.sh <release>...
+#
+# During a release upgrade this runs twice. Before the image tag is patched it
+# is called with both releases, which drops the gates the new release does not
+# know - otherwise the new module fails validation and never installs. After the
+# upgrade it is called with the new release alone, which enables the gates only
+# that release supports.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/bash/e2e/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+if [ "$#" -eq 0 ]; then
+  echo "[ERROR] Usage: $(basename -- "${BASH_SOURCE[0]}") <release>..." >&2
+  exit 1
+fi
+
+gates_json="$(virtualization_feature_gates "$@" | jq -Rsc 'split("\n") | map(select(length > 0))')"
+current_json="$(kubectl get mc virtualization -o jsonpath='{.spec.settings.featureGates}')"
+
+echo "[INFO] Feature gates supported by $*: ${gates_json}"
+
+if [ "${current_json}" = "${gates_json}" ]; then
+  echo "[INFO] Module config already lists exactly these gates, nothing to patch"
+  exit 0
+fi
+
+echo "[INFO] Patching feature gates: ${current_json:-none} -> ${gates_json}"
+kubectl patch mc virtualization --type merge -p "{\"spec\":{\"settings\":{\"featureGates\":${gates_json}}}}"
+
+patched_json="$(kubectl get mc virtualization -o jsonpath='{.spec.settings.featureGates}')"
+if [ "${patched_json}" != "${gates_json}" ]; then
+  echo "[ERROR] Feature gates were not applied: expected ${gates_json}, got ${patched_json:-none}" >&2
+  exit 1
+fi
+
+echo "[INFO] Feature gates in effect: ${patched_json}"
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index c5ccf60c65..559d34e53e 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -623,6 +623,13 @@ jobs:
           echo "[INFO] Current ModulePullOverride before patching:"
           kubectl get mpo virtualization -o yaml
 
+      # Gates the new release does not know would fail its validation, so they
+      # go before the image tag is switched, while the old module is still live.
+      - name: Drop feature gates the new release does not support
+        run: |
+          bash "${E2E_SCRIPT_DIR}/patch-virtualization-feature-gates.sh" \
+            "${CURRENT_RELEASE}" "${NEW_RELEASE}"
+
       - name: "Patch ModulePullOverride to new-release: ${{ env.NEW_RELEASE }}"
         id: patch-modulepulloverride
         run: |
@@ -647,6 +654,11 @@ jobs:
           NEW_RELEASE: ${{ env.NEW_RELEASE }}
         run: bash "${E2E_SCRIPT_DIR}/verify-image-digests.sh"
 
+      # Now that the new images are running, the gates only that release knows
+      # can be enabled, so the new release is tested with all of them on.
+      - name: Enable every feature gate the new release supports
+        run: bash "${E2E_SCRIPT_DIR}/patch-virtualization-feature-gates.sh" "${NEW_RELEASE}"
+
       - name: Show ModulePullOverride state after upgrade
         run: |
           echo "[INFO] ModulePullOverride after upgrade:"

From da7b5a4bae041469954901ebee7c82425bd84aa8 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 01:08:57 +0300
Subject: [PATCH 05/23] fix(ci): skip the migration wait when the upgrade
 cannot migrate vms

The release pipeline waited for one Evict operation per running virtual
machine after every upgrade, and failed after 20 minutes when none appeared.
Two releases that ship the same virt-handler and virt-launcher never move a
virtual machine, so the wait could not be satisfied and reported a timeout
instead of an upgrade that simply had nothing to migrate.

The expectation is now derived from the workload image digests of both
releases: unchanged images end the step with a message, and anything that
cannot be determined keeps the previous waiting behaviour.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/common.sh            |  9 +++++
 .../scripts/bash/e2e/verify-image-digests.sh  |  3 +-
 .../bash/e2e/wait-vmops-migration-terminal.sh | 37 +++++++++++++++++++
 .../e2e-test-releases-reusable-pipeline.yml   |  1 +
 4 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index e9189e974e..92fc259176 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -69,6 +69,15 @@ virtualization_feature_gates() {
   echo "HotplugCPUAndMemoryWithInPlaceResize"
 }
 
+# Echoes images_digests.json packaged in the module image of a given release.
+# Usage: module_images_digests <module_source> <release>
+module_images_digests() {
+  local module_source="$1"
+  local release="$2"
+
+  crane export "${module_source}/virtualization:${release}" - | tar -Oxf - images_digests.json
+}
+
 # Reads a manifest from stdin and applies it with retries.
 # Usage: kubectl_apply_with_retry [count] [delay] [diag_fn]
 # diag_fn is an optional function name invoked on each failed attempt.
diff --git a/.github/scripts/bash/e2e/verify-image-digests.sh b/.github/scripts/bash/e2e/verify-image-digests.sh
index d161eebfe9..73bd57f07e 100644
--- a/.github/scripts/bash/e2e/verify-image-digests.sh
+++ b/.github/scripts/bash/e2e/verify-image-digests.sh
@@ -33,9 +33,8 @@ required_env_value() {
 new_release="$(required_env_value NEW_RELEASE)"
 dev_module_source="$(required_env_value DEV_MODULE_SOURCE)"
 
-MODULE_IMAGE="${dev_module_source}/virtualization:${new_release}"
 echo "[INFO] Extracting images_digests.json from virtualization:${new_release}"
-images_hash="$(crane export "${MODULE_IMAGE}" - | tar -Oxf - images_digests.json)"
+images_hash="$(module_images_digests "${dev_module_source}" "${new_release}")"
 echo "[INFO] Expected image digests:"
 echo "::group::images_digests.json"
 echo "${images_hash}" | jq .
diff --git a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
index 6eda51d2fc..6eb19395c1 100644
--- a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
+++ b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
@@ -33,6 +33,43 @@ release_namespace="$(required_env_value RELEASE_NAMESPACE)"
 
 sleep_interval="${SLEEP_INTERVAL:-10}"
 timeout_seconds="${TIMEOUT_SECONDS:-1200}"
+# Virtual machines are only moved when the workload images change: a new
+# virt-handler drains the VMs off its node, a new virt-launcher makes the
+# workload updater migrate the running ones. Releases that leave both untouched
+# never trigger a migration, so there would be nothing to wait for.
+migration_expected() {
+  local module_source="${DEV_MODULE_SOURCE:-}"
+  local current="${CURRENT_RELEASE:-}"
+  local new="${NEW_RELEASE:-}"
+  local current_digests new_digests image
+
+  if [ -z "${module_source}" ] || [ -z "${current}" ] || [ -z "${new}" ]; then
+    echo "[WARN] DEV_MODULE_SOURCE, CURRENT_RELEASE or NEW_RELEASE is not set, cannot tell whether the upgrade migrates VMs; waiting anyway"
+    return 0
+  fi
+
+  if ! current_digests="$(module_images_digests "${module_source}" "${current}")" ||
+     ! new_digests="$(module_images_digests "${module_source}" "${new}")"; then
+    echo "[WARN] Failed to read the image digests of ${current} or ${new}, cannot tell whether the upgrade migrates VMs; waiting anyway"
+    return 0
+  fi
+
+  for image in virtHandler virtLauncher; do
+    if [ "$(jq -r --arg i "${image}" '.[$i] // ""' <<< "${current_digests}")" \
+      != "$(jq -r --arg i "${image}" '.[$i] // ""' <<< "${new_digests}")" ]; then
+      echo "[INFO] The ${image} image differs between ${current} and ${new}, virtual machines will be migrated"
+      return 0
+    fi
+  done
+
+  return 1
+}
+
+if ! migration_expected; then
+  echo "[INFO] ${CURRENT_RELEASE} and ${NEW_RELEASE} ship the same virt-handler and virt-launcher: the upgrade does not migrate virtual machines, nothing to wait for"
+  exit 0
+fi
+
 deadline=$(( $(date +%s) + timeout_seconds ))
 
 # The number of Running VMs at the start is the number of Evict VMOPs we
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index 559d34e53e..dc7277d2c3 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -669,6 +669,7 @@ jobs:
       - name: Wait for all migrations to reach a terminal phase
         env:
           RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }}
+          DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
         run: bash "${E2E_SCRIPT_DIR}/wait-vmops-migration-terminal.sh"
 
       # Closes the upgrade window for the ClusterAlerts report; runs even on

From 6fb8388580bda613f8cd4fea3e0c19599537d5b2 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 01:15:10 +0300
Subject: [PATCH 06/23] chore(ci): bump actions off the deprecated node 20
 runtime

Runners force these actions onto Node.js 24 already and annotate every run
with a deprecation warning. Moved each one to its first major built for
Node.js 24: cache v4 to v5, setup-go v5 to v6, github-script v7 to v8,
setup-kubectl and k8s-set-context v4 to v5.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/actions/setup-e2e-toolchain/action.yml       |  4 ++--
 .github/workflows/dev_module_build.yml               | 12 ++++++------
 .github/workflows/dev_validation.yaml                |  8 ++++----
 .github/workflows/e2e-nightly-reusable-pipeline.yml  | 12 ++++++------
 .github/workflows/e2e-nightly.yml                    | 10 +++++-----
 .../e2e-test-releases-reusable-pipeline.yml          | 12 ++++++------
 .github/workflows/e2e-test-releases.yml              |  2 +-
 .../workflows/release_module_release-channels.yml    |  4 ++--
 8 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/.github/actions/setup-e2e-toolchain/action.yml b/.github/actions/setup-e2e-toolchain/action.yml
index 1fb7ac9b91..2a38a65580 100644
--- a/.github/actions/setup-e2e-toolchain/action.yml
+++ b/.github/actions/setup-e2e-toolchain/action.yml
@@ -40,7 +40,7 @@ runs:
 
     - name: Restore d8 cache
       id: d8-cache
-      uses: actions/cache@v4
+      uses: actions/cache@v5
       with:
         path: /opt/deckhouse/bin/d8
         key: d8-${{ inputs.d8-version }}-${{ runner.os }}
@@ -57,7 +57,7 @@ runs:
 
     - name: Install kubectl CLI
       if: inputs.install-kubectl == 'true'
-      uses: azure/setup-kubectl@v4
+      uses: azure/setup-kubectl@v5
 
     - name: Install htpasswd utility
       if: inputs.install-htpasswd == 'true'
diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 12f9bef117..cf570f2629 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -63,7 +63,7 @@ jobs:
     steps:
       - name: Get Pull Request Labels
         id: get-labels
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           script: |
             function processDelveLabels(labelList) {
@@ -212,7 +212,7 @@ jobs:
     name: Run go linter
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -314,7 +314,7 @@ jobs:
     name: Run unit test
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -464,7 +464,7 @@ jobs:
     needs: set_vars
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -644,7 +644,7 @@ jobs:
     steps:
       - name: Create Initial PR Comment
         id: create_comment
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           github-token: ${{secrets.RELEASE_PLEASE_TOKEN}}
           script: |
@@ -696,7 +696,7 @@ jobs:
         uses: ./.github/actions/install-d8
 
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
diff --git a/.github/workflows/dev_validation.yaml b/.github/workflows/dev_validation.yaml
index 9376e293d3..070482c97b 100644
--- a/.github/workflows/dev_validation.yaml
+++ b/.github/workflows/dev_validation.yaml
@@ -58,7 +58,7 @@ jobs:
     name: Validation no-cyrillic
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -83,7 +83,7 @@ jobs:
     name: Validation doc-changes
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -108,7 +108,7 @@ jobs:
     name: Validation go-work
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -220,7 +220,7 @@ jobs:
     steps:
       - name: Setup Go ${{ matrix.components.go-version }}
         if: matrix.components.component != 'vm-route-forge' || needs.paths_filter.outputs.vm_route_forge == 'true'
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: ${{ matrix.components.go-version }}
 
diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml
index 3e44dac1a6..b00559753d 100644
--- a/.github/workflows/e2e-nightly-reusable-pipeline.yml
+++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml
@@ -237,8 +237,8 @@ jobs:
         with:
           docker_cfg: ${{ secrets.REGISTRY_DOCKER_CFG }}
 
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
@@ -584,7 +584,7 @@ jobs:
       - uses: actions/checkout@v6
 
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -696,7 +696,7 @@ jobs:
 
       - name: Determine failed stage and prepare report
         id: determine-stage
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         env:
           STORAGE_TYPE: ${{ inputs.storage_type }}
           PIPELINE_JOB_NAME: ${{ inputs.pipeline_job_name }}
@@ -763,8 +763,8 @@ jobs:
             "$artifact_path"
           unzip -o "${RUNNER_TEMP}/${ARTIFACT_NAME}.zip" -d "${{ env.SETUP_CLUSTER_TYPE_PATH }}"
 
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml
index 8c1a4626b8..65e31c5dc7 100644
--- a/.github/workflows/e2e-nightly.yml
+++ b/.github/workflows/e2e-nightly.yml
@@ -36,8 +36,8 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v6
 
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
@@ -60,8 +60,8 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v6
 
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
@@ -240,7 +240,7 @@ jobs:
 
       - name: Send results to channel
         id: render-report
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         env:
           EXPECTED_STORAGE_TYPES: '["replicated","nfs","ceph"]'
           LOOP_API_BASE_URL: ${{ secrets.LOOP_API_BASE_URL }}
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index dc7277d2c3..5f8fc41cbc 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -191,8 +191,8 @@ jobs:
         with:
           docker_cfg: ${{ secrets.REGISTRY_DOCKER_CFG }}
 
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
@@ -522,7 +522,7 @@ jobs:
       - uses: actions/checkout@v6
 
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -702,7 +702,7 @@ jobs:
           check-api: "false"
 
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -859,8 +859,8 @@ jobs:
       - report-clusteralerts
     if: always()
     steps:
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml
index dc0cdfae03..10e23de868 100644
--- a/.github/workflows/e2e-test-releases.yml
+++ b/.github/workflows/e2e-test-releases.yml
@@ -76,7 +76,7 @@ jobs:
     steps:
       - name: Resolve release refs
         id: resolve
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         env:
           CURRENT_RELEASE_INPUT: ${{ github.event.inputs.current-release }}
           NEXT_RELEASE_INPUT: ${{ github.event.inputs.next-release }}
diff --git a/.github/workflows/release_module_release-channels.yml b/.github/workflows/release_module_release-channels.yml
index a14434a935..d2fe7063bd 100644
--- a/.github/workflows/release_module_release-channels.yml
+++ b/.github/workflows/release_module_release-channels.yml
@@ -142,7 +142,7 @@ jobs:
 
       - name: Set up Go ${{ env.GO_VERSION }}
         if: ${{ !inputs.skip_requirements_check }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -449,7 +449,7 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 

From 158cca9ed8bd0a0e3e60ac984d1d5b5414e552d6 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 11:23:59 +0300
Subject: [PATCH 07/23] fix(ci): run the e2e pipelines on the project go
 version

setup-go v6 pins GOTOOLCHAIN to the requested version instead of letting Go
fetch a newer one, and the pipelines asked for a Go older than the go.work of
the sources they test, so every Go build failed. Both the release and the
nightly pipelines now ask for the version the project builds with, and keep
the toolchain automatic for refs that need another one.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/workflows/e2e-nightly-reusable-pipeline.yml       | 2 ++
 .github/workflows/e2e-nightly.yml                         | 4 ++--
 .github/workflows/e2e-test-releases-reusable-pipeline.yml | 2 ++
 .github/workflows/e2e-test-releases.yml                   | 2 +-
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml
index b00559753d..8c40f4d24d 100644
--- a/.github/workflows/e2e-nightly-reusable-pipeline.yml
+++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml
@@ -187,6 +187,8 @@ env:
   DECKHOUSE_VERSION: ${{ inputs.deckhouse_version }}
   DEFAULT_USER: ${{ inputs.default_user }}
   GO_VERSION: ${{ inputs.go_version }}
+  # setup-go v6 would pin the toolchain; a checked out ref may need a newer one.
+  GOTOOLCHAIN: auto
   SETUP_CLUSTER_TYPE_PATH: test/dvp-static-cluster
   E2E_SCRIPT_DIR: ${{ github.workspace }}/.github/scripts/bash/e2e
   K8S_VERSION: ${{ inputs.cluster_config_k8s_version }}
diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml
index 65e31c5dc7..1547a7e0db 100644
--- a/.github/workflows/e2e-nightly.yml
+++ b/.github/workflows/e2e-nightly.yml
@@ -146,7 +146,7 @@ jobs:
       deckhouse_version: ${{ needs.set-vars.outputs.deckhouse_version }}
       registry_profile: ${{ needs.set-vars.outputs.registry_profile }}
       default_user: cloud
-      go_version: "1.24.13"
+      go_version: "1.25.12"
       e2e_timeout: "3.5h"
       e2e_image_base_url: ${{ needs.set-vars.outputs.e2e_image_base_url }}
       date_start: ${{ needs.set-vars.outputs.date_start }}
@@ -180,7 +180,7 @@ jobs:
       deckhouse_version: ${{ needs.set-vars.outputs.deckhouse_version }}
       registry_profile: ${{ needs.set-vars.outputs.registry_profile }}
       default_user: cloud
-      go_version: "1.24.13"
+      go_version: "1.25.12"
       e2e_timeout: "3.5h"
       e2e_focus_tests: "VirtualDiskProvisioning|VirtualDiskSnapshots|VirtualImageCreation|VirtualDiskResizing|DiskAttachment|BlockDeviceHotplug|Migration|StorageClassMigration|RWOVirtualDiskMigration|VMSOP|Restore|DataExports"
       e2e_image_base_url: ${{ needs.set-vars.outputs.e2e_image_base_url }}
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index 5f8fc41cbc..eb8b102aed 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -140,6 +140,8 @@ env:
   DECKHOUSE_VERSION: ${{ inputs.deckhouse_version }}
   DEFAULT_USER: ${{ inputs.default_user }}
   GO_VERSION: ${{ inputs.go_version }}
+  # setup-go v6 would pin the toolchain; a checked out ref may need a newer one.
+  GOTOOLCHAIN: auto
   SETUP_CLUSTER_TYPE_PATH: test/dvp-static-cluster
   E2E_SCRIPT_DIR: ${{ github.workspace }}/.github/scripts/bash/e2e
   K8S_VERSION: ${{ inputs.cluster_config_k8s_version }}
diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml
index 10e23de868..3018b68250 100644
--- a/.github/workflows/e2e-test-releases.yml
+++ b/.github/workflows/e2e-test-releases.yml
@@ -229,7 +229,7 @@ jobs:
       deckhouse_version: ${{ needs.set-vars.outputs.deckhouse_version }}
       registry_profile: ${{ needs.set-vars.outputs.registry_profile }}
       default_user: cloud
-      go_version: "1.25.8"
+      go_version: "1.25.12"
       date_start: ${{ needs.set-vars.outputs.date_start }}
       randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }}
       cluster_config_workers_memory: "9Gi"

From d36db1e0a21f102cd9312941215ae54f6777c36e Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 12:48:08 +0300
Subject: [PATCH 08/23] delete before merge, ci concurrency - group

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/workflows/e2e-test-releases.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml
index 3018b68250..5555e68de0 100644
--- a/.github/workflows/e2e-test-releases.yml
+++ b/.github/workflows/e2e-test-releases.yml
@@ -34,7 +34,7 @@ on:
         type: boolean
 
 concurrency:
-  group: "${{ github.workflow }}-${{ github.event.inputs.current-release }}-${{ github.event.inputs.next-release || 'no-next' }}"
+  group: "${{ github.workflow }}- ${{ github.ref }}"
   cancel-in-progress: true
 
 defaults:

From 5fe8d2ed9d1da004ff07ff3820995360416dd081 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 15:35:14 +0300
Subject: [PATCH 09/23] fix(ci): skip the migration window check when the
 upgrade cannot migrate vms

The release smoke test compares the iperf window against the migration of
the iperf server, and failed when no migration operation existed at all.
Releases that ship the same virt-handler and virt-launcher never move a
virtual machine, so the pipeline now passes its own verdict to the tests and
the comparison is skipped when there is nothing to compare against. An unset
value still expects a migration, so a missing one keeps failing the test.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../bash/e2e/wait-vmops-migration-terminal.sh    | 10 ++++++++++
 .../e2e-test-releases-reusable-pipeline.yml      |  3 +++
 test/e2e/release/current_release_smoke.go        | 16 ++++++++++++++++
 3 files changed, 29 insertions(+)

diff --git a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
index 6eb19395c1..da14b7520e 100644
--- a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
+++ b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
@@ -65,11 +65,21 @@ migration_expected() {
   return 1
 }
 
+# The verdict is published so the new-release tests can tell a missing migration
+# from one that was never going to happen.
+publish_verdict() {
+  [ -n "${GITHUB_OUTPUT:-}" ] || return 0
+  echo "migrates_vms=$1" >> "${GITHUB_OUTPUT}"
+}
+
 if ! migration_expected; then
+  publish_verdict false
   echo "[INFO] ${CURRENT_RELEASE} and ${NEW_RELEASE} ship the same virt-handler and virt-launcher: the upgrade does not migrate virtual machines, nothing to wait for"
   exit 0
 fi
 
+publish_verdict true
+
 deadline=$(( $(date +%s) + timeout_seconds ))
 
 # The number of Running VMs at the start is the number of Evict VMOPs we
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index eb8b102aed..daeb1c00d8 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -605,6 +605,7 @@ jobs:
     outputs:
       upgrade_started_at: ${{ steps.patch-modulepulloverride.outputs.upgrade_started_at }}
       upgrade_finished_at: ${{ steps.upgrade-finished.outputs.upgrade_finished_at }}
+      migrates_vms: ${{ steps.wait-migrations.outputs.migrates_vms }}
     steps:
       - uses: actions/checkout@v6
 
@@ -669,6 +670,7 @@ jobs:
           kubectl get modules virtualization
 
       - name: Wait for all migrations to reach a terminal phase
+        id: wait-migrations
         env:
           RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }}
           DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
@@ -741,6 +743,7 @@ jobs:
           RELEASE_TEST_PHASE: post-upgrade
           RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }}
           RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
+          RELEASE_UPGRADE_MIGRATES_VMS: ${{ needs.patch-modulepulloverride.outputs.migrates_vms }}
         run: bash "${E2E_SCRIPT_DIR}/run-release-e2e.sh"
 
   # Runs alongside the whole test and upgrade sequence: starts together with
diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go
index 9e6516829a..fb23f86aa6 100644
--- a/test/e2e/release/current_release_smoke.go
+++ b/test/e2e/release/current_release_smoke.go
@@ -42,6 +42,7 @@ const (
 	releaseTestPhasePostUpgrade  = "post-upgrade"
 	releaseUpgradeContextPathEnv = "RELEASE_UPGRADE_CONTEXT_PATH"
 	releaseNamespaceEnv          = "RELEASE_NAMESPACE"
+	releaseUpgradeMigratesVMsEnv = "RELEASE_UPGRADE_MIGRATES_VMS"
 )
 
 var _ = Describe("CurrentReleaseSmoke", func() {
@@ -205,6 +206,14 @@ func (t *currentReleaseSmokeTest) verifyIPerfContinuityAfterUpgrade() {
 	report := getIPerfClientReport(t.framework, t.iperfClient.vm, releaseIPerfReportPath)
 	Expect(isExpectedIPerfReportError(report.Error)).To(BeTrue(), "iperf3 report contains an unexpected error: %q", report.Error)
 
+	if !upgradeMigratesVMs() {
+		By("Skipping the migration window checks: the upgrade does not migrate virtual machines")
+		Expect(report.End.SumSent.Bytes).To(BeNumerically(">", 0), "iperf3 client should send data")
+		Expect(report.End.SumSent.BitsPerSecond).To(BeNumerically(">", 0), "iperf3 client should report throughput")
+
+		return
+	}
+
 	By("Verifying the iperf test brackets the migration window (started before, stopped after)")
 	migration := getMigrationWindow(t.framework, t.iperfServer.vm.Name, t.iperfServer.vm.Namespace)
 
@@ -278,6 +287,13 @@ func getReleaseTestPhase() string {
 	return releaseTestPhasePreUpgrade
 }
 
+// Upgrades between releases that ship the same virt-handler and virt-launcher
+// never move a virtual machine. An unset value means the pipeline could not tell,
+// so a migration is still expected.
+func upgradeMigratesVMs() bool {
+	return os.Getenv(releaseUpgradeMigratesVMsEnv) != "false"
+}
+
 func mustGetEnv(name string) string {
 	value := os.Getenv(name)
 	Expect(value).NotTo(BeEmpty(), "environment variable %s must be set", name)

From a96cd45c093907e89ac4385d341c6838ecc233d8 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 15:35:21 +0300
Subject: [PATCH 10/23] fix(ci): scope the clusteralerts watch stop marker to
 the run attempt

The watch deleted the stop marker on start to ignore leftovers from earlier
runs, which could delete the marker another job had just created for it: the
watch then kept polling until its own timeout hours later, with the report
job waiting on it. The marker name now carries the run and attempt, so
foreign markers are invisible and none has to be deleted.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/common.sh                        | 8 ++++++++
 .../scripts/bash/e2e/signal-clusteralerts-watch-stop.sh   | 2 +-
 .github/scripts/bash/e2e/watch-clusteralerts.sh           | 6 +-----
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index 92fc259176..b3a0eee73a 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -69,6 +69,14 @@ virtualization_feature_gates() {
   echo "HotplugCPUAndMemoryWithInPlaceResize"
 }
 
+# Echoes the name of the ConfigMap that stops the ClusterAlerts watch. The name
+# carries the run attempt so a marker left by another run, or by a previous
+# attempt of this one, can never stop this watch - and this watch never has to
+# delete a marker that another job may have just created for it.
+watch_stop_configmap_name() {
+  printf 'e2e-clusteralerts-watch-stop-%s-%s' "${GITHUB_RUN_ID:-local}" "${GITHUB_RUN_ATTEMPT:-1}"
+}
+
 # Echoes images_digests.json packaged in the module image of a given release.
 # Usage: module_images_digests <module_source> <release>
 module_images_digests() {
diff --git a/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh b/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
index 05029e67a9..e5a672e492 100644
--- a/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
+++ b/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
@@ -25,7 +25,7 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 source "${SCRIPT_DIR}/common.sh"
 
 stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
-stop_configmap="${WATCH_STOP_CONFIGMAP:-e2e-clusteralerts-watch-stop}"
+stop_configmap="$(watch_stop_configmap_name)"
 
 echo "[INFO] Creating stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
 
diff --git a/.github/scripts/bash/e2e/watch-clusteralerts.sh b/.github/scripts/bash/e2e/watch-clusteralerts.sh
index 50cfc86b89..9558bc698b 100644
--- a/.github/scripts/bash/e2e/watch-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/watch-clusteralerts.sh
@@ -36,17 +36,13 @@ poll_interval="${POLL_INTERVAL:-15}"
 # collected alerts can be uploaded.
 timeout_seconds="${TIMEOUT_SECONDS:-17400}"
 stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
-stop_configmap="${WATCH_STOP_CONFIGMAP:-e2e-clusteralerts-watch-stop}"
+stop_configmap="$(watch_stop_configmap_name)"
 
 mkdir -p "$(dirname -- "${alerts_log}")"
 : > "${alerts_log}"
 
 deadline=$(( $(date +%s) + timeout_seconds ))
 
-# A marker left over from a previous run against the same cluster (a re-run of
-# failed jobs, for example) would stop the watch immediately.
-kubectl -n "${stop_namespace}" delete configmap "${stop_configmap}" --ignore-not-found
-
 echo "[INFO] Watching ClusterAlerts matching '${alert_prefix}*'"
 echo "[INFO] Poll interval: ${poll_interval}s, watch timeout: ${timeout_seconds}s, log: ${alerts_log}"
 echo "[INFO] Stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"

From d9f15eef375e673dce0a86c17ad0add4c10ad85a Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 15:48:18 +0300
Subject: [PATCH 11/23] feat(ci): archive the object of every collected
 clusteralert

The report shows a fixed set of fields, and the ClusterAlert itself disappears
from the cluster as soon as the alert stops firing - together with the cluster
the pipeline deletes. Every matching alert is now also archived as the whole
object in YAML, as first seen, next to the collected log.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../scripts/bash/e2e/watch-clusteralerts.sh   | 32 ++++++++++++++++++-
 .../e2e-test-releases-reusable-pipeline.yml   |  2 +-
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/bash/e2e/watch-clusteralerts.sh b/.github/scripts/bash/e2e/watch-clusteralerts.sh
index 9558bc698b..7e67824b3e 100644
--- a/.github/scripts/bash/e2e/watch-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/watch-clusteralerts.sh
@@ -38,7 +38,9 @@ timeout_seconds="${TIMEOUT_SECONDS:-17400}"
 stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
 stop_configmap="$(watch_stop_configmap_name)"
 
-mkdir -p "$(dirname -- "${alerts_log}")"
+alerts_dir="$(dirname -- "${alerts_log}")"
+
+mkdir -p "${alerts_dir}"
 : > "${alerts_log}"
 
 deadline=$(( $(date +%s) + timeout_seconds ))
@@ -47,6 +49,32 @@ echo "[INFO] Watching ClusterAlerts matching '${alert_prefix}*'"
 echo "[INFO] Poll interval: ${poll_interval}s, watch timeout: ${timeout_seconds}s, log: ${alerts_log}"
 echo "[INFO] Stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
 
+# Keeps the whole object of every matching alert, as first seen, next to the log:
+# the ClusterAlert is gone from the cluster once the alert stops firing, and the
+# cluster itself does not outlive the pipeline.
+dump_alert_objects() {
+  local snapshot="$1"
+  local id name dump
+
+  while IFS=$'\t' read -r id name; do
+    [ -n "${id}" ] || continue
+    dump="${alerts_dir}/${name:-clusteralert}-${id}.yaml"
+    [ -e "${dump}" ] && continue
+
+    if ! printf '%s' "${snapshot}" \
+      | jq --arg id "${id}" '.items[] | select(.metadata.name == $id)' \
+      | yq -p=json -o=yaml > "${dump}"; then
+      echo "[WARN] Failed to dump the object of ClusterAlert ${id}"
+      rm -f "${dump}"
+    fi
+  done < <(printf '%s' "${snapshot}" | jq -r \
+    --arg prefix "${alert_prefix}" \
+    '.items[]
+     | select((.alert.name // "") | startswith($prefix))
+     | [.metadata.name, (.alert.name // "")]
+     | @tsv')
+}
+
 # A ClusterAlert object exists only while the alert is firing, so the log
 # accumulates one line per poll per firing alert. Deduplication and the split
 # into pipeline phases happen in report-clusteralerts.sh.
@@ -81,6 +109,8 @@ while [ "$(date +%s)" -lt "${deadline}" ]; do
        }' >> "${alerts_log}" \
     || echo "[WARN] Failed to parse the ClusterAlerts snapshot, skipping this poll"
 
+  dump_alert_objects "${snapshot}"
+
   sleep "${poll_interval}"
 done
 
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index daeb1c00d8..d89843a15f 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -783,7 +783,7 @@ jobs:
         if: always()
         with:
           name: clusteralerts-${{ github.run_id }}
-          path: ${{ env.CLUSTERALERTS_DIR }}/*.jsonl
+          path: ${{ env.CLUSTERALERTS_DIR }}
           if-no-files-found: ignore
           retention-days: 3
 

From ec5eaf989ac7752dca7fb866be74507e63efe916 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 17:38:15 +0300
Subject: [PATCH 12/23] refactor(ci): collect clusteralerts from prometheus in
 a single step

Watching the ClusterAlerts objects took a job that lived as long as the whole
pipeline, a stop marker in the nested cluster to end it, and an artifact round
trip to hand the collected log to the report - because a ClusterAlert exists
only while its alert is active.

The ALERTS series keep that history, so one range query at the end of the
pipeline replaces all of it. The phase now comes from the intersection of the
firing interval with the upgrade window instead of the moment a poll happened
to catch the alert, and pending alerts are reported as well: with a 'for'
clause they are what a short rollover produces. Only a fired alert paints the
job red, or components restarting would make that the norm.

The rendered summary and description are kept: their templates come from the
alerting rules of the same Prometheus and the labels of the series render
them.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../scripts/bash/e2e/collect-clusteralerts.sh | 237 ++++++++++++++++++
 .github/scripts/bash/e2e/common.sh            |   8 -
 .../scripts/bash/e2e/report-clusteralerts.sh  | 112 +++++----
 .../e2e/signal-clusteralerts-watch-stop.sh    |  39 ---
 .../scripts/bash/e2e/watch-clusteralerts.sh   | 117 ---------
 .../e2e-test-releases-reusable-pipeline.yml   | 111 +++-----
 6 files changed, 330 insertions(+), 294 deletions(-)
 create mode 100644 .github/scripts/bash/e2e/collect-clusteralerts.sh
 delete mode 100644 .github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
 delete mode 100644 .github/scripts/bash/e2e/watch-clusteralerts.sh

diff --git a/.github/scripts/bash/e2e/collect-clusteralerts.sh b/.github/scripts/bash/e2e/collect-clusteralerts.sh
new file mode 100644
index 0000000000..7ede3ce77e
--- /dev/null
+++ b/.github/scripts/bash/e2e/collect-clusteralerts.sh
@@ -0,0 +1,237 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Collects the alerts of the virtualization module that were active in the
+# nested cluster during the release rollover. Runs once, at the end of the
+# pipeline, against the nested kubeconfig.
+#
+# Prometheus is the source rather than the ClusterAlerts objects: a ClusterAlert
+# exists only while its alert is active, so a single late look at the API server
+# would see nothing of what happened during the upgrade, while the ALERTS series
+# keep the whole history of the observation window.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/bash/e2e/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+require_env CLUSTERALERTS_DIR
+require_env CLUSTERALERTS_WINDOW_STARTED_AT
+
+alerts_dir="${CLUSTERALERTS_DIR:-}"
+alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
+prometheus_namespace="${PROMETHEUS_NAMESPACE:-d8-monitoring}"
+prometheus_selector="${PROMETHEUS_SELECTOR:-prometheus=main}"
+# Port of the prometheus container inside the pod, discovered below when not
+# pinned. port-forward joins the pod network namespace, so a listener bound to
+# localhost there is reachable too.
+prometheus_port="${PROMETHEUS_PORT:-}"
+local_port="${PROMETHEUS_LOCAL_PORT:-19090}"
+query_step="${QUERY_STEP:-30}"
+ready_attempts="${READY_ATTEMPTS:-30}"
+ready_delay="${READY_DELAY:-2}"
+
+# The window starts when virtualization was configured, not when the pipeline
+# started: before that the module does not exist, and its alerts cannot either.
+window_start="${CLUSTERALERTS_WINDOW_STARTED_AT:-}"
+window_end="$(date +%s)"
+
+# Same semantics as in report-clusteralerts.sh: a zero means the corresponding
+# job never reported a timestamp.
+started="${RELEASE_UPGRADE_STARTED_AT:-0}"
+finished="${RELEASE_UPGRADE_FINISHED_AT:-0}"
+[[ "${started}" =~ ^[0-9]+$ ]] || started=0
+[[ "${finished}" =~ ^[0-9]+$ ]] || finished=0
+
+if [[ ! "${window_start}" =~ ^[0-9]+$ ]] || [ "${window_start}" -eq 0 ]; then
+  echo "[ERROR] CLUSTERALERTS_WINDOW_STARTED_AT must be a unix timestamp, got: '${window_start}'" >&2
+  exit 1
+fi
+
+range_file="${alerts_dir}/clusteralerts-query-range.json"
+rules_file="${alerts_dir}/clusteralerts-rules.json"
+alerts_log="${alerts_dir}/clusteralerts.jsonl"
+port_forward_log="${alerts_dir}/clusteralerts-port-forward.log"
+
+port_forward_pid=""
+
+cleanup() {
+  [ -n "${port_forward_pid}" ] || return 0
+  kill "${port_forward_pid}" 2>/dev/null || true
+  wait "${port_forward_pid}" 2>/dev/null || true
+  port_forward_pid=""
+}
+
+prom_api() {
+  local path="$1"
+  shift
+  curl -sS -f --max-time 120 -G "http://127.0.0.1:${local_port}${path}" "$@"
+}
+
+# port-forward plus curl on the runner, not kubectl exec plus curl in the
+# container: the Prometheus image ships no shell tools to query itself with.
+start_port_forward() {
+  local pod="$1" attempt
+
+  echo "[INFO] Forwarding 127.0.0.1:${local_port} to ${pod}:${prometheus_port} in ${prometheus_namespace}"
+  kubectl -n "${prometheus_namespace}" port-forward \
+    "pod/${pod}" "${local_port}:${prometheus_port}" > "${port_forward_log}" 2>&1 &
+  port_forward_pid=$!
+  trap cleanup EXIT
+
+  # The tunnel needs a moment, and probing the query API instead of /-/ready
+  # checks exactly what the collection is about to use.
+  for ((attempt = 1; attempt <= ready_attempts; attempt++)); do
+    if prom_api /api/v1/query --data-urlencode 'query=1' > /dev/null 2>&1; then
+      echo "[INFO] Prometheus API is reachable after ${attempt} attempt(s)"
+      return 0
+    fi
+
+    if ! kill -0 "${port_forward_pid}" 2>/dev/null; then
+      echo "[ERROR] kubectl port-forward exited early:" >&2
+      cat "${port_forward_log}" >&2 || true
+      return 1
+    fi
+
+    sleep "${ready_delay}"
+  done
+
+  echo "[ERROR] Prometheus API did not become reachable after ${ready_attempts} attempt(s):" >&2
+  cat "${port_forward_log}" >&2 || true
+  return 1
+}
+
+# Annotation templates live in the rules, not in the series, so they are fetched
+# separately and rendered per series below.
+fetch_rules() {
+  if prom_api /api/v1/rules --data-urlencode 'type=alert' > "${rules_file}" &&
+    [ "$(jq -r '.status // ""' "${rules_file}")" = "success" ]; then
+    return 0
+  fi
+
+  # A missing summary must never sink the report: the alert itself is the news.
+  echo "[WARN] Failed to fetch alerting rules, alerts will be reported without a summary"
+  printf '%s\n' '{"status":"error","data":{"groups":[]}}' > "${rules_file}"
+}
+
+templates_program='
+[ .data.groups[]?.rules[]?
+  | select(.type == "alerting")
+  | { key: .name,
+      value: {
+        summary: (.annotations.summary // ""),
+        description: (.annotations.description // "")
+      }
+    }
+] | from_entries
+'
+
+# shellcheck disable=SC2016 # $started, $finished and $templates are jq variables, passed in via --argjson
+records_program='
+def phase_of($t):
+  if $started == 0 or $t < $started then { phase: "pre-upgrade", order: 0 }
+  elif $finished == 0 or $t < $finished then { phase: "upgrade", order: 1 }
+  else { phase: "post-upgrade", order: 2 }
+  end;
+
+# The annotations are Go templates that reference $labels only (verified over
+# monitoring/prometheus-rules), so replacing every label by its value renders
+# them. A reference to a label the series does not carry is left as it is.
+def render($labels):
+  reduce ($labels | to_entries[]) as $l
+    (.; gsub("\\{\\{\\s*\\$labels\\." + $l.key + "\\s*\\}\\}"; $l.value));
+
+[ .data.result[]
+  | . as $series
+  | ($series.metric.alertname // "") as $name
+  | ($series.metric | del(.__name__, .alertname, .alertstate)) as $labels
+  # One record per phase the series touches: an alert that spans the upgrade is
+  # news in every phase it was active in, and grouping the samples by phase
+  # intersects its active interval with the upgrade window.
+  | [ $series.values[] | .[0] | floor | { t: ., ph: phase_of(.) } ]
+  | group_by(.ph.order)
+  | .[]
+  | { phase: .[0].ph.phase,
+      order: .[0].ph.order,
+      name: $name,
+      alertstate: ($series.metric.alertstate // ""),
+      severityLevel: ($series.metric.severity_level // ""),
+      labels: $labels,
+      firstSeen: (map(.t) | min | todate),
+      lastSeen: (map(.t) | max | todate),
+      summary: (($templates[$name].summary // "") | render($labels)),
+      description: (($templates[$name].description // "") | render($labels)),
+      id: ([$name, ($series.metric.alertstate // ""), ($labels | tojson)] | join("|"))
+    }
+]
+| unique_by([.order, .id])
+| sort_by([.order, .name, .alertstate])
+| .[]
+'
+
+mkdir -p "${alerts_dir}"
+
+echo "[INFO] Collecting alerts matching '${alert_prefix}*' from Prometheus in ${prometheus_namespace}"
+echo "[INFO] Observation window: ${window_start}..${window_end} ($(( window_end - window_start ))s), step ${query_step}s"
+echo "[INFO] Upgrade window: started_at=${started}, finished_at=${finished}"
+
+pod="$(kubectl -n "${prometheus_namespace}" get pod \
+  -l "${prometheus_selector}" \
+  --field-selector=status.phase=Running \
+  -o jsonpath='{.items[0].metadata.name}')"
+
+if [ -z "${pod}" ]; then
+  echo "[ERROR] No Running pod matching '${prometheus_selector}' in namespace ${prometheus_namespace}" >&2
+  exit 1
+fi
+
+# Asking the pod which port carries the API beats assuming one: an authenticating
+# sidecar may well be the container that owns 9090 there.
+if [ -z "${prometheus_port}" ]; then
+  prometheus_port="$(kubectl -n "${prometheus_namespace}" get pod "${pod}" \
+    -o jsonpath='{.spec.containers[?(@.name=="prometheus")].ports[?(@.name=="web")].containerPort}' || true)"
+  prometheus_port="${prometheus_port:-9090}"
+fi
+
+start_port_forward "${pod}"
+
+# query_range and not query: an instant query would only see what is still
+# active now, while the report is about what was active during the rollover.
+prom_api /api/v1/query_range \
+  --data-urlencode "query=ALERTS{alertname=~\"${alert_prefix}.*\"}" \
+  --data-urlencode "start=${window_start}" \
+  --data-urlencode "end=${window_end}" \
+  --data-urlencode "step=${query_step}" > "${range_file}"
+
+if [ "$(jq -r '.status // ""' "${range_file}")" != "success" ]; then
+  echo "[ERROR] Prometheus rejected the range query: $(jq -c '.' "${range_file}")" >&2
+  exit 1
+fi
+
+fetch_rules
+
+templates="$(jq "${templates_program}" "${rules_file}")"
+
+jq -c \
+  --argjson started "${started}" \
+  --argjson finished "${finished}" \
+  --argjson templates "${templates}" \
+  "${records_program}" "${range_file}" > "${alerts_log}"
+
+count="$(grep -c . "${alerts_log}" || true)"
+echo "[INFO] Collected ${count} alert record(s) into ${alerts_log}"
+jq -r '"  [\(.phase)] \(.name) \(.alertstate) (\(.firstSeen) .. \(.lastSeen))"' "${alerts_log}"
diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index b3a0eee73a..92fc259176 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -69,14 +69,6 @@ virtualization_feature_gates() {
   echo "HotplugCPUAndMemoryWithInPlaceResize"
 }
 
-# Echoes the name of the ConfigMap that stops the ClusterAlerts watch. The name
-# carries the run attempt so a marker left by another run, or by a previous
-# attempt of this one, can never stop this watch - and this watch never has to
-# delete a marker that another job may have just created for it.
-watch_stop_configmap_name() {
-  printf 'e2e-clusteralerts-watch-stop-%s-%s' "${GITHUB_RUN_ID:-local}" "${GITHUB_RUN_ATTEMPT:-1}"
-}
-
 # Echoes images_digests.json packaged in the module image of a given release.
 # Usage: module_images_digests <module_source> <release>
 module_images_digests() {
diff --git a/.github/scripts/bash/e2e/report-clusteralerts.sh b/.github/scripts/bash/e2e/report-clusteralerts.sh
index 412ae8759e..d202e3125b 100644
--- a/.github/scripts/bash/e2e/report-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/report-clusteralerts.sh
@@ -25,87 +25,91 @@ require_env CLUSTERALERTS_DIR
 alerts_dir="${CLUSTERALERTS_DIR:-}"
 alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
 fail_on_alerts="${FAIL_ON_ALERTS:-true}"
-watch_result="${WATCH_RESULT:-}"
+collect_result="${COLLECT_RESULT:-}"
 summary_file="${GITHUB_STEP_SUMMARY:-/dev/stdout}"
 
-# The watch runs as a single job and does not know which pipeline phase it is
-# observing, so the phase is derived here from the upgrade timestamps. A zero
-# means the corresponding job never reported one.
-started="${RELEASE_UPGRADE_STARTED_AT:-0}"
-finished="${RELEASE_UPGRADE_FINISHED_AT:-0}"
-[[ "${started}" =~ ^[0-9]+$ ]] || started=0
-[[ "${finished}" =~ ^[0-9]+$ ]] || finished=0
-
-# shellcheck disable=SC2016 # $started and $finished are jq variables, passed in via --argjson
-phase_program='
-def phase_of(upgrade_started; upgrade_finished):
-  if upgrade_started == 0 or .observedAt < upgrade_started
-  then { phase: "pre-upgrade", order: 0 }
-  elif upgrade_finished == 0 or .observedAt < upgrade_finished
-  then { phase: "upgrade", order: 1 }
-  else { phase: "post-upgrade", order: 2 }
-  end;
-map(. + phase_of($started; $finished))
-| unique_by([.phase, .name, .id])
-| sort_by([.order, .name])
-'
+{
+  echo "## ClusterAlerts in the nested cluster"
+  echo
+} >> "${summary_file}"
+
+# Renders the "we could not check" verdict: an empty report is only good news
+# when the collection itself succeeded.
+not_collected() {
+  local reason="$1"
+
+  echo "${reason}" >> "${summary_file}"
+  echo "::warning title=ClusterAlerts were not collected::${reason}"
+  exit 0
+}
 
 shopt -s nullglob
 logs=("${alerts_dir}"/*.jsonl)
 shopt -u nullglob
 
+if [ -n "${collect_result}" ] && [ "${collect_result}" != "success" ]; then
+  not_collected "The collection step did not complete (result: \`${collect_result}\`), so alerts were **not** checked."
+fi
+
+# collect-clusteralerts.sh always writes its log, empty or not, so a missing one
+# means the collection never got that far - a step that never ran reports no
+# result at all.
 if [ "${#logs[@]}" -eq 0 ]; then
-  echo "[WARN] No ClusterAlerts logs found in ${alerts_dir}"
-  alerts='[]'
-else
-  echo "[INFO] Reading collected ClusterAlerts from: ${logs[*]}"
-  echo "[INFO] Upgrade window: started_at=${started}, finished_at=${finished}"
-  alerts="$(jq -s \
-    --argjson started "${started}" \
-    --argjson finished "${finished}" \
-    "${phase_program}" "${logs[@]}")"
+  echo "[WARN] No collected ClusterAlerts found in ${alerts_dir}"
+  not_collected "No collected alerts were found, so alerts were **not** checked."
 fi
 
+echo "[INFO] Reading collected ClusterAlerts from: ${logs[*]}"
+# The phase, the state and the firing interval come from the collector; sorting
+# is repeated here only to keep the order stable across several log files.
+alerts="$(jq -s 'sort_by([.order, .name, .alertstate])' "${logs[@]}")"
+
 count="$(jq 'length' <<< "${alerts}")"
 
 # Alert summaries are markdown ending with a newline; flatten them for the
 # table and for annotations.
 oneline='def oneline: gsub("\\s+"; " ") | sub("^ "; "") | sub(" $"; "");'
 
-{
-  echo "## ClusterAlerts in the nested cluster"
-  echo
-} >> "${summary_file}"
-
 if [ "${count}" -eq 0 ]; then
-  # An empty report means "nothing was firing" only if the watch actually ran.
-  if [ -n "${watch_result}" ] && [ "${watch_result}" != "success" ]; then
-    echo "The watch job did not complete (result: \`${watch_result}\`), so alerts were **not** monitored." >> "${summary_file}"
-    echo "::warning title=ClusterAlerts were not monitored::The watch job result is '${watch_result}'"
-    exit 0
-  fi
-
-  echo "No \`${alert_prefix}*\` alerts were firing during the release rollover." >> "${summary_file}"
-  echo "[INFO] No ${alert_prefix}* alerts were firing during the release rollover"
+  echo "No \`${alert_prefix}*\` alerts were active during the release rollover." >> "${summary_file}"
+  echo "[INFO] No ${alert_prefix}* alerts were active during the release rollover"
   exit 0
 fi
 
 {
-  echo "| Phase | Alert | Severity | First seen | Summary |"
-  echo "|---|---|---|---|---|"
-  jq -r "${oneline}"' .[] | "| \(.phase) | \(.name) | \(.severityLevel) | \(.firstSeen) | \(.summary | oneline | gsub("\\|"; "\\|")) |"' <<< "${alerts}"
+  echo "| Phase | Alert | State | Severity | First seen | Summary |"
+  echo "|---|---|---|---|---|---|"
+  jq -r "${oneline}"' .[] | "| \(.phase) | \(.name) | \(.alertstate) | \(.severityLevel) | \(.firstSeen) | \(.summary | oneline | gsub("\\|"; "\\|")) |"' <<< "${alerts}"
   echo
   echo "<details><summary>Alert details</summary>"
   echo
-  jq -r '.[] | "#### \(.name) — \(.phase)\n\n- severity level: \(.severityLevel)\n- labels: `\(.labels | tojson)`\n\n\(.description)\n"' <<< "${alerts}"
+  jq -r '.[] | "#### \(.name) — \(.phase) (\(.alertstate))\n\n- severity level: \(.severityLevel)\n- active: \(.firstSeen) .. \(.lastSeen)\n- labels: `\(.labels | tojson)`\n\n\(.description)\n"' <<< "${alerts}"
   echo "</details>"
 } >> "${summary_file}"
 
 # Annotations put the alerts on top of the run page, not only in the summary.
-jq -r "${oneline}"' .[] | "::warning title=ClusterAlert \(.name)::[\(.phase)] \(.summary | oneline)"' <<< "${alerts}"
-
-echo "[INFO] Firing alerts:"
-jq -r '.[] | "  [\(.phase)] \(.name) (severity \(.severityLevel))"' <<< "${alerts}"
+# A pending alert is a notice rather than a warning: it did not hold long enough
+# to be one.
+jq -r "${oneline}"' .[]
+  | (if .alertstate == "firing" then "::warning" else "::notice" end)
+    + " title=ClusterAlert \(.name)::[\(.phase)/\(.alertstate)] \(.summary | oneline)"' <<< "${alerts}"
+
+echo "[INFO] Active alerts:"
+jq -r '.[] | "  [\(.phase)] \(.name) \(.alertstate) (severity \(.severityLevel))"' <<< "${alerts}"
+
+firing_count="$(jq '[.[] | select(.alertstate == "firing")] | length' <<< "${alerts}")"
+
+# Only a fired alert paints the job red. Components restart during a rollover,
+# so rules with a `for` clause go pending on almost every run: failing on those
+# would make a red job the norm and tell the reviewer nothing.
+if [ "${firing_count}" -eq 0 ]; then
+  {
+    echo
+    echo "No \`${alert_prefix}*\` alert reached the firing state; ${count} were pending only."
+  } >> "${summary_file}"
+  echo "[INFO] No ${alert_prefix}* alert reached the firing state, ${count} were pending only"
+  exit 0
+fi
 
 if [ "${fail_on_alerts}" != "true" ]; then
   echo "[INFO] FAIL_ON_ALERTS is not 'true', not failing the job"
@@ -114,6 +118,6 @@ fi
 
 # Failing here is what paints this job red; the job itself is
 # continue-on-error, so the workflow conclusion stays successful.
-echo "[ERROR] ${count} ClusterAlert(s) were firing in the nested cluster, see the job summary" >&2
+echo "[ERROR] ${firing_count} ClusterAlert(s) were firing in the nested cluster, see the job summary" >&2
 trap - ERR
 exit 1
diff --git a/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh b/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
deleted file mode 100644
index e5a672e492..0000000000
--- a/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2026 Flant JSC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Tells the ClusterAlerts watch that the pipeline is over. The marker is a
-# ConfigMap in the nested cluster because the watch runs on its own runner and
-# runners share no filesystem.
-
-set -Eeuo pipefail
-
-SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=.github/scripts/bash/e2e/common.sh
-source "${SCRIPT_DIR}/common.sh"
-
-stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
-stop_configmap="$(watch_stop_configmap_name)"
-
-echo "[INFO] Creating stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
-
-# Never fail the pipeline over the marker: if it cannot be created, the watch
-# ends on its own timeout instead.
-if kubectl -n "${stop_namespace}" create configmap "${stop_configmap}" \
-  --from-literal=run_id="${GITHUB_RUN_ID:-unknown}"; then
-  echo "[INFO] Stop marker created"
-else
-  echo "[WARN] Failed to create the stop marker, the watch will end on its own timeout"
-fi
diff --git a/.github/scripts/bash/e2e/watch-clusteralerts.sh b/.github/scripts/bash/e2e/watch-clusteralerts.sh
deleted file mode 100644
index 7e67824b3e..0000000000
--- a/.github/scripts/bash/e2e/watch-clusteralerts.sh
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2026 Flant JSC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Collects firing ClusterAlerts of the virtualization module from the cluster
-# the current kubeconfig points at, until the stop marker appears in that same
-# cluster (see signal-clusteralerts-watch-stop.sh) or the timeout is reached.
-#
-# The marker lives in the cluster rather than on disk on purpose: the watch runs
-# on its own runner, and runners share no filesystem.
-
-set -Eeuo pipefail
-
-SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=.github/scripts/bash/e2e/common.sh
-source "${SCRIPT_DIR}/common.sh"
-
-require_env CLUSTERALERTS_LOG
-
-alerts_log="${CLUSTERALERTS_LOG:-}"
-alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
-poll_interval="${POLL_INTERVAL:-15}"
-# Keep this below the job timeout, otherwise the job is cancelled before the
-# collected alerts can be uploaded.
-timeout_seconds="${TIMEOUT_SECONDS:-17400}"
-stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
-stop_configmap="$(watch_stop_configmap_name)"
-
-alerts_dir="$(dirname -- "${alerts_log}")"
-
-mkdir -p "${alerts_dir}"
-: > "${alerts_log}"
-
-deadline=$(( $(date +%s) + timeout_seconds ))
-
-echo "[INFO] Watching ClusterAlerts matching '${alert_prefix}*'"
-echo "[INFO] Poll interval: ${poll_interval}s, watch timeout: ${timeout_seconds}s, log: ${alerts_log}"
-echo "[INFO] Stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
-
-# Keeps the whole object of every matching alert, as first seen, next to the log:
-# the ClusterAlert is gone from the cluster once the alert stops firing, and the
-# cluster itself does not outlive the pipeline.
-dump_alert_objects() {
-  local snapshot="$1"
-  local id name dump
-
-  while IFS=$'\t' read -r id name; do
-    [ -n "${id}" ] || continue
-    dump="${alerts_dir}/${name:-clusteralert}-${id}.yaml"
-    [ -e "${dump}" ] && continue
-
-    if ! printf '%s' "${snapshot}" \
-      | jq --arg id "${id}" '.items[] | select(.metadata.name == $id)' \
-      | yq -p=json -o=yaml > "${dump}"; then
-      echo "[WARN] Failed to dump the object of ClusterAlert ${id}"
-      rm -f "${dump}"
-    fi
-  done < <(printf '%s' "${snapshot}" | jq -r \
-    --arg prefix "${alert_prefix}" \
-    '.items[]
-     | select((.alert.name // "") | startswith($prefix))
-     | [.metadata.name, (.alert.name // "")]
-     | @tsv')
-}
-
-# A ClusterAlert object exists only while the alert is firing, so the log
-# accumulates one line per poll per firing alert. Deduplication and the split
-# into pipeline phases happen in report-clusteralerts.sh.
-while [ "$(date +%s)" -lt "${deadline}" ]; do
-  if kubectl -n "${stop_namespace}" get configmap "${stop_configmap}" >/dev/null 2>&1; then
-    echo "[INFO] Stop marker found, ending the watch"
-    exit 0
-  fi
-
-  # The nested API server can blink while the module is being rolled over,
-  # so a failed poll must never end the watch.
-  if ! snapshot="$(kubectl get clusteralerts -o json 2>&1)"; then
-    echo "[WARN] Failed to read ClusterAlerts, retrying in ${poll_interval}s: ${snapshot}"
-    sleep "${poll_interval}"
-    continue
-  fi
-
-  printf '%s' "${snapshot}" | jq -c \
-    --arg prefix "${alert_prefix}" \
-    --argjson observedAt "$(date +%s)" \
-    '.items[]
-     | select((.alert.name // "") | startswith($prefix))
-     | {
-         observedAt: $observedAt,
-         name: .alert.name,
-         severityLevel: (.alert.severityLevel // ""),
-         summary: (.alert.summary // ""),
-         description: (.alert.description // ""),
-         labels: (.alert.labels // {}),
-         id: .metadata.name,
-         firstSeen: (.status.startsAt // .metadata.creationTimestamp // "")
-       }' >> "${alerts_log}" \
-    || echo "[WARN] Failed to parse the ClusterAlerts snapshot, skipping this poll"
-
-  dump_alert_objects "${snapshot}"
-
-  sleep "${poll_interval}"
-done
-
-echo "[WARN] Watch timeout of ${timeout_seconds}s reached before the stop marker appeared"
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index d89843a15f..a4adf9fecf 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -478,6 +478,8 @@ jobs:
     needs:
       - bootstrap
       - configure-storage
+    outputs:
+      configured_at: ${{ steps.mark-configured.outputs.configured_at }}
     steps:
       - uses: actions/checkout@v6
 
@@ -512,6 +514,13 @@ jobs:
           echo "[INFO] Checking virt-handler pods "
           virt_handler_ready
 
+      # Opens the observation window of the ClusterAlerts report. Runs even on
+      # failure, so a module that never became ready is still observed.
+      - name: Mark the start of the ClusterAlerts observation window
+        id: mark-configured
+        if: always()
+        run: echo "configured_at=$(date +%s)" >> "$GITHUB_OUTPUT"
+
   test-current-release:
     name: "E2E test (current-release: ${{ inputs.current_release }})"
     runs-on: ubuntu-latest
@@ -746,18 +755,22 @@ jobs:
           RELEASE_UPGRADE_MIGRATES_VMS: ${{ needs.patch-modulepulloverride.outputs.migrates_vms }}
         run: bash "${E2E_SCRIPT_DIR}/run-release-e2e.sh"
 
-  # Runs alongside the whole test and upgrade sequence: starts together with
-  # test-current-release and ends when signal-watch-stop places the marker in
-  # the nested cluster.
-  watch-clusteralerts:
-    name: Watch ClusterAlerts in nested cluster
+  report-clusteralerts:
+    name: ClusterAlerts in nested cluster
     runs-on: ubuntu-latest
+    # Waits for the whole test and upgrade sequence: Prometheus is queried once,
+    # at the end, so everything that could fire must have happened by then.
     needs:
       - bootstrap
       - configure-virtualization
-    # Safety net only: the watch itself stops earlier, on its own timeout, so
-    # that the collected alerts are still uploaded.
-    timeout-minutes: 300
+      - test-current-release
+      - patch-modulepulloverride
+      - test-new-release
+    if: always()
+    # A firing alert must be visible but must not fail the pipeline: this job
+    # exits non-zero (turning red in the UI) while the workflow conclusion
+    # stays successful.
+    continue-on-error: true
     steps:
       - uses: actions/checkout@v6
 
@@ -773,10 +786,21 @@ jobs:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
           check-api: "false"
 
-      - name: Watch ClusterAlerts
+      - name: Collect ClusterAlerts
+        id: collect
         env:
-          CLUSTERALERTS_LOG: ${{ env.CLUSTERALERTS_DIR }}/clusteralerts.jsonl
-        run: bash "${E2E_SCRIPT_DIR}/watch-clusteralerts.sh"
+          CLUSTERALERTS_WINDOW_STARTED_AT: ${{ needs.configure-virtualization.outputs.configured_at }}
+          RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
+          RELEASE_UPGRADE_FINISHED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_finished_at }}
+        run: bash "${E2E_SCRIPT_DIR}/collect-clusteralerts.sh"
+
+      - name: Report ClusterAlerts
+        # Runs even when the collection failed: the report is the only place
+        # that says so, and a silent job would read as "no alerts".
+        if: always()
+        env:
+          COLLECT_RESULT: ${{ steps.collect.outcome }}
+        run: bash "${E2E_SCRIPT_DIR}/report-clusteralerts.sh"
 
       - name: Upload collected ClusterAlerts
         uses: actions/upload-artifact@v7
@@ -787,71 +811,6 @@ jobs:
           if-no-files-found: ignore
           retention-days: 3
 
-  # Separate job rather than a final step of test-new-release: it must also run
-  # when that job never started because an earlier one failed, otherwise the
-  # watch would keep polling until its timeout.
-  signal-watch-stop:
-    name: Stop the ClusterAlerts watch
-    runs-on: ubuntu-latest
-    needs:
-      - bootstrap
-      - test-new-release
-    # Without a bootstrapped cluster there is no watch to stop and no
-    # kubeconfig to reach it with.
-    if: always() && needs.bootstrap.result == 'success'
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: Setup E2E toolchain
-        uses: ./.github/actions/setup-e2e-toolchain
-        with:
-          checkout: "false"
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Setup kubeconfig
-        uses: ./.github/actions/use-nested-kubeconfig
-        with:
-          kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
-          check-api: "false"
-
-      - name: Signal the ClusterAlerts watch to stop
-        run: bash "${E2E_SCRIPT_DIR}/signal-clusteralerts-watch-stop.sh"
-
-  report-clusteralerts:
-    name: ClusterAlerts in nested cluster
-    runs-on: ubuntu-latest
-    # Depends on the test jobs as well, not only on the watch: should the watch
-    # job die early, the report must still wait for the whole rollover instead
-    # of declaring "no alerts" while the tests are still running.
-    needs:
-      - test-current-release
-      - patch-modulepulloverride
-      - test-new-release
-      - watch-clusteralerts
-    if: always()
-    # A firing alert must be visible but must not fail the pipeline: this job
-    # exits non-zero (turning red in the UI) while the workflow conclusion
-    # stays successful.
-    continue-on-error: true
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: Download collected ClusterAlerts
-        uses: actions/download-artifact@v8
-        # Nothing to download when the pipeline failed before the watch ran;
-        # that must not be mistaken for a firing alert.
-        continue-on-error: true
-        with:
-          name: clusteralerts-${{ github.run_id }}
-          path: ${{ env.CLUSTERALERTS_DIR }}
-
-      - name: Report ClusterAlerts
-        env:
-          RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
-          RELEASE_UPGRADE_FINISHED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_finished_at }}
-          WATCH_RESULT: ${{ needs.watch-clusteralerts.result }}
-        run: bash "${E2E_SCRIPT_DIR}/report-clusteralerts.sh"
-
   # TEMPORARY: drop before merge. Frees the nested cluster right after the
   # ClusterAlerts report instead of waiting for the nightly cleanup.
   delete-nested-cluster:

From 5d31b25e4ef84f15f938561e224b5408a5a00ec8 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 18:17:11 +0300
Subject: [PATCH 13/23] fix(ci): keep the clusteralerts job red only for a
 fired alert

Collecting in the same job that reports means the job now needs the nested
cluster, and a pipeline that died before the cluster existed painted it red -
which reads as a firing alert. Neither the kubeconfig setup nor the query is
allowed to fail the job any more; the report step turns both into the "not
checked" verdict it already had.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/workflows/e2e-test-releases-reusable-pipeline.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index a4adf9fecf..cce43a7190 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -780,7 +780,11 @@ jobs:
           checkout: "false"
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
+      # Red must mean "an alert fired" and nothing else, so neither a missing
+      # cluster nor a failed query is allowed to fail this job: the report step
+      # turns both into a "not checked" verdict.
       - name: Setup kubeconfig
+        continue-on-error: true
         uses: ./.github/actions/use-nested-kubeconfig
         with:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
@@ -788,6 +792,8 @@ jobs:
 
       - name: Collect ClusterAlerts
         id: collect
+        if: always()
+        continue-on-error: true
         env:
           CLUSTERALERTS_WINDOW_STARTED_AT: ${{ needs.configure-virtualization.outputs.configured_at }}
           RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}

From 76cfea4ad68728919b48a0ea55dd5a85a4f101e8 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 19:51:22 +0300
Subject: [PATCH 14/23] delete before merge: comment destroy cluster

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../e2e-test-releases-reusable-pipeline.yml   | 46 +++++++++----------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index cce43a7190..fffe22cf9b 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -819,26 +819,26 @@ jobs:
 
   # TEMPORARY: drop before merge. Frees the nested cluster right after the
   # ClusterAlerts report instead of waiting for the nightly cleanup.
-  delete-nested-cluster:
-    name: Delete nested cluster (temporary)
-    runs-on: ubuntu-latest
-    # report-clusteralerts already waits for the whole test and upgrade
-    # sequence, so this runs once nothing needs the cluster any more.
-    needs:
-      - bootstrap
-      - report-clusteralerts
-    if: always()
-    steps:
-      - name: Configure kubectl via azure/k8s-set-context@v5
-        uses: azure/k8s-set-context@v5
-        with:
-          method: kubeconfig
-          context: e2e-cluster-nightly-e2e-virt-sa
-          kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }}
-
-      - name: Delete the namespace and the cluster-scoped VirtualMachineClass
-        env:
-          NAMESPACE: ${{ needs.bootstrap.outputs.namespace }}
-        run: |
-          kubectl delete namespace "${NAMESPACE}" --timeout=300s || true
-          kubectl delete vmclass "${NAMESPACE}-cpu" --timeout=300s || true
+  # delete-nested-cluster:
+  #   name: Delete nested cluster (temporary)
+  #   runs-on: ubuntu-latest
+  #   # report-clusteralerts already waits for the whole test and upgrade
+  #   # sequence, so this runs once nothing needs the cluster any more.
+  #   needs:
+  #     - bootstrap
+  #     - report-clusteralerts
+  #   if: always() && 
+  #   steps:
+  #     - name: Configure kubectl via azure/k8s-set-context@v5
+  #       uses: azure/k8s-set-context@v5
+  #       with:
+  #         method: kubeconfig
+  #         context: e2e-cluster-nightly-e2e-virt-sa
+  #         kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }}
+
+  #     - name: Delete the namespace and the cluster-scoped VirtualMachineClass
+  #       env:
+  #         NAMESPACE: ${{ needs.bootstrap.outputs.namespace }}
+  #       run: |
+  #         kubectl delete namespace "${NAMESPACE}" --timeout=300s || true
+  #         kubectl delete vmclass "${NAMESPACE}-cpu" --timeout=300s || true

From f6807386ac58c8e70376459c3bd75ebf1d9244ff Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 21:05:45 +0300
Subject: [PATCH 15/23] fix(ci): pass the alert annotation templates to jq
 through a file

The templates of every alerting rule of a cluster add up to more than the
128 KiB Linux allows for a single command line argument, so passing them with
--argjson failed the collection with E2BIG on the runner while it worked on a
developer machine with a larger limit. They go through a file now.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/collect-clusteralerts.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/bash/e2e/collect-clusteralerts.sh b/.github/scripts/bash/e2e/collect-clusteralerts.sh
index 7ede3ce77e..4be300f071 100644
--- a/.github/scripts/bash/e2e/collect-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/collect-clusteralerts.sh
@@ -64,6 +64,7 @@ fi
 
 range_file="${alerts_dir}/clusteralerts-query-range.json"
 rules_file="${alerts_dir}/clusteralerts-rules.json"
+templates_file="${alerts_dir}/clusteralerts-templates.json"
 alerts_log="${alerts_dir}/clusteralerts.jsonl"
 port_forward_log="${alerts_dir}/clusteralerts-port-forward.log"
 
@@ -140,8 +141,10 @@ templates_program='
 ] | from_entries
 '
 
-# shellcheck disable=SC2016 # $started, $finished and $templates are jq variables, passed in via --argjson
+# shellcheck disable=SC2016 # $started, $finished and $templates_wrapper are jq variables, passed in on the command line
 records_program='
+($templates_wrapper[0] // {}) as $templates
+|
 def phase_of($t):
   if $started == 0 or $t < $started then { phase: "pre-upgrade", order: 0 }
   elif $finished == 0 or $t < $finished then { phase: "upgrade", order: 1 }
@@ -224,12 +227,15 @@ fi
 
 fetch_rules
 
-templates="$(jq "${templates_program}" "${rules_file}")"
+# Through a file rather than --argjson: a cluster carries hundreds of alerting
+# rules, and their annotations do not fit in the 128 KiB Linux allows a single
+# command line argument.
+jq "${templates_program}" "${rules_file}" > "${templates_file}"
 
 jq -c \
   --argjson started "${started}" \
   --argjson finished "${finished}" \
-  --argjson templates "${templates}" \
+  --slurpfile templates_wrapper "${templates_file}" \
   "${records_program}" "${range_file}" > "${alerts_log}"
 
 count="$(grep -c . "${alerts_log}" || true)"

From ecf0fd50b0e25d915afbd9c30775be4efb3b7be1 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 21:44:55 +0300
Subject: [PATCH 16/23] fix(ci): add the gpu feature gate to the release e2e
 config

The GPU gate was missing from the gates an e2e run enables, so nothing covered
it. It ships in no release yet, and a gate the pulled module does not know
fails ModulePullOverride validation, so the gates now come from a list that
carries the release each one shipped in - an empty version meaning only a build
off main or a pull request has it. Adding the next gate is one line.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/common.sh | 56 ++++++++++++++++++++++--------
 1 file changed, 41 insertions(+), 15 deletions(-)

diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index 92fc259176..0dfe7b91d4 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -47,26 +47,52 @@ modules_repo_for_registry() {
   fi
 }
 
-# Echoes the virtualization feature gates supported by every given release, one
-# per line. A gate the pulled module does not know fails ModulePullOverride
-# validation and leaves the module uninstalled, so a gate is only listed when
-# none of the releases predate it. In-place resize never shipped in the 1.9
-# line. Anything that is not a release tag (a PR reference, a build off main)
-# carries the current gates.
+# The feature gates an e2e run enables, each with the release it first shipped
+# in. An empty version marks a gate no release carries yet, so only a build off
+# main or a pull request has it. Keep this list in step with the enum of
+# openapi/config-values.yaml: a gate the pulled module does not know fails
+# ModulePullOverride validation and leaves the module uninstalled.
+VIRTUALIZATION_FEATURE_GATES=(
+  "HotplugCPUWithLiveMigration:v1.0"
+  "HotplugMemoryWithLiveMigration:v1.0"
+  "HotplugCPUAndMemoryWithInPlaceResize:v1.10"
+  "GPU:"
+)
+
+# Tells whether a release ref knows a gate that first shipped in a given version.
+# Anything that is not a release tag - a pull request reference, a build off main
+# - carries every gate the repository has.
+release_knows_feature_gate() {
+  local release="$1"
+  local since="$2"
+  local major minor since_major since_minor
+
+  [[ "${release}" =~ ^v([0-9]+)\.([0-9]+)\. ]] || return 0
+  major="${BASH_REMATCH[1]}"
+  minor="${BASH_REMATCH[2]}"
+
+  [[ "${since}" =~ ^v([0-9]+)\.([0-9]+) ]] || return 1
+  since_major="${BASH_REMATCH[1]}"
+  since_minor="${BASH_REMATCH[2]}"
+
+  (( major > since_major || ( major == since_major && minor >= since_minor ) ))
+}
+
+# Echoes the feature gates every given release supports, one per line.
 # Usage: virtualization_feature_gates [release]...
 virtualization_feature_gates() {
-  local release
+  local entry gate since release
 
-  echo "HotplugCPUWithLiveMigration"
-  echo "HotplugMemoryWithLiveMigration"
+  for entry in "${VIRTUALIZATION_FEATURE_GATES[@]}"; do
+    gate="${entry%%:*}"
+    since="${entry#*:}"
 
-  for release in "$@"; do
-    if [[ "${release}" =~ ^v([0-9]+)\.([0-9]+)\. ]] && (( BASH_REMATCH[1] == 1 && BASH_REMATCH[2] < 10 )); then
-      return 0
-    fi
-  done
+    for release in "$@"; do
+      release_knows_feature_gate "${release}" "${since}" || continue 2
+    done
 
-  echo "HotplugCPUAndMemoryWithInPlaceResize"
+    echo "${gate}"
+  done
 }
 
 # Echoes images_digests.json packaged in the module image of a given release.

From 5944562c89e3685d9668cc71a4c5306bb221f2fd Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Wed, 12 Aug 2026 01:03:56 +0300
Subject: [PATCH 17/23] fix(ci): report clusteralerts without leftover
 annotation templates

Alert annotations are Go templates, and only $labels references were substituted,
so a description built on $value - node evacuation stuck, outdated firmware -
reached the report with the raw template in it. The sample value is not carried
by the ALERTS series, so whatever template action survives the label
substitution now becomes a "?" placeholder.

Drop the error branches that set -e made unreachable: the jsonpath of an empty
item list and curl --fail both abort the script before the check meant to
explain the failure, so the range query now reports its own failure instead.

Drop the environment knobs nothing sets, the FAIL_ON_ALERTS switch a
continue-on-error job has no use for, and the trap - ERR that never fires on an
explicit exit.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../scripts/bash/e2e/collect-clusteralerts.sh | 49 +++++++++----------
 .../scripts/bash/e2e/report-clusteralerts.sh  |  7 ---
 2 files changed, 24 insertions(+), 32 deletions(-)

diff --git a/.github/scripts/bash/e2e/collect-clusteralerts.sh b/.github/scripts/bash/e2e/collect-clusteralerts.sh
index 4be300f071..3217791c15 100644
--- a/.github/scripts/bash/e2e/collect-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/collect-clusteralerts.sh
@@ -30,20 +30,19 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 source "${SCRIPT_DIR}/common.sh"
 
 require_env CLUSTERALERTS_DIR
-require_env CLUSTERALERTS_WINDOW_STARTED_AT
 
 alerts_dir="${CLUSTERALERTS_DIR:-}"
 alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
-prometheus_namespace="${PROMETHEUS_NAMESPACE:-d8-monitoring}"
-prometheus_selector="${PROMETHEUS_SELECTOR:-prometheus=main}"
-# Port of the prometheus container inside the pod, discovered below when not
-# pinned. port-forward joins the pod network namespace, so a listener bound to
+prometheus_namespace="d8-monitoring"
+prometheus_selector="prometheus=main"
+# Port of the prometheus container inside the pod, discovered below.
+# port-forward joins the pod network namespace, so a listener bound to
 # localhost there is reachable too.
-prometheus_port="${PROMETHEUS_PORT:-}"
-local_port="${PROMETHEUS_LOCAL_PORT:-19090}"
-query_step="${QUERY_STEP:-30}"
-ready_attempts="${READY_ATTEMPTS:-30}"
-ready_delay="${READY_DELAY:-2}"
+prometheus_port=""
+local_port="19090"
+query_step="30"
+ready_attempts=30
+ready_delay=2
 
 # The window starts when virtualization was configured, not when the pipeline
 # started: before that the module does not exist, and its alerts cannot either.
@@ -151,12 +150,14 @@ def phase_of($t):
   else { phase: "post-upgrade", order: 2 }
   end;
 
-# The annotations are Go templates that reference $labels only (verified over
-# monitoring/prometheus-rules), so replacing every label by its value renders
-# them. A reference to a label the series does not carry is left as it is.
+# The annotations are Go templates. Every $labels.X reference is substituted by
+# the value the series carries for that label, and whatever template action is
+# left afterwards becomes a "?" placeholder: notably $value, the sample value the
+# ALERTS series does not carry, and references to labels absent from the series.
 def render($labels):
   reduce ($labels | to_entries[]) as $l
-    (.; gsub("\\{\\{\\s*\\$labels\\." + $l.key + "\\s*\\}\\}"; $l.value));
+    (.; gsub("\\{\\{\\s*\\$labels\\." + $l.key + "\\s*\\}\\}"; $l.value))
+  | gsub("\\{\\{[^{}]*\\}\\}"; "?");
 
 [ .data.result[]
   | . as $series
@@ -192,10 +193,12 @@ echo "[INFO] Collecting alerts matching '${alert_prefix}*' from Prometheus in ${
 echo "[INFO] Observation window: ${window_start}..${window_end} ($(( window_end - window_start ))s), step ${query_step}s"
 echo "[INFO] Upgrade window: started_at=${started}, finished_at=${finished}"
 
+# "|| true" so an empty item list, on which the jsonpath itself fails, reaches the
+# explicit error below instead of aborting on the jsonpath error.
 pod="$(kubectl -n "${prometheus_namespace}" get pod \
   -l "${prometheus_selector}" \
   --field-selector=status.phase=Running \
-  -o jsonpath='{.items[0].metadata.name}')"
+  -o jsonpath='{.items[0].metadata.name}' || true)"
 
 if [ -z "${pod}" ]; then
   echo "[ERROR] No Running pod matching '${prometheus_selector}' in namespace ${prometheus_namespace}" >&2
@@ -204,24 +207,20 @@ fi
 
 # Asking the pod which port carries the API beats assuming one: an authenticating
 # sidecar may well be the container that owns 9090 there.
-if [ -z "${prometheus_port}" ]; then
-  prometheus_port="$(kubectl -n "${prometheus_namespace}" get pod "${pod}" \
-    -o jsonpath='{.spec.containers[?(@.name=="prometheus")].ports[?(@.name=="web")].containerPort}' || true)"
-  prometheus_port="${prometheus_port:-9090}"
-fi
+prometheus_port="$(kubectl -n "${prometheus_namespace}" get pod "${pod}" \
+  -o jsonpath='{.spec.containers[?(@.name=="prometheus")].ports[?(@.name=="web")].containerPort}' || true)"
+prometheus_port="${prometheus_port:-9090}"
 
 start_port_forward "${pod}"
 
 # query_range and not query: an instant query would only see what is still
 # active now, while the report is about what was active during the rollover.
-prom_api /api/v1/query_range \
+if ! prom_api /api/v1/query_range \
   --data-urlencode "query=ALERTS{alertname=~\"${alert_prefix}.*\"}" \
   --data-urlencode "start=${window_start}" \
   --data-urlencode "end=${window_end}" \
-  --data-urlencode "step=${query_step}" > "${range_file}"
-
-if [ "$(jq -r '.status // ""' "${range_file}")" != "success" ]; then
-  echo "[ERROR] Prometheus rejected the range query: $(jq -c '.' "${range_file}")" >&2
+  --data-urlencode "step=${query_step}" > "${range_file}"; then
+  echo "[ERROR] Prometheus rejected the range query 'ALERTS{alertname=~\"${alert_prefix}.*\"}' over ${window_start}..${window_end} with step ${query_step}s" >&2
   exit 1
 fi
 
diff --git a/.github/scripts/bash/e2e/report-clusteralerts.sh b/.github/scripts/bash/e2e/report-clusteralerts.sh
index d202e3125b..74977a2096 100644
--- a/.github/scripts/bash/e2e/report-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/report-clusteralerts.sh
@@ -24,7 +24,6 @@ require_env CLUSTERALERTS_DIR
 
 alerts_dir="${CLUSTERALERTS_DIR:-}"
 alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
-fail_on_alerts="${FAIL_ON_ALERTS:-true}"
 collect_result="${COLLECT_RESULT:-}"
 summary_file="${GITHUB_STEP_SUMMARY:-/dev/stdout}"
 
@@ -111,13 +110,7 @@ if [ "${firing_count}" -eq 0 ]; then
   exit 0
 fi
 
-if [ "${fail_on_alerts}" != "true" ]; then
-  echo "[INFO] FAIL_ON_ALERTS is not 'true', not failing the job"
-  exit 0
-fi
-
 # Failing here is what paints this job red; the job itself is
 # continue-on-error, so the workflow conclusion stays successful.
 echo "[ERROR] ${firing_count} ClusterAlert(s) were firing in the nested cluster, see the job summary" >&2
-trap - ERR
 exit 1

From d31d3bfcec4a887f77bbcc97b9d12940535293bf Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Wed, 12 Aug 2026 01:03:57 +0300
Subject: [PATCH 18/23] refactor(ci): state the limit of the vm migration
 heuristic

Identical virt-handler and virt-launcher digests were described as a guarantee
that no virtual machine moves. They are not: a template-only change - new args,
env, resources or tolerations - rolls the virt-handler DaemonSet at unchanged
digests and migrates virtual machines all the same. Say so where the comparison
is made, and note that a wrong verdict is not silent, since test-new-release
then fails on an Evict VMOP still InProgress.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../bash/e2e/wait-vmops-migration-terminal.sh     | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
index da14b7520e..0a00d4aa1a 100644
--- a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
+++ b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
@@ -33,10 +33,13 @@ release_namespace="$(required_env_value RELEASE_NAMESPACE)"
 
 sleep_interval="${SLEEP_INTERVAL:-10}"
 timeout_seconds="${TIMEOUT_SECONDS:-1200}"
-# Virtual machines are only moved when the workload images change: a new
-# virt-handler drains the VMs off its node, a new virt-launcher makes the
-# workload updater migrate the running ones. Releases that leave both untouched
-# never trigger a migration, so there would be nothing to wait for.
+# A heuristic over the workload images: a new virt-handler drains the VMs off its
+# node, a new virt-launcher makes the workload updater migrate the running ones,
+# so an upgrade that changes neither digest usually moves nothing.
+# It does not see template-only changes: new args, env, resources or tolerations
+# roll the virt-handler DaemonSet at unchanged digests and migrate VMs all the
+# same. A wrong "no migration" verdict is not silent - the wait is skipped and
+# test-new-release then fails on an Evict VMOP still InProgress.
 migration_expected() {
   local module_source="${DEV_MODULE_SOURCE:-}"
   local current="${CURRENT_RELEASE:-}"
@@ -57,7 +60,7 @@ migration_expected() {
   for image in virtHandler virtLauncher; do
     if [ "$(jq -r --arg i "${image}" '.[$i] // ""' <<< "${current_digests}")" \
       != "$(jq -r --arg i "${image}" '.[$i] // ""' <<< "${new_digests}")" ]; then
-      echo "[INFO] The ${image} image differs between ${current} and ${new}, virtual machines will be migrated"
+      echo "[INFO] The ${image} image differs between ${current} and ${new}, virtual machines are expected to be migrated"
       return 0
     fi
   done
@@ -74,7 +77,7 @@ publish_verdict() {
 
 if ! migration_expected; then
   publish_verdict false
-  echo "[INFO] ${CURRENT_RELEASE} and ${NEW_RELEASE} ship the same virt-handler and virt-launcher: the upgrade does not migrate virtual machines, nothing to wait for"
+  echo "[INFO] ${CURRENT_RELEASE} and ${NEW_RELEASE} ship the same virt-handler and virt-launcher: no migration is expected, nothing to wait for"
   exit 0
 fi
 

From 9142f29446d08ed1b91bdb407dffb9321bf377a2 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Wed, 12 Aug 2026 01:03:57 +0300
Subject: [PATCH 19/23] test(ci): assert the iperf totals once

Both branches of the post-upgrade check asserted the same two totals, which do
not depend on whether the upgrade migrates virtual machines. Assert them before
the branch instead.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 test/e2e/release/current_release_smoke.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go
index fb23f86aa6..6680de0f75 100644
--- a/test/e2e/release/current_release_smoke.go
+++ b/test/e2e/release/current_release_smoke.go
@@ -205,11 +205,11 @@ func (t *currentReleaseSmokeTest) verifyIPerfContinuityAfterUpgrade() {
 	By("Validating the iperf report spans the module upgrade")
 	report := getIPerfClientReport(t.framework, t.iperfClient.vm, releaseIPerfReportPath)
 	Expect(isExpectedIPerfReportError(report.Error)).To(BeTrue(), "iperf3 report contains an unexpected error: %q", report.Error)
+	Expect(report.End.SumSent.Bytes).To(BeNumerically(">", 0), "iperf3 client should send data")
+	Expect(report.End.SumSent.BitsPerSecond).To(BeNumerically(">", 0), "iperf3 client should report throughput")
 
 	if !upgradeMigratesVMs() {
 		By("Skipping the migration window checks: the upgrade does not migrate virtual machines")
-		Expect(report.End.SumSent.Bytes).To(BeNumerically(">", 0), "iperf3 client should send data")
-		Expect(report.End.SumSent.BitsPerSecond).To(BeNumerically(">", 0), "iperf3 client should report throughput")
 
 		return
 	}
@@ -249,8 +249,6 @@ func (t *currentReleaseSmokeTest) verifyIPerfContinuityAfterUpgrade() {
 
 	Expect(transmittedAroundUpgrade).To(BeNumerically(">", 0), "iperf3 should transmit data around the module upgrade")
 	Expect(zeroIntervals).To(BeNumerically("<=", 1), "iperf3 should not be interrupted during the module upgrade")
-	Expect(report.End.SumSent.Bytes).To(BeNumerically(">", 0), "iperf3 client should send data")
-	Expect(report.End.SumSent.BitsPerSecond).To(BeNumerically(">", 0), "iperf3 client should report throughput")
 }
 
 func (t *currentReleaseSmokeTest) verifyEvictVMOPsCompleted() {

From 44d26cf64825799bbb9ded559d934978ba56611b Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Wed, 12 Aug 2026 01:03:57 +0300
Subject: [PATCH 20/23] refactor(ci): take the virtualization feature gates
 from the release bundle

The gates an e2e run enables came from a hand-kept table of the release each
gate first shipped in, which had to be updated by hand and, for a gate no
release carries yet, was never enabled on a release tag at all. The nightly
pipeline carried a second, plain hardcoded copy of the list.

Every module bundle ships its own openapi/config-values.yaml, and the enum in it
is the very schema the ModuleConfig is validated against, so each release now
states its own gate list and no table is needed. The list of releases is
intersected for the step that runs before the image tag is switched.

A gate the live cluster refuses - locked in this edition, or needing a newer
Kubernetes - is now dropped with a warning instead of failing the run: the point
of the release e2e is the upgrade, not the gate. Since the moduleconfig webhook
declares sideEffects: None, a server-side dry run answers whether a gate would
be admitted without writing anything, and the dropped gates land in the job
summary so the lost coverage stays visible.

That webhook is served by virtualization-controller itself and has no
failurePolicy, so a patch right after an image switch is rejected for reasons
that say nothing about the gate. Get an answer out of the webhook first and
treat lasting silence as the defect it is. A patch that goes through re-renders
the module, so wait for it to settle before the next job starts.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/common.sh            | 110 +++++++++++-------
 .../e2e/configure-virtualization-release.sh   |  17 ++-
 .../bash/e2e/configure-virtualization.sh      |  20 +++-
 .../e2e/patch-virtualization-feature-gates.sh |  71 ++++++++++-
 .../e2e-nightly-reusable-pipeline.yml         |  12 ++
 .github/workflows/e2e-nightly.yml             |   3 +
 .../e2e-test-releases-reusable-pipeline.yml   |  30 +++--
 7 files changed, 205 insertions(+), 58 deletions(-)

diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index 0dfe7b91d4..02f4c20235 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -47,52 +47,84 @@ modules_repo_for_registry() {
   fi
 }
 
-# The feature gates an e2e run enables, each with the release it first shipped
-# in. An empty version marks a gate no release carries yet, so only a build off
-# main or a pull request has it. Keep this list in step with the enum of
-# openapi/config-values.yaml: a gate the pulled module does not know fails
-# ModulePullOverride validation and leaves the module uninstalled.
-VIRTUALIZATION_FEATURE_GATES=(
-  "HotplugCPUWithLiveMigration:v1.0"
-  "HotplugMemoryWithLiveMigration:v1.0"
-  "HotplugCPUAndMemoryWithInPlaceResize:v1.10"
-  "GPU:"
-)
-
-# Tells whether a release ref knows a gate that first shipped in a given version.
-# Anything that is not a release tag - a pull request reference, a build off main
-# - carries every gate the repository has.
-release_knows_feature_gate() {
-  local release="$1"
-  local since="$2"
-  local major minor since_major since_minor
-
-  [[ "${release}" =~ ^v([0-9]+)\.([0-9]+)\. ]] || return 0
-  major="${BASH_REMATCH[1]}"
-  minor="${BASH_REMATCH[2]}"
-
-  [[ "${since}" =~ ^v([0-9]+)\.([0-9]+) ]] || return 1
-  since_major="${BASH_REMATCH[1]}"
-  since_minor="${BASH_REMATCH[2]}"
-
-  (( major > since_major || ( major == since_major && minor >= since_minor ) ))
+# Echoes the feature gates a release accepts, one per line. The module bundle
+# carries openapi/config-values.yaml, and its enum is the very schema the
+# ModuleConfig of that release is validated against - so every release states
+# its own gate list and no version table has to be kept by hand.
+# Usage: module_feature_gates <module_source> <release>
+module_feature_gates() {
+  local module_source="$1"
+  local release="$2"
+
+  crane export "${module_source}/virtualization:${release}" - |
+    tar -Oxf - openapi/config-values.yaml |
+    yq '.properties.featureGates.items.enum[]'
 }
 
-# Echoes the feature gates every given release supports, one per line.
-# Usage: virtualization_feature_gates [release]...
+# Echoes the gates every given release accepts, one per line, in the order the
+# first release lists them.
+# Usage: virtualization_feature_gates <module_source> <release>...
 virtualization_feature_gates() {
-  local entry gate since release
+  local module_source="$1"
+  shift
+  local gates release other
+
+  gates="$(module_feature_gates "${module_source}" "$1")"
+  shift
+
+  for release in "$@"; do
+    other="$(module_feature_gates "${module_source}" "${release}")"
+    gates="$(grep -xF -f <(printf '%s\n' "${other}") <<< "${gates}" || true)"
+  done
+
+  if [ -n "${gates}" ]; then
+    printf '%s\n' "${gates}"
+  fi
+}
+
+# Server-side dry-run of the feature gate patch: the moduleconfig webhook
+# declares sideEffects: None, so the real admission chain can be asked whether a
+# gate list would be accepted without writing anything. Echoes the admission
+# error when it is not.
+# Usage: gate_accepted <gates_json>
+gate_accepted() {
+  local gates_json="$1"
+  local output
+
+  if output="$(kubectl patch mc virtualization --type merge --dry-run=server \
+    -p "{\"spec\":{\"settings\":{\"featureGates\":${gates_json}}}}" 2>&1)"; then
+    return 0
+  fi
+
+  printf '%s' "${output}"
+  return 1
+}
 
-  for entry in "${VIRTUALIZATION_FEATURE_GATES[@]}"; do
-    gate="${entry%%:*}"
-    since="${entry#*:}"
+# Waits until the moduleconfig webhook answers again. It is served by
+# virtualization-controller itself and has no failurePolicy, so right after an
+# image switch every patch is rejected for a while. The probe dry-runs the gates
+# the config already carries: that adds no gate, so the validator returns nil
+# and only an unreachable webhook can fail it.
+# Usage: moduleconfig_writable [count] [delay]
+moduleconfig_writable() {
+  local count="${1:-12}"
+  local delay="${2:-10}"
+  local current error i
 
-    for release in "$@"; do
-      release_knows_feature_gate "${release}" "${since}" || continue 2
-    done
+  for ((i = 1; i <= count; i++)); do
+    current="$(kubectl get mc virtualization -o jsonpath='{.spec.settings.featureGates}' 2>/dev/null || true)"
 
-    echo "${gate}"
+    if error="$(gate_accepted "${current:-[]}")"; then
+      return 0
+    fi
+
+    echo "[WARN] Module config is not writable yet (attempt ${i}/${count}): ${error}"
+    if [ "$i" -lt "$count" ]; then
+      sleep "$delay"
+    fi
   done
+
+  return 1
 }
 
 # Echoes images_digests.json packaged in the module image of a given release.
diff --git a/.github/scripts/bash/e2e/configure-virtualization-release.sh b/.github/scripts/bash/e2e/configure-virtualization-release.sh
index d8fc6d86da..1df7c7a400 100644
--- a/.github/scripts/bash/e2e/configure-virtualization-release.sh
+++ b/.github/scripts/bash/e2e/configure-virtualization-release.sh
@@ -24,6 +24,7 @@ source "${SCRIPT_DIR}/deckhouse.sh"
 
 require_env DEV_REGISTRY_DOCKER_CFG
 require_env CURRENT_RELEASE
+require_env DEV_MODULE_SOURCE
 
 required_env_value() {
   local name="$1"
@@ -34,13 +35,21 @@ required_env_value() {
 
 dev_registry_docker_cfg="$(required_env_value DEV_REGISTRY_DOCKER_CFG)"
 current_release="$(required_env_value CURRENT_RELEASE)"
+dev_module_source="$(required_env_value DEV_MODULE_SOURCE)"
 
 REGISTRY="$(registry_host_from_docker_cfg "${dev_registry_docker_cfg}")"
 
-# Only the gates this release knows: a gate it does not support fails
-# ModulePullOverride validation and the module never installs. The upgrade
-# revisits the list for the new release (patch-virtualization-feature-gates.sh).
-feature_gates_yaml="$(virtualization_feature_gates "${current_release}" | sed 's/^/      - /')"
+# The gate list of this very release, taken as is: nothing can dry-run it here,
+# because the ModuleConfig is created before the module exists and its webhook
+# guards updates only. A gate the release does not know fails ModulePullOverride
+# validation and the module never installs. The upgrade revisits the list for the
+# new release (patch-virtualization-feature-gates.sh).
+feature_gates_yaml="$(virtualization_feature_gates "${dev_module_source}" "${current_release}" | sed 's/^/      - /')"
+if [ -z "${feature_gates_yaml}" ]; then
+  echo "[ERROR] No feature gates were read from the ${current_release} module bundle; an empty list would render featureGates as null" >&2
+  exit 1
+fi
+
 echo "[INFO] Feature gates for ${current_release}:"
 echo "${feature_gates_yaml}"
 
diff --git a/.github/scripts/bash/e2e/configure-virtualization.sh b/.github/scripts/bash/e2e/configure-virtualization.sh
index 33fcd23a6c..e892dd8f34 100644
--- a/.github/scripts/bash/e2e/configure-virtualization.sh
+++ b/.github/scripts/bash/e2e/configure-virtualization.sh
@@ -25,6 +25,7 @@ source "${SCRIPT_DIR}/deckhouse.sh"
 require_env DEV_REGISTRY_DOCKER_CFG
 require_env NESTED_STORAGE_CLASS_NAME
 require_env VIRTUALIZATION_TAG
+require_env DEV_MODULE_SOURCE
 
 # shellcheck disable=SC2153,SC2154
 dev_registry_docker_cfg="${DEV_REGISTRY_DOCKER_CFG}"
@@ -32,6 +33,21 @@ dev_registry_docker_cfg="${DEV_REGISTRY_DOCKER_CFG}"
 nested_storage_class_name="${NESTED_STORAGE_CLASS_NAME}"
 # shellcheck disable=SC2153,SC2154
 virtualization_tag="${VIRTUALIZATION_TAG}"
+# shellcheck disable=SC2153,SC2154
+dev_module_source="${DEV_MODULE_SOURCE}"
+
+# The gate list of the build under test, taken as is: the ModuleConfig is created
+# before the module exists, so no webhook can be asked about it here. Read before
+# anything is applied, so a registry problem fails the job before it touches the
+# cluster.
+feature_gates_yaml="$(virtualization_feature_gates "${dev_module_source}" "${virtualization_tag}" | sed 's/^/      - /')"
+if [ -z "${feature_gates_yaml}" ]; then
+  echo "[ERROR] No feature gates were read from the ${virtualization_tag} module bundle; an empty list would render featureGates as null" >&2
+  exit 1
+fi
+
+echo "[INFO] Feature gates for ${virtualization_tag}:"
+echo "${feature_gates_yaml}"
 
 show_modulesource_status() {
   local ms_json
@@ -167,9 +183,7 @@ spec:
     virtualMachineCIDRs:
       - 192.168.10.0/24
     featureGates:
-      - HotplugCPUWithLiveMigration
-      - HotplugMemoryWithLiveMigration
-      - HotplugCPUAndMemoryWithInPlaceResize
+${feature_gates_yaml}
   source: deckhouse-dev
   version: 1
 ---
diff --git a/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh b/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh
index bb675a88a2..45a20a7d21 100644
--- a/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh
+++ b/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh
@@ -14,32 +14,84 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Sets the virtualization feature gates of the module config to those supported
-# by every given release, and verifies the result.
+# Sets the virtualization feature gates of the module config to those every
+# given release accepts and the live cluster admits, and verifies the result.
 #
-# Usage: patch-virtualization-feature-gates.sh <release>...
+# Usage: DEV_MODULE_SOURCE=<repo> patch-virtualization-feature-gates.sh <release>...
 #
 # During a release upgrade this runs twice. Before the image tag is patched it
 # is called with both releases, which drops the gates the new release does not
 # know - otherwise the new module fails validation and never installs. After the
 # upgrade it is called with the new release alone, which enables the gates only
 # that release supports.
+#
+# A gate the cluster refuses - locked in this edition, or needing a newer
+# Kubernetes - is dropped with a warning instead of failing the run: the point of
+# the release e2e is the upgrade, not the gate.
 
 set -Eeuo pipefail
 
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=.github/scripts/bash/e2e/common.sh
 source "${SCRIPT_DIR}/common.sh"
+# Sourced for virtualization_ready and virt_handler_ready, used after the patch.
+# shellcheck source=.github/scripts/bash/e2e/wait-virtualization-ready.sh
+source "${SCRIPT_DIR}/wait-virtualization-ready.sh"
 
 if [ "$#" -eq 0 ]; then
   echo "[ERROR] Usage: $(basename -- "${BASH_SOURCE[0]}") <release>..." >&2
   exit 1
 fi
 
-gates_json="$(virtualization_feature_gates "$@" | jq -Rsc 'split("\n") | map(select(length > 0))')"
+require_env DEV_MODULE_SOURCE
+# shellcheck disable=SC2153,SC2154 # set in the workflow, checked by require_env above
+dev_module_source="${DEV_MODULE_SOURCE}"
+summary_file="${GITHUB_STEP_SUMMARY:-/dev/stdout}"
+
+wanted="$(virtualization_feature_gates "${dev_module_source}" "$@")"
+echo "[INFO] Releases: $*"
+echo "[INFO] Feature gates they all accept: ${wanted//$'\n'/ }"
+
+# The webhook validating the gates is served by virtualization-controller itself,
+# so a rejection right after an image switch says nothing about the gate. Get an
+# answer out of the webhook first, and treat silence as the defect it is.
+if ! moduleconfig_writable; then
+  echo "[ERROR] The module config webhook never answered, feature gates cannot be validated" >&2
+  exit 1
+fi
+
+accepted=""
+dropped_rows=""
+while IFS= read -r gate; do
+  [ -n "${gate}" ] || continue
+
+  candidate_json="$(printf '%s\n%s\n' "${accepted}" "${gate}" | jq -Rsc 'split("\n") | map(select(length > 0))')"
+  if error="$(gate_accepted "${candidate_json}")"; then
+    accepted="$(printf '%s\n%s' "${accepted}" "${gate}")"
+    continue
+  fi
+
+  error="$(tr '\n' ' ' <<< "${error}")"
+  echo "[WARN] Feature gate ${gate} is not admitted by the cluster, dropping it: ${error}"
+  echo "::warning title=Feature gate ${gate} was dropped::${error}"
+  dropped_rows="${dropped_rows}- \`${gate}\`: ${error}"$'\n'
+done <<< "${wanted}"
+
+if [ -n "${dropped_rows}" ]; then
+  {
+    # The release list tells the two runs of this script apart: both write into
+    # the summary of the same job.
+    echo "## Feature gates dropped from the module config ($*)"
+    echo
+    printf '%s' "${dropped_rows}"
+    echo
+  } >> "${summary_file}"
+fi
+
+gates_json="$(jq -Rsc 'split("\n") | map(select(length > 0))' <<< "${accepted}")"
 current_json="$(kubectl get mc virtualization -o jsonpath='{.spec.settings.featureGates}')"
 
-echo "[INFO] Feature gates supported by $*: ${gates_json}"
+echo "[INFO] Feature gates to apply: ${gates_json}"
 
 if [ "${current_json}" = "${gates_json}" ]; then
   echo "[INFO] Module config already lists exactly these gates, nothing to patch"
@@ -56,3 +108,12 @@ if [ "${patched_json}" != "${gates_json}" ]; then
 fi
 
 echo "[INFO] Feature gates in effect: ${patched_json}"
+
+# A new gate set re-renders the module - the GPU gate, for one, adds GPUsWithDRA
+# to the KubeVirt CR - which restarts the virtualization components, and the next
+# job must not start against them mid-restart. Known limit: the re-render is only
+# requested here, so it may not have begun by the time this wait starts, and then
+# the wait passes on the state the restart is about to leave.
+echo "[INFO] Waiting for the module to settle after the feature gate patch"
+virtualization_ready
+virt_handler_ready
diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml
index 8c40f4d24d..f0f34825be 100644
--- a/.github/workflows/e2e-nightly-reusable-pipeline.yml
+++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml
@@ -153,6 +153,8 @@ on:
         required: false
       BOOTSTRAP_DEV_PROXY:
         required: true
+      DEV_MODULES_REGISTRY_PASSWORD:
+        required: true
       E2E_ARTIFACTS_GPG_PASSPHRASE:
         required: true
       FOX_TOKEN:
@@ -547,9 +549,19 @@ jobs:
         with:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
           check-api: "false"
+
+      # crane reads the feature gate list out of the module bundle image.
+      - name: Setup crane and registry auth
+        uses: deckhouse/modules-actions/setup@v2
+        with:
+          registry: ${{ vars.DEV_REGISTRY }}
+          registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }}
+          registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
+
       - name: Configure Virtualization
         env:
           DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }}
+          DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
           NESTED_STORAGE_CLASS_NAME: ${{ inputs.nested_storageclass_name }}
           VIRTUALIZATION_TAG: ${{ env.VIRTUALIZATION_TAG }}
         run: |
diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml
index 1547a7e0db..53eb3cb227 100644
--- a/.github/workflows/e2e-nightly.yml
+++ b/.github/workflows/e2e-nightly.yml
@@ -127,6 +127,7 @@ jobs:
       REGISTRY_DOCKER_CFG: ${{ needs.set-vars.outputs.registry_profile == 'stage' && secrets.STAGE_IO_REGISTRY_DOCKER_CFG || secrets.PROD_IO_REGISTRY_DOCKER_CFG }}
       PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }}
       BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}
+      DEV_MODULES_REGISTRY_PASSWORD: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
       E2E_ARTIFACTS_GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }}
       FOX_TOKEN: ${{ secrets.FOX_TOKEN }}
 
@@ -161,6 +162,7 @@ jobs:
       REGISTRY_DOCKER_CFG: ${{ needs.set-vars.outputs.registry_profile == 'stage' && secrets.STAGE_IO_REGISTRY_DOCKER_CFG || secrets.PROD_IO_REGISTRY_DOCKER_CFG }}
       PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }}
       BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}
+      DEV_MODULES_REGISTRY_PASSWORD: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
       E2E_ARTIFACTS_GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }}
       FOX_TOKEN: ${{ secrets.FOX_TOKEN }}
 
@@ -198,6 +200,7 @@ jobs:
       REGISTRY_DOCKER_CFG: ${{ needs.set-vars.outputs.registry_profile == 'stage' && secrets.STAGE_IO_REGISTRY_DOCKER_CFG || secrets.PROD_IO_REGISTRY_DOCKER_CFG }}
       PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }}
       BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}
+      DEV_MODULES_REGISTRY_PASSWORD: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
       E2E_ARTIFACTS_GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }}
       FOX_TOKEN: ${{ secrets.FOX_TOKEN }}
 
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index fffe22cf9b..bf74a3c331 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -494,10 +494,20 @@ jobs:
         with:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
           check-api: "false"
+
+      # crane reads the feature gate list out of the release bundle image.
+      - name: Setup crane and registry auth
+        uses: deckhouse/modules-actions/setup@v2
+        with:
+          registry: ${{ vars.DEV_REGISTRY }}
+          registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }}
+          registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
+
       - name: Configure Virtualization
         env:
           CURRENT_RELEASE: ${{ env.CURRENT_RELEASE }}
           DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }}
+          DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
         run: bash "${E2E_SCRIPT_DIR}/configure-virtualization-release.sh"
       - name: Wait for Virtualization to be ready
         run: |
@@ -630,6 +640,15 @@ jobs:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
           check-api: "false"
 
+      # crane reads the feature gate lists and the image digests out of the
+      # release bundle images.
+      - name: Setup crane and registry auth
+        uses: deckhouse/modules-actions/setup@v2
+        with:
+          registry: ${{ vars.DEV_REGISTRY }}
+          registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }}
+          registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
+
       - name: Show current MPO state
         run: |
           echo "[INFO] Current ModulePullOverride before patching:"
@@ -638,6 +657,8 @@ jobs:
       # Gates the new release does not know would fail its validation, so they
       # go before the image tag is switched, while the old module is still live.
       - name: Drop feature gates the new release does not support
+        env:
+          DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
         run: |
           bash "${E2E_SCRIPT_DIR}/patch-virtualization-feature-gates.sh" \
             "${CURRENT_RELEASE}" "${NEW_RELEASE}"
@@ -653,13 +674,6 @@ jobs:
           echo "[INFO] Show patched ModulePullOverride:"
           kubectl get mpo virtualization -o yaml
 
-      - name: Setup crane and registry auth
-        uses: deckhouse/modules-actions/setup@v2
-        with:
-          registry: ${{ vars.DEV_REGISTRY }}
-          registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }}
-          registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
-
       - name: Verify image digests in pods after upgrade
         env:
           DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
@@ -669,6 +683,8 @@ jobs:
       # Now that the new images are running, the gates only that release knows
       # can be enabled, so the new release is tested with all of them on.
       - name: Enable every feature gate the new release supports
+        env:
+          DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
         run: bash "${E2E_SCRIPT_DIR}/patch-virtualization-feature-gates.sh" "${NEW_RELEASE}"
 
       - name: Show ModulePullOverride state after upgrade

From 49543f935421601e6741f815047da86af2270426 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Wed, 12 Aug 2026 01:16:35 +0300
Subject: [PATCH 21/23] refactor(ci): create the module config without feature
 gates

The gates were rendered into the ModuleConfig at creation time, where nothing
validates them: the module webhook guards updates only, and the controller that
serves it is not running yet. A gate that lands in a fresh config is therefore
never checked, and the per-gate dry run cannot drop it afterwards either - by
then it is not a gate being added any more. A gate this edition locks makes the
controller exit on start, and since it serves that very webhook, the config can
no longer be repaired.

Create the config without the key and let patch-virtualization-feature-gates.sh
add the gates once the module is Ready. Every gate then goes through the webhook
as an addition, install and upgrade share one code path, and the enum-to-YAML
rendering leaves both install scripts. The guard against an unreadable bundle
moves into virtualization_feature_gates, where an empty enum for one release is
told apart from a legitimately empty intersection of two.

In the nightly pipeline maintenance mode becomes a step of its own after the
gate patch: it stops module reconciliation, so gates applied later would never
reach the rendered resources.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/common.sh            | 18 +++++++++++----
 .../e2e/configure-virtualization-release.sh   | 22 ++++---------------
 .../bash/e2e/configure-virtualization.sh      | 22 ++++---------------
 .../e2e/patch-virtualization-feature-gates.sh | 17 +++++++++-----
 .../e2e-nightly-reusable-pipeline.yml         | 22 +++++++++++++++----
 .../e2e-test-releases-reusable-pipeline.yml   | 11 ++++++++--
 6 files changed, 61 insertions(+), 51 deletions(-)

diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index 02f4c20235..698c3c67a4 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -67,13 +67,23 @@ module_feature_gates() {
 virtualization_feature_gates() {
   local module_source="$1"
   shift
-  local gates release other
-
-  gates="$(module_feature_gates "${module_source}" "$1")"
-  shift
+  local gates="" release other first=1
 
   for release in "$@"; do
     other="$(module_feature_gates "${module_source}" "${release}")"
+    # An empty intersection of two releases is legitimate, an empty enum of a
+    # single release is not: crane or yq failed to read the bundle.
+    if [ -z "${other}" ]; then
+      echo "[ERROR] No feature gates were read from the ${release} module bundle" >&2
+      return 1
+    fi
+
+    if [ "${first}" = 1 ]; then
+      gates="${other}"
+      first=0
+      continue
+    fi
+
     gates="$(grep -xF -f <(printf '%s\n' "${other}") <<< "${gates}" || true)"
   done
 
diff --git a/.github/scripts/bash/e2e/configure-virtualization-release.sh b/.github/scripts/bash/e2e/configure-virtualization-release.sh
index 1df7c7a400..0fab9d6924 100644
--- a/.github/scripts/bash/e2e/configure-virtualization-release.sh
+++ b/.github/scripts/bash/e2e/configure-virtualization-release.sh
@@ -24,7 +24,6 @@ source "${SCRIPT_DIR}/deckhouse.sh"
 
 require_env DEV_REGISTRY_DOCKER_CFG
 require_env CURRENT_RELEASE
-require_env DEV_MODULE_SOURCE
 
 required_env_value() {
   local name="$1"
@@ -35,24 +34,9 @@ required_env_value() {
 
 dev_registry_docker_cfg="$(required_env_value DEV_REGISTRY_DOCKER_CFG)"
 current_release="$(required_env_value CURRENT_RELEASE)"
-dev_module_source="$(required_env_value DEV_MODULE_SOURCE)"
 
 REGISTRY="$(registry_host_from_docker_cfg "${dev_registry_docker_cfg}")"
 
-# The gate list of this very release, taken as is: nothing can dry-run it here,
-# because the ModuleConfig is created before the module exists and its webhook
-# guards updates only. A gate the release does not know fails ModulePullOverride
-# validation and the module never installs. The upgrade revisits the list for the
-# new release (patch-virtualization-feature-gates.sh).
-feature_gates_yaml="$(virtualization_feature_gates "${dev_module_source}" "${current_release}" | sed 's/^/      - /')"
-if [ -z "${feature_gates_yaml}" ]; then
-  echo "[ERROR] No feature gates were read from the ${current_release} module bundle; an empty list would render featureGates as null" >&2
-  exit 1
-fi
-
-echo "[INFO] Feature gates for ${current_release}:"
-echo "${feature_gates_yaml}"
-
 echo "[INFO] Apply ModuleSource prod config"
 kubectl_apply_with_retry 20 10 show_deckhouse_state <<EOF
 apiVersion: deckhouse.io/v1alpha1
@@ -69,6 +53,10 @@ EOF
 
 kubectl wait --for=jsonpath='{.status.phase}'=Active ms deckhouse-dev --timeout=30s
 
+# No featureGates here: the module webhook validates only gates being added to a
+# live config, so gates set at creation time reach the controller unchecked and a
+# gate this edition locks makes it exit on start - taking that very webhook with
+# it. patch-virtualization-feature-gates.sh adds them once the module is Ready.
 echo "[INFO] Apply Virtualization module config with current-release tag: ${current_release}"
 kubectl_apply_with_retry 20 10 show_deckhouse_state <<EOF
 apiVersion: deckhouse.io/v1alpha1
@@ -86,8 +74,6 @@ spec:
         type: PersistentVolumeClaim
     virtualMachineCIDRs:
       - 192.168.10.0/24
-    featureGates:
-${feature_gates_yaml}
   source: deckhouse-dev
   version: 1
 ---
diff --git a/.github/scripts/bash/e2e/configure-virtualization.sh b/.github/scripts/bash/e2e/configure-virtualization.sh
index e892dd8f34..f3210b303f 100644
--- a/.github/scripts/bash/e2e/configure-virtualization.sh
+++ b/.github/scripts/bash/e2e/configure-virtualization.sh
@@ -25,7 +25,6 @@ source "${SCRIPT_DIR}/deckhouse.sh"
 require_env DEV_REGISTRY_DOCKER_CFG
 require_env NESTED_STORAGE_CLASS_NAME
 require_env VIRTUALIZATION_TAG
-require_env DEV_MODULE_SOURCE
 
 # shellcheck disable=SC2153,SC2154
 dev_registry_docker_cfg="${DEV_REGISTRY_DOCKER_CFG}"
@@ -33,21 +32,6 @@ dev_registry_docker_cfg="${DEV_REGISTRY_DOCKER_CFG}"
 nested_storage_class_name="${NESTED_STORAGE_CLASS_NAME}"
 # shellcheck disable=SC2153,SC2154
 virtualization_tag="${VIRTUALIZATION_TAG}"
-# shellcheck disable=SC2153,SC2154
-dev_module_source="${DEV_MODULE_SOURCE}"
-
-# The gate list of the build under test, taken as is: the ModuleConfig is created
-# before the module exists, so no webhook can be asked about it here. Read before
-# anything is applied, so a registry problem fails the job before it touches the
-# cluster.
-feature_gates_yaml="$(virtualization_feature_gates "${dev_module_source}" "${virtualization_tag}" | sed 's/^/      - /')"
-if [ -z "${feature_gates_yaml}" ]; then
-  echo "[ERROR] No feature gates were read from the ${virtualization_tag} module bundle; an empty list would render featureGates as null" >&2
-  exit 1
-fi
-
-echo "[INFO] Feature gates for ${virtualization_tag}:"
-echo "${feature_gates_yaml}"
 
 show_modulesource_status() {
   local ms_json
@@ -164,6 +148,10 @@ spec:
 EOF
 }
 
+# No featureGates here: the module webhook validates only gates being added to a
+# live config, so gates set at creation time reach the controller unchecked and a
+# gate this edition locks makes it exit on start - taking that very webhook with
+# it. patch-virtualization-feature-gates.sh adds them once the module is Ready.
 apply_virtualization_module_config() {
   echo "[INFO] Apply Virtualization module config"
   kubectl_apply_with_retry 20 10 show_deckhouse_state <<EOF
@@ -182,8 +170,6 @@ spec:
         type: PersistentVolumeClaim
     virtualMachineCIDRs:
       - 192.168.10.0/24
-    featureGates:
-${feature_gates_yaml}
   source: deckhouse-dev
   version: 1
 ---
diff --git a/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh b/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh
index 45a20a7d21..20eb052e91 100644
--- a/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh
+++ b/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh
@@ -19,11 +19,15 @@
 #
 # Usage: DEV_MODULE_SOURCE=<repo> patch-virtualization-feature-gates.sh <release>...
 #
-# During a release upgrade this runs twice. Before the image tag is patched it
-# is called with both releases, which drops the gates the new release does not
-# know - otherwise the new module fails validation and never installs. After the
-# upgrade it is called with the new release alone, which enables the gates only
-# that release supports.
+# This is the only place that sets the gates: the install scripts create the
+# ModuleConfig without them, so every gate goes through the webhook as an
+# addition and can be dropped here instead of breaking the module.
+#
+# During a release upgrade this runs again, twice. Before the image tag is
+# patched it is called with both releases, which drops the gates the new release
+# does not know - otherwise the new module fails validation and never installs.
+# After the upgrade it is called with the new release alone, which enables the
+# gates only that release supports.
 #
 # A gate the cluster refuses - locked in this edition, or needing a newer
 # Kubernetes - is dropped with a warning instead of failing the run: the point of
@@ -89,6 +93,9 @@ if [ -n "${dropped_rows}" ]; then
 fi
 
 gates_json="$(jq -Rsc 'split("\n") | map(select(length > 0))' <<< "${accepted}")"
+# Empty on the install path, where the config carries no featureGates key yet.
+# jq always prints an array, so it never compares equal to that and the patch
+# below happens.
 current_json="$(kubectl get mc virtualization -o jsonpath='{.spec.settings.featureGates}')"
 
 echo "[INFO] Feature gates to apply: ${gates_json}"
diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml
index f0f34825be..281ebfdf32 100644
--- a/.github/workflows/e2e-nightly-reusable-pipeline.yml
+++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml
@@ -550,7 +550,8 @@ jobs:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
           check-api: "false"
 
-      # crane reads the feature gate list out of the module bundle image.
+      # crane reads the feature gate list of the module bundle image for the
+      # feature gate step below.
       - name: Setup crane and registry auth
         uses: deckhouse/modules-actions/setup@v2
         with:
@@ -561,7 +562,6 @@ jobs:
       - name: Configure Virtualization
         env:
           DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }}
-          DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
           NESTED_STORAGE_CLASS_NAME: ${{ inputs.nested_storageclass_name }}
           VIRTUALIZATION_TAG: ${{ env.VIRTUALIZATION_TAG }}
         run: |
@@ -569,8 +569,6 @@ jobs:
           # Deckhouse queue/source propagation before applying ModuleConfig.
           bash "${E2E_SCRIPT_DIR}/configure-virtualization.sh"
       - name: Wait for Virtualization to be ready
-        env:
-          STORAGE_TYPE: ${{ inputs.storage_type }}
         run: |
           source "${E2E_SCRIPT_DIR}/wait-virtualization-ready.sh"
 
@@ -585,6 +583,22 @@ jobs:
           echo "[INFO] Checking virt-handler pods "
           virt_handler_ready
 
+      # Only now, with the module Ready, its webhook can validate each gate as an
+      # addition and reject the ones this cluster does not admit.
+      - name: Enable every feature gate the build under test supports
+        env:
+          DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
+        run: bash "${E2E_SCRIPT_DIR}/patch-virtualization-feature-gates.sh" "${VIRTUALIZATION_TAG}"
+
+      # Maintenance mode stops module reconciliation, so it has to come after the
+      # feature gate patch: otherwise the new gates never reach the rendered
+      # resources.
+      - name: Switch modules to maintenance mode
+        env:
+          STORAGE_TYPE: ${{ inputs.storage_type }}
+        run: |
+          source "${E2E_SCRIPT_DIR}/wait-virtualization-ready.sh"
+
           enable_maintenance_mode "${STORAGE_TYPE}"
 
   e2e-test:
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index bf74a3c331..001689cd2a 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -495,7 +495,8 @@ jobs:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
           check-api: "false"
 
-      # crane reads the feature gate list out of the release bundle image.
+      # crane reads the feature gate list of the release bundle image for the
+      # feature gate step below.
       - name: Setup crane and registry auth
         uses: deckhouse/modules-actions/setup@v2
         with:
@@ -507,7 +508,6 @@ jobs:
         env:
           CURRENT_RELEASE: ${{ env.CURRENT_RELEASE }}
           DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }}
-          DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
         run: bash "${E2E_SCRIPT_DIR}/configure-virtualization-release.sh"
       - name: Wait for Virtualization to be ready
         run: |
@@ -524,6 +524,13 @@ jobs:
           echo "[INFO] Checking virt-handler pods "
           virt_handler_ready
 
+      # Only now, with the module Ready, its webhook can validate each gate as an
+      # addition and reject the ones this cluster does not admit.
+      - name: Enable every feature gate the current release supports
+        env:
+          DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
+        run: bash "${E2E_SCRIPT_DIR}/patch-virtualization-feature-gates.sh" "${CURRENT_RELEASE}"
+
       # Opens the observation window of the ClusterAlerts report. Runs even on
       # failure, so a module that never became ready is still observed.
       - name: Mark the start of the ClusterAlerts observation window

From b3e246881945eb17ec10f2dabc7f6d1a880f04b1 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Wed, 12 Aug 2026 12:12:52 +0300
Subject: [PATCH 22/23] fix(ci): keep the reason Prometheus gives for a
 rejected query

curl -f throws the response body away, and Prometheus puts the reason a query was
rejected exactly there, so a failed collection reported nothing but the script's
own guess at what had happened. --fail-with-body keeps the non-zero exit code and
the body both, which also makes the status check in fetch_rules meaningful again
instead of unreachable behind the same flag.

Sample the ALERTS series every 10 seconds instead of 30. The series exists only
while its alert is active, so an alert shorter than the step can fall between two
samples and be reported as no alert at all - the single alert that fired during
the last run lasted two samples at the old step. A rollover window holds a few
hundred points at the new one, far below the 11000 Prometheus allows per series.

Say what actually makes the module config probe a reachability check: the webhook
skips its whole validator chain when the generation does not change, so a no-op
patch can fail on nothing but being unable to reach it. Record the other side of
that predicate too - a real gate patch runs every validator against the whole
config and can be refused over CIDRs or storage classes, with a message that says
nothing about gates.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/collect-clusteralerts.sh | 12 ++++++++++--
 .github/scripts/bash/e2e/common.sh                | 10 ++++++++--
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/.github/scripts/bash/e2e/collect-clusteralerts.sh b/.github/scripts/bash/e2e/collect-clusteralerts.sh
index 3217791c15..54a8052804 100644
--- a/.github/scripts/bash/e2e/collect-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/collect-clusteralerts.sh
@@ -40,7 +40,11 @@ prometheus_selector="prometheus=main"
 # localhost there is reachable too.
 prometheus_port=""
 local_port="19090"
-query_step="30"
+# The ALERTS series exists only while its alert is active, so an alert shorter
+# than the step can fall between two samples and be missed entirely. Prometheus
+# allows 11000 points per series, and a rollover window is under half an hour, so
+# a step this small is nowhere near the limit.
+query_step="10"
 ready_attempts=30
 ready_delay=2
 
@@ -76,10 +80,13 @@ cleanup() {
   port_forward_pid=""
 }
 
+# --fail-with-body and not -f: Prometheus reports a rejected query with an HTTP
+# error code and puts the reason in the body, which -f would throw away along
+# with the only explanation of what went wrong.
 prom_api() {
   local path="$1"
   shift
-  curl -sS -f --max-time 120 -G "http://127.0.0.1:${local_port}${path}" "$@"
+  curl -sS --fail-with-body --max-time 120 -G "http://127.0.0.1:${local_port}${path}" "$@"
 }
 
 # port-forward plus curl on the runner, not kubectl exec plus curl in the
@@ -221,6 +228,7 @@ if ! prom_api /api/v1/query_range \
   --data-urlencode "end=${window_end}" \
   --data-urlencode "step=${query_step}" > "${range_file}"; then
   echo "[ERROR] Prometheus rejected the range query 'ALERTS{alertname=~\"${alert_prefix}.*\"}' over ${window_start}..${window_end} with step ${query_step}s" >&2
+  echo "[ERROR] Response: $(jq -rc '.error // .' "${range_file}" 2>/dev/null || head -c 500 "${range_file}")" >&2
   exit 1
 fi
 
diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index 698c3c67a4..a02fab1d35 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -113,8 +113,14 @@ gate_accepted() {
 # Waits until the moduleconfig webhook answers again. It is served by
 # virtualization-controller itself and has no failurePolicy, so right after an
 # image switch every patch is rejected for a while. The probe dry-runs the gates
-# the config already carries: that adds no gate, so the validator returns nil
-# and only an unreachable webhook can fail it.
+# the config already carries: the spec does not change, so the generation does
+# not either, and the webhook skips its whole validator chain on that predicate -
+# leaving reachability as the only thing the probe can fail on. The API server
+# still calls the webhook, which is what makes it a reachability check at all.
+#
+# A real gate patch is a different matter: the chain then runs every validator
+# against the whole config - CIDRs, storage classes, DVCR, live migration - so
+# such a patch can be refused for a reason that has nothing to do with gates.
 # Usage: moduleconfig_writable [count] [delay]
 moduleconfig_writable() {
   local count="${1:-12}"

From d26251cb50a65e5c35ae0edd6ccef02cc4df50e5 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Wed, 12 Aug 2026 12:51:24 +0300
Subject: [PATCH 23/23] fix(ci): report only the alerts that fired

The report listed pending alerts as well, and a rollover puts almost every rule
with a `for` clause into pending for a moment, so the alerts that actually fired
were buried among a dozen that never did. Collect alertstate="firing" only - the
same set a cluster's ClusterAlerts hold, since such an object exists only while
its alert fires.

Filtering in the query and not in the report keeps the whole pipeline free of
records nobody wants. With pending gone the state is a constant, so it leaves the
record, the dedup key, the table column and the notice-versus-warning branch:
every alert that reaches the report now deserves the warning and the red job.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../scripts/bash/e2e/collect-clusteralerts.sh | 26 +++++-----
 .../scripts/bash/e2e/report-clusteralerts.sh  | 48 +++++++------------
 2 files changed, 31 insertions(+), 43 deletions(-)

diff --git a/.github/scripts/bash/e2e/collect-clusteralerts.sh b/.github/scripts/bash/e2e/collect-clusteralerts.sh
index 54a8052804..155b25da26 100644
--- a/.github/scripts/bash/e2e/collect-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/collect-clusteralerts.sh
@@ -14,14 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Collects the alerts of the virtualization module that were active in the
-# nested cluster during the release rollover. Runs once, at the end of the
-# pipeline, against the nested kubeconfig.
+# Collects the alerts of the virtualization module that fired in the nested
+# cluster during the release rollover. Runs once, at the end of the pipeline,
+# against the nested kubeconfig.
 #
 # Prometheus is the source rather than the ClusterAlerts objects: a ClusterAlert
-# exists only while its alert is active, so a single late look at the API server
+# exists only while its alert fires, so a single late look at the API server
 # would see nothing of what happened during the upgrade, while the ALERTS series
 # keep the whole history of the observation window.
+#
+# Only alertstate="firing" is collected, which is the same set the ClusterAlerts
+# of a cluster hold. Pending is deliberately left out: components restart during
+# a rollover, so rules with a `for` clause go pending on almost every run, and
+# reporting those buries the alerts that actually fired.
 
 set -Eeuo pipefail
 
@@ -179,24 +184,23 @@ def render($labels):
   | { phase: .[0].ph.phase,
       order: .[0].ph.order,
       name: $name,
-      alertstate: ($series.metric.alertstate // ""),
       severityLevel: ($series.metric.severity_level // ""),
       labels: $labels,
       firstSeen: (map(.t) | min | todate),
       lastSeen: (map(.t) | max | todate),
       summary: (($templates[$name].summary // "") | render($labels)),
       description: (($templates[$name].description // "") | render($labels)),
-      id: ([$name, ($series.metric.alertstate // ""), ($labels | tojson)] | join("|"))
+      id: ([$name, ($labels | tojson)] | join("|"))
     }
 ]
 | unique_by([.order, .id])
-| sort_by([.order, .name, .alertstate])
+| sort_by([.order, .name])
 | .[]
 '
 
 mkdir -p "${alerts_dir}"
 
-echo "[INFO] Collecting alerts matching '${alert_prefix}*' from Prometheus in ${prometheus_namespace}"
+echo "[INFO] Collecting firing alerts matching '${alert_prefix}*' from Prometheus in ${prometheus_namespace}"
 echo "[INFO] Observation window: ${window_start}..${window_end} ($(( window_end - window_start ))s), step ${query_step}s"
 echo "[INFO] Upgrade window: started_at=${started}, finished_at=${finished}"
 
@@ -223,11 +227,11 @@ start_port_forward "${pod}"
 # query_range and not query: an instant query would only see what is still
 # active now, while the report is about what was active during the rollover.
 if ! prom_api /api/v1/query_range \
-  --data-urlencode "query=ALERTS{alertname=~\"${alert_prefix}.*\"}" \
+  --data-urlencode "query=ALERTS{alertname=~\"${alert_prefix}.*\",alertstate=\"firing\"}" \
   --data-urlencode "start=${window_start}" \
   --data-urlencode "end=${window_end}" \
   --data-urlencode "step=${query_step}" > "${range_file}"; then
-  echo "[ERROR] Prometheus rejected the range query 'ALERTS{alertname=~\"${alert_prefix}.*\"}' over ${window_start}..${window_end} with step ${query_step}s" >&2
+  echo "[ERROR] Prometheus rejected the range query 'ALERTS{alertname=~\"${alert_prefix}.*\",alertstate=\"firing\"}' over ${window_start}..${window_end} with step ${query_step}s" >&2
   echo "[ERROR] Response: $(jq -rc '.error // .' "${range_file}" 2>/dev/null || head -c 500 "${range_file}")" >&2
   exit 1
 fi
@@ -247,4 +251,4 @@ jq -c \
 
 count="$(grep -c . "${alerts_log}" || true)"
 echo "[INFO] Collected ${count} alert record(s) into ${alerts_log}"
-jq -r '"  [\(.phase)] \(.name) \(.alertstate) (\(.firstSeen) .. \(.lastSeen))"' "${alerts_log}"
+jq -r '"  [\(.phase)] \(.name) (\(.firstSeen) .. \(.lastSeen))"' "${alerts_log}"
diff --git a/.github/scripts/bash/e2e/report-clusteralerts.sh b/.github/scripts/bash/e2e/report-clusteralerts.sh
index 74977a2096..3cfbcf5829 100644
--- a/.github/scripts/bash/e2e/report-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/report-clusteralerts.sh
@@ -59,9 +59,9 @@ if [ "${#logs[@]}" -eq 0 ]; then
 fi
 
 echo "[INFO] Reading collected ClusterAlerts from: ${logs[*]}"
-# The phase, the state and the firing interval come from the collector; sorting
-# is repeated here only to keep the order stable across several log files.
-alerts="$(jq -s 'sort_by([.order, .name, .alertstate])' "${logs[@]}")"
+# The phase and the firing interval come from the collector; sorting is repeated
+# here only to keep the order stable across several log files.
+alerts="$(jq -s 'sort_by([.order, .name])' "${logs[@]}")"
 
 count="$(jq 'length' <<< "${alerts}")"
 
@@ -70,47 +70,31 @@ count="$(jq 'length' <<< "${alerts}")"
 oneline='def oneline: gsub("\\s+"; " ") | sub("^ "; "") | sub(" $"; "");'
 
 if [ "${count}" -eq 0 ]; then
-  echo "No \`${alert_prefix}*\` alerts were active during the release rollover." >> "${summary_file}"
-  echo "[INFO] No ${alert_prefix}* alerts were active during the release rollover"
+  echo "No \`${alert_prefix}*\` alert fired during the release rollover." >> "${summary_file}"
+  echo "[INFO] No ${alert_prefix}* alert fired during the release rollover"
   exit 0
 fi
 
 {
-  echo "| Phase | Alert | State | Severity | First seen | Summary |"
-  echo "|---|---|---|---|---|---|"
-  jq -r "${oneline}"' .[] | "| \(.phase) | \(.name) | \(.alertstate) | \(.severityLevel) | \(.firstSeen) | \(.summary | oneline | gsub("\\|"; "\\|")) |"' <<< "${alerts}"
+  echo "| Phase | Alert | Severity | First seen | Summary |"
+  echo "|---|---|---|---|---|"
+  jq -r "${oneline}"' .[] | "| \(.phase) | \(.name) | \(.severityLevel) | \(.firstSeen) | \(.summary | oneline | gsub("\\|"; "\\|")) |"' <<< "${alerts}"
   echo
   echo "<details><summary>Alert details</summary>"
   echo
-  jq -r '.[] | "#### \(.name) — \(.phase) (\(.alertstate))\n\n- severity level: \(.severityLevel)\n- active: \(.firstSeen) .. \(.lastSeen)\n- labels: `\(.labels | tojson)`\n\n\(.description)\n"' <<< "${alerts}"
+  jq -r '.[] | "#### \(.name) — \(.phase)\n\n- severity level: \(.severityLevel)\n- active: \(.firstSeen) .. \(.lastSeen)\n- labels: `\(.labels | tojson)`\n\n\(.description)\n"' <<< "${alerts}"
   echo "</details>"
 } >> "${summary_file}"
 
 # Annotations put the alerts on top of the run page, not only in the summary.
-# A pending alert is a notice rather than a warning: it did not hold long enough
-# to be one.
 jq -r "${oneline}"' .[]
-  | (if .alertstate == "firing" then "::warning" else "::notice" end)
-    + " title=ClusterAlert \(.name)::[\(.phase)/\(.alertstate)] \(.summary | oneline)"' <<< "${alerts}"
-
-echo "[INFO] Active alerts:"
-jq -r '.[] | "  [\(.phase)] \(.name) \(.alertstate) (severity \(.severityLevel))"' <<< "${alerts}"
-
-firing_count="$(jq '[.[] | select(.alertstate == "firing")] | length' <<< "${alerts}")"
-
-# Only a fired alert paints the job red. Components restart during a rollover,
-# so rules with a `for` clause go pending on almost every run: failing on those
-# would make a red job the norm and tell the reviewer nothing.
-if [ "${firing_count}" -eq 0 ]; then
-  {
-    echo
-    echo "No \`${alert_prefix}*\` alert reached the firing state; ${count} were pending only."
-  } >> "${summary_file}"
-  echo "[INFO] No ${alert_prefix}* alert reached the firing state, ${count} were pending only"
-  exit 0
-fi
+  | "::warning title=ClusterAlert \(.name)::[\(.phase)] \(.summary | oneline)"' <<< "${alerts}"
+
+echo "[INFO] Fired alerts:"
+jq -r '.[] | "  [\(.phase)] \(.name) (severity \(.severityLevel))"' <<< "${alerts}"
 
 # Failing here is what paints this job red; the job itself is
-# continue-on-error, so the workflow conclusion stays successful.
-echo "[ERROR] ${firing_count} ClusterAlert(s) were firing in the nested cluster, see the job summary" >&2
+# continue-on-error, so the workflow conclusion stays successful. Only firing
+# alerts reach this script, so any of them is worth the red.
+echo "[ERROR] ${count} ClusterAlert(s) fired in the nested cluster, see the job summary" >&2
 exit 1