From b11981a95a2cac2eb2a50e9d305598459daaf808 Mon Sep 17 00:00:00 2001 From: Daniel Lawton Date: Tue, 7 Jul 2026 18:38:09 +0100 Subject: [PATCH] DNM: Add OCP upgrade stage for OSP 18 Port upgrade functionality from openshift-ir-plugin to support sequential OCP upgrades on OSP 18. Supports versions 4.16-4.23, automatically builds upgrade paths skipping unsupported versions like 4.17. Includes version-specific workarounds for 4.16 consoleplugins and 4.19 k8s API acknowledgment. Collects must-gather on failure. Change-Id: I8c0b804cb91774fb5bb5fccf47cdb6015d46d771 Signed-off-by: Daniel Lawton --- .../stages/roles/upgrade/defaults/main.yml | 27 ++ collection/stages/roles/upgrade/meta/main.yml | 3 + .../stages/roles/upgrade/tasks/main.yml | 51 ++++ .../roles/upgrade/tasks/single_upgrade.yml | 234 ++++++++++++++++++ .../upgrade_4.16_to_4.18_osp18.yaml | 16 ++ .../upgrade_4.16_to_4.21_osp18.yaml | 20 ++ .../upgrade_4.19_to_4.21_osp18.yaml | 16 ++ .../upgrade_4.22_to_4.23_osp18.yaml | 16 ++ playbooks/ocp_testing.yaml | 4 + playbooks/plays/upgrade.yaml | 17 ++ 10 files changed, 404 insertions(+) create mode 100644 collection/stages/roles/upgrade/defaults/main.yml create mode 100644 collection/stages/roles/upgrade/meta/main.yml create mode 100644 collection/stages/roles/upgrade/tasks/main.yml create mode 100644 collection/stages/roles/upgrade/tasks/single_upgrade.yml create mode 100644 jobs_definitions/upgrade_4.16_to_4.18_osp18.yaml create mode 100644 jobs_definitions/upgrade_4.16_to_4.21_osp18.yaml create mode 100644 jobs_definitions/upgrade_4.19_to_4.21_osp18.yaml create mode 100644 jobs_definitions/upgrade_4.22_to_4.23_osp18.yaml create mode 100644 playbooks/plays/upgrade.yaml diff --git a/collection/stages/roles/upgrade/defaults/main.yml b/collection/stages/roles/upgrade/defaults/main.yml new file mode 100644 index 00000000..5810d916 --- /dev/null +++ b/collection/stages/roles/upgrade/defaults/main.yml @@ -0,0 +1,27 @@ +--- +# Target version to upgrade to (e.g., "4.21") +# This should be set in job definitions +upgrade_target_version: "" + +# Optional: specific build name to upgrade to (e.g., "4.19.3") +# If empty, will use latest from channel +upgrade_build_name: "" + +# Optional: release channel (candidate, fast, stable, eus) +# Used if upgrade_build_name is empty +upgrade_channel: "candidate" + +# Wait parameters for health checks +wait_retries: 90 +wait_delay: 120 + +# OSP 18 supported OCP versions +# Note: 4.17 is NOT supported on OSP 18 +osp18_supported_versions: + - "4.16" + - "4.18" + - "4.19" + - "4.20" + - "4.21" + - "4.22" + - "4.23" diff --git a/collection/stages/roles/upgrade/meta/main.yml b/collection/stages/roles/upgrade/meta/main.yml new file mode 100644 index 00000000..ce0638dc --- /dev/null +++ b/collection/stages/roles/upgrade/meta/main.yml @@ -0,0 +1,3 @@ +--- +collections: + - shiftstack.tools diff --git a/collection/stages/roles/upgrade/tasks/main.yml b/collection/stages/roles/upgrade/tasks/main.yml new file mode 100644 index 00000000..8222bbf5 --- /dev/null +++ b/collection/stages/roles/upgrade/tasks/main.yml @@ -0,0 +1,51 @@ +--- +# Main entry point - orchestrates sequential OCP upgrades +# This role upgrades from the current cluster version to upgrade_target_version +# following OSP 18 supported version path (skipping unsupported versions like 4.17) + +- name: Validate required variables + ansible.builtin.assert: + that: + - upgrade_target_version is defined + - upgrade_target_version | length > 0 + fail_msg: "upgrade_target_version must be defined (e.g., '4.21')" + +- name: Discover current OCP version + ansible.builtin.include_role: + name: shiftstack.tools.tools_get_deploy_info + tasks_from: discover_ocp_version.yml + +- name: Display current cluster version + ansible.builtin.debug: + msg: "Current OCP version: {{ discovered_openshift_release }}" + +- name: Build upgrade path from current version to target (only OSP 18 supported versions) + ansible.builtin.set_fact: + upgrade_path: "{{ osp18_supported_versions | + select('version', discovered_openshift_release, '>') | + select('version', upgrade_target_version, '<=') | + list }}" + +- name: Validate upgrade path is not empty + ansible.builtin.assert: + that: + - upgrade_path | length > 0 + fail_msg: | + No upgrade path found from {{ discovered_openshift_release }} to {{ upgrade_target_version }}. + OSP 18 supported versions: {{ osp18_supported_versions | join(', ') }} + success_msg: "Upgrade path validated: {{ discovered_openshift_release }} → {{ upgrade_path | join(' → ') }}" + +- name: Display upgrade path + ansible.builtin.debug: + msg: "Upgrade path: {{ discovered_openshift_release }} → {{ upgrade_path | join(' → ') }}" + +- name: Execute sequential upgrades + ansible.builtin.include_tasks: single_upgrade.yml + loop: "{{ upgrade_path }}" + loop_control: + loop_var: target_release + label: "Upgrade to {{ target_release }}" + +- name: Display upgrade completion + ansible.builtin.debug: + msg: "Successfully upgraded cluster from {{ discovered_openshift_release }} to {{ upgrade_target_version }}" diff --git a/collection/stages/roles/upgrade/tasks/single_upgrade.yml b/collection/stages/roles/upgrade/tasks/single_upgrade.yml new file mode 100644 index 00000000..6e5841de --- /dev/null +++ b/collection/stages/roles/upgrade/tasks/single_upgrade.yml @@ -0,0 +1,234 @@ +--- +# Single upgrade step: upgrade from current version to target_release +# This file is included in a loop from main.yml with target_release variable + +- name: Check cluster health before upgrading to {{ target_release }} + ansible.builtin.include_role: + name: shiftstack.tools.tools_cluster_checks + vars: + actions: + - wait_until_cluster_is_healthy + - wait_until_nodes_ready + - wait_until_no_unschedulable_nodes + wait_retries: 1 + wait_delay: 0 + +- name: Get the ClusterVersion object before upgrade + ansible.builtin.shell: | + oc get clusterversion version -o json + register: cluster_version_json_before + changed_when: false + +- name: Parse ClusterVersion before upgrade + ansible.builtin.set_fact: + cluster_version_before_upgrade: "{{ cluster_version_json_before.stdout | from_json }}" + +- name: Get current cluster version from history + ansible.builtin.set_fact: + current_version: "{{ cluster_version_before_upgrade.status.history[0].version }}" + +- name: Check the version to upgrade to is >= the current version + ansible.builtin.assert: + that: + - target_release is version(current_version | regex_replace('^(\\d+\\.\\d+).*', '\\1'), '>=') + fail_msg: "The version to upgrade to ({{ target_release }}) must be >= the current version ({{ current_version }})" + +- name: Get the OCP release build name to upgrade to + ansible.builtin.include_role: + name: shiftstack.tools.tools_get_openshift_release + tasks_from: get_openshift_release_build_name.yml + vars: + openshift_release: "{{ target_release }}" + openshift_build_name: "{{ upgrade_build_name }}" + +- name: Print upgrade versions + ansible.builtin.debug: + msg: "Upgrading FROM: '{{ current_version }}' TO: '{{ openshift_release_build_name }}'" + +- name: Mark the OpenShift version to upgrade to + ansible.builtin.debug: + msg: "Build mark: upgrade_to={{ openshift_release_build_name }}" + +# Version-specific workarounds for OSP 18 +- name: Set patch data for consoleplugins (4.16 only) + ansible.builtin.set_fact: + console_patch: + storedVersions: + - v1 + when: openshift_release_build_name is regex('^4\\.16\\..*') + +- name: Set consoleplugins to v1 for 4.16 upgrade + ansible.builtin.shell: | + oc patch crd consoleplugins.console.openshift.io --subresource='status' --type="merge" -p '{{ console_patch | to_json }}' + when: openshift_release_build_name is regex('^4\\.16\\..*') + changed_when: true + +- name: Set ack data for 4.19 upgrade handholding + ansible.builtin.set_fact: + ack419_patch: + data: + ack-4.18-kube-1.32-api-removals-in-4.19: "true" + when: openshift_release_build_name is regex('^4\\.19\\..*') + +- name: Manual ack for 4.19 upgrade (k8s 1.32 API removals) + ansible.builtin.shell: | + oc -n openshift-config patch cm admin-acks --type="merge" -p '{{ ack419_patch | to_json }}' + when: openshift_release_build_name is regex('^4\\.19\\..*') + changed_when: true + +# Get the digest ID from the release image +- name: Create secret.json for OCP release info + ansible.builtin.copy: + content: "{{ ocp_pull_secret }}" + dest: /tmp/secret.json + mode: '0600' + +- name: Get the OCP release digest + ansible.builtin.shell: | + set -o pipefail + oc adm release info registry.ci.openshift.org/ocp/release:{{ openshift_release_build_name }} --registry-config=/tmp/secret.json | grep Digest | awk '{print $2}' + args: + executable: /bin/bash + register: release_digest + changed_when: false + +- name: Display release digest + ansible.builtin.debug: + msg: "{{ openshift_release_build_name }} release digest: {{ release_digest.stdout }}" + +- name: Run openshift upgrade command + ansible.builtin.shell: | + oc adm upgrade --to-image=registry.ci.openshift.org/ocp/release@{{ release_digest.stdout }} --allow-explicit-upgrade --force + register: upgrade + changed_when: true + +- name: Display upgrade command output + ansible.builtin.debug: + var: upgrade + +- name: Check the upgrade command returns ok + ansible.builtin.assert: + that: + - upgrade.rc == 0 + fail_msg: "oc adm upgrade command returned {{ upgrade.rc }} and stderr {{ upgrade.stderr_lines }}" + +- name: Set upgrade start datetime + ansible.builtin.set_fact: + start_time: "{{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%S') }}" + +- name: Wait until the upgrade starts (Progressing=True) + ansible.builtin.shell: | + set -o pipefail + oc get clusterversion version -o json | jq '.status.conditions[] | select(.type=="Progressing" and .status=="True")' + args: + executable: /bin/bash + register: progressing_check + until: progressing_check.stdout | length > 0 + retries: 20 + delay: 15 + changed_when: false + +# Main upgrade validation block with must-gather rescue +- name: Block for upgrade checks with must-gather on failure + block: + - name: Wait until the upgrade is completed (cluster is healthy) + ansible.builtin.include_role: + name: shiftstack.tools.tools_cluster_checks + vars: + actions: + - wait_until_cluster_is_healthy + wait_retries: 90 + wait_delay: 120 + + - name: Wait until cluster nodes are ready after OCP upgrade + ansible.builtin.include_role: + name: shiftstack.tools.tools_cluster_checks + vars: + actions: + - wait_until_nodes_ready + wait_retries: 80 + wait_delay: 30 + + - name: Wait until there are no unschedulable nodes after OCP upgrade + ansible.builtin.include_role: + name: shiftstack.tools.tools_cluster_checks + vars: + actions: + - wait_until_no_unschedulable_nodes + wait_retries: 80 + wait_delay: 30 + + - name: Set upgrade finish datetime + ansible.builtin.set_fact: + finish_time: "{{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%S') }}" + + - name: Set the upgrade execution time in minutes + ansible.builtin.set_fact: + upgrade_time: "{{ (((finish_time | to_datetime('%Y-%m-%dT%H:%M:%S')) - (start_time | to_datetime('%Y-%m-%dT%H:%M:%S'))).total_seconds() / 60) | int }}" + + - name: Mark the upgrade execution time + ansible.builtin.debug: + msg: "Build mark: upgrade_minutes={{ upgrade_time }}" + + - name: Check there are no pods in CrashLoopBackOff + ansible.builtin.include_role: + name: shiftstack.tools.tools_cluster_checks + vars: + actions: + - check_no_crashloopbackoff + + - name: Wait until the running cluster pods are ready after OCP upgrade + ansible.builtin.include_role: + name: shiftstack.tools.tools_cluster_checks + vars: + actions: + - wait_until_pods_ready + wait_retries: 20 + wait_delay: 30 + + - name: Get the ClusterVersion object after upgrade + ansible.builtin.shell: | + oc get clusterversion version -o json + register: cluster_version_json_after + changed_when: false + + - name: Parse ClusterVersion after upgrade + ansible.builtin.set_fact: + cluster_version_after: "{{ cluster_version_json_after.stdout | from_json }}" + + - name: Check the upgraded cluster version is the desired one + ansible.builtin.assert: + that: + - cluster_version_after.status.history[0].version == openshift_release_build_name + fail_msg: "Cluster upgraded to '{{ cluster_version_after.status.history[0].version }}' instead of '{{ openshift_release_build_name }}'" + success_msg: "Cluster successfully upgraded to '{{ cluster_version_after.status.history[0].version }}'" + + - name: Check cluster operators are ready after the OCP upgrade + ansible.builtin.include_role: + name: shiftstack.tools.tools_cluster_checks + vars: + actions: + - check_cluster_operators + release_name: "{{ openshift_release_build_name | default('') }}" + + rescue: + - name: Run must-gather for failed upgrade + ansible.builtin.include_role: + name: shiftstack.tools.tools_must-gather + vars: + must_gather_dir_suffix: "must-gather-upgrade-{{ target_release }}" + + - name: Fail inside rescue block + ansible.builtin.fail: + msg: "The cluster is not healthy after the upgrade to {{ target_release }}. Check must-gather artifacts." + +# Post-upgrade tasks +- name: Get the installer and oc client binaries for {{ openshift_release_build_name }} release + ansible.builtin.include_role: + name: shiftstack.tools.tools_get_openshift_release + tasks_from: get_openshift_release_binaries.yml + vars: + binaries: + - installer + - client + release_name: "{{ openshift_release_build_name | default('') }}" diff --git a/jobs_definitions/upgrade_4.16_to_4.18_osp18.yaml b/jobs_definitions/upgrade_4.16_to_4.18_osp18.yaml new file mode 100644 index 00000000..290a90d1 --- /dev/null +++ b/jobs_definitions/upgrade_4.16_to_4.18_osp18.yaml @@ -0,0 +1,16 @@ +--- +# Example job: Single-hop upgrade on OSP 18 +# Upgrades from 4.16 to 4.18 (skipping 4.17 which is not supported on OSP 18) +openshift_release: 4.16 +installation_type: ipi +stages: + - cleanup + - prepare + - install + - post + - verification + - upgrade + +# Target version for upgrade stage +upgrade_target_version: "4.18" +upgrade_channel: candidate diff --git a/jobs_definitions/upgrade_4.16_to_4.21_osp18.yaml b/jobs_definitions/upgrade_4.16_to_4.21_osp18.yaml new file mode 100644 index 00000000..33cde9da --- /dev/null +++ b/jobs_definitions/upgrade_4.16_to_4.21_osp18.yaml @@ -0,0 +1,20 @@ +--- +# Example job: Full OSP 18 upgrade chain +# Sequential upgrade: 4.16 → 4.18 → 4.19 → 4.20 → 4.21 +# Note: OSP 18 supports 4.16, 4.18, 4.19, 4.20, 4.21 (NOT 4.17) +openshift_release: 4.16 +installation_type: ipi +stages: + - cleanup + - prepare + - install + - post + - verification + - upgrade + +# Target version for upgrade stage +upgrade_target_version: "4.21" +upgrade_channel: candidate + +# Optional: Enable availability zones for testing CPMS updates +# provision_az_enable: true diff --git a/jobs_definitions/upgrade_4.19_to_4.21_osp18.yaml b/jobs_definitions/upgrade_4.19_to_4.21_osp18.yaml new file mode 100644 index 00000000..cf859208 --- /dev/null +++ b/jobs_definitions/upgrade_4.19_to_4.21_osp18.yaml @@ -0,0 +1,16 @@ +--- +# Example job: Upgrade from 4.19 to 4.21 on OSP 18 +# Tests the 4.19 ACK workaround when upgrading TO 4.19 +openshift_release: 4.19 +installation_type: ipi +stages: + - cleanup + - prepare + - install + - post + - verification + - upgrade + +# Target version for upgrade stage +upgrade_target_version: "4.21" +upgrade_channel: candidate diff --git a/jobs_definitions/upgrade_4.22_to_4.23_osp18.yaml b/jobs_definitions/upgrade_4.22_to_4.23_osp18.yaml new file mode 100644 index 00000000..c65ae727 --- /dev/null +++ b/jobs_definitions/upgrade_4.22_to_4.23_osp18.yaml @@ -0,0 +1,16 @@ +--- +# Example job: Upgrade from 4.22 to 4.23 on OSP 18 +# Single-hop upgrade for testing latest versions +openshift_release: 4.22 +installation_type: ipi +stages: + - cleanup + - prepare + - install + - post + - verification + - upgrade + +# Target version for upgrade stage +upgrade_target_version: "4.23" +upgrade_channel: candidate diff --git a/playbooks/ocp_testing.yaml b/playbooks/ocp_testing.yaml index 5e7a2031..c9592c60 100644 --- a/playbooks/ocp_testing.yaml +++ b/playbooks/ocp_testing.yaml @@ -84,6 +84,10 @@ ansible.builtin.import_playbook: plays/verification.yaml when: "'verification' in stages" +- name: Upgrade OpenShift cluster + ansible.builtin.import_playbook: plays/upgrade.yaml + when: "'upgrade' in stages" + - name: Day2ops OpenShift stage ansible.builtin.import_playbook: plays/day2ops.yaml when: "'day2ops' in stages" diff --git a/playbooks/plays/upgrade.yaml b/playbooks/plays/upgrade.yaml new file mode 100644 index 00000000..4e896c89 --- /dev/null +++ b/playbooks/plays/upgrade.yaml @@ -0,0 +1,17 @@ +--- +- name: Upgrade OpenShift cluster + hosts: installer + gather_facts: no + vars_files: + - "../../configs/global.yml" + tasks: + - name: Main block + block: + - name: Run OCP upgrade + ansible.builtin.include_role: + name: shiftstack.stages.upgrade + always: + - name: Synchronize artifacts + ansible.builtin.include_role: + name: shiftstack.tools.tools_ansible_inventory + tasks_from: sync_artifacts.yml