Skip to content

Commit b11981a

Browse files
dlaw4608dlaw4608
committed
DNM: Add OCP upgrade stage for OSP 18
 Port upgrade functionality from openshift-ir-plugin to support sequential OCP upgrades on OSP 18. Supports versions 4.16-4.23, automatically builds upgrade paths skipping unsupported versions like 4.17. Includes version-specific workarounds for 4.16 consoleplugins and 4.19 k8s API acknowledgment. Collects must-gather on failure. Change-Id: I8c0b804cb91774fb5bb5fccf47cdb6015d46d771 Signed-off-by: Daniel Lawton <dlawton@redhat.com>
1 parent 736ee2b commit b11981a

10 files changed

Lines changed: 404 additions & 0 deletions

File tree

‎collection/stages/roles/upgrade/defaults/main.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
---
2+
# Target version to upgrade to (e.g., "4.21")
3+
# This should be set in job definitions
4+
upgrade_target_version: ""
5+
6+
# Optional: specific build name to upgrade to (e.g., "4.19.3")
7+
# If empty, will use latest from channel
8+
upgrade_build_name: ""
9+
10+
# Optional: release channel (candidate, fast, stable, eus)
11+
# Used if upgrade_build_name is empty
12+
upgrade_channel: "candidate"
13+
14+
# Wait parameters for health checks
15+
wait_retries: 90
16+
wait_delay: 120
17+
18+
# OSP 18 supported OCP versions
19+
# Note: 4.17 is NOT supported on OSP 18
20+
osp18_supported_versions:
21+
- "4.16"
22+
- "4.18"
23+
- "4.19"
24+
- "4.20"
25+
- "4.21"
26+
- "4.22"
27+
- "4.23"

‎collection/stages/roles/upgrade/meta/main.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
---
2+
collections:
3+
- shiftstack.tools

‎collection/stages/roles/upgrade/tasks/main.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
---
2+
# Main entry point - orchestrates sequential OCP upgrades
3+
# This role upgrades from the current cluster version to upgrade_target_version
4+
# following OSP 18 supported version path (skipping unsupported versions like 4.17)
5+
6+
- name: Validate required variables
7+
ansible.builtin.assert:
8+
that:
9+
- upgrade_target_version is defined
10+
- upgrade_target_version | length > 0
11+
fail_msg: "upgrade_target_version must be defined (e.g., '4.21')"
12+
13+
- name: Discover current OCP version
14+
ansible.builtin.include_role:
15+
name: shiftstack.tools.tools_get_deploy_info
16+
tasks_from: discover_ocp_version.yml
17+
18+
- name: Display current cluster version
19+
ansible.builtin.debug:
20+
msg: "Current OCP version: {{ discovered_openshift_release }}"
21+
22+
- name: Build upgrade path from current version to target (only OSP 18 supported versions)
23+
ansible.builtin.set_fact:
24+
upgrade_path: "{{ osp18_supported_versions |
25+
select('version', discovered_openshift_release, '>') |
26+
select('version', upgrade_target_version, '<=') |
27+
list }}"
28+
29+
- name: Validate upgrade path is not empty
30+
ansible.builtin.assert:
31+
that:
32+
- upgrade_path | length > 0
33+
fail_msg: |
34+
No upgrade path found from {{ discovered_openshift_release }} to {{ upgrade_target_version }}.
35+
OSP 18 supported versions: {{ osp18_supported_versions | join(', ') }}
36+
success_msg: "Upgrade path validated: {{ discovered_openshift_release }} → {{ upgrade_path | join(' → ') }}"
37+
38+
- name: Display upgrade path
39+
ansible.builtin.debug:
40+
msg: "Upgrade path: {{ discovered_openshift_release }} → {{ upgrade_path | join(' → ') }}"
41+
42+
- name: Execute sequential upgrades
43+
ansible.builtin.include_tasks: single_upgrade.yml
44+
loop: "{{ upgrade_path }}"
45+
loop_control:
46+
loop_var: target_release
47+
label: "Upgrade to {{ target_release }}"
48+
49+
- name: Display upgrade completion
50+
ansible.builtin.debug:
51+
msg: "Successfully upgraded cluster from {{ discovered_openshift_release }} to {{ upgrade_target_version }}"

‎collection/stages/roles/upgrade/tasks/single_upgrade.yml‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
---
2+
# Single upgrade step: upgrade from current version to target_release
3+
# This file is included in a loop from main.yml with target_release variable
4+
5+
- name: Check cluster health before upgrading to {{ target_release }}
6+
ansible.builtin.include_role:
7+
name: shiftstack.tools.tools_cluster_checks
8+
vars:
9+
actions:
10+
- wait_until_cluster_is_healthy
11+
- wait_until_nodes_ready
12+
- wait_until_no_unschedulable_nodes
13+
wait_retries: 1
14+
wait_delay: 0
15+
16+
- name: Get the ClusterVersion object before upgrade
17+
ansible.builtin.shell: |
18+
oc get clusterversion version -o json
19+
register: cluster_version_json_before
20+
changed_when: false
21+
22+
- name: Parse ClusterVersion before upgrade
23+
ansible.builtin.set_fact:
24+
cluster_version_before_upgrade: "{{ cluster_version_json_before.stdout | from_json }}"
25+
26+
- name: Get current cluster version from history
27+
ansible.builtin.set_fact:
28+
current_version: "{{ cluster_version_before_upgrade.status.history[0].version }}"
29+
30+
- name: Check the version to upgrade to is >= the current version
31+
ansible.builtin.assert:
32+
that:
33+
- target_release is version(current_version | regex_replace('^(\\d+\\.\\d+).*', '\\1'), '>=')
34+
fail_msg: "The version to upgrade to ({{ target_release }}) must be >= the current version ({{ current_version }})"
35+
36+
- name: Get the OCP release build name to upgrade to
37+
ansible.builtin.include_role:
38+
name: shiftstack.tools.tools_get_openshift_release
39+
tasks_from: get_openshift_release_build_name.yml
40+
vars:
41+
openshift_release: "{{ target_release }}"
42+
openshift_build_name: "{{ upgrade_build_name }}"
43+
44+
- name: Print upgrade versions
45+
ansible.builtin.debug:
46+
msg: "Upgrading FROM: '{{ current_version }}' TO: '{{ openshift_release_build_name }}'"
47+
48+
- name: Mark the OpenShift version to upgrade to
49+
ansible.builtin.debug:
50+
msg: "Build mark: upgrade_to={{ openshift_release_build_name }}"
51+
52+
# Version-specific workarounds for OSP 18
53+
- name: Set patch data for consoleplugins (4.16 only)
54+
ansible.builtin.set_fact:
55+
console_patch:
56+
storedVersions:
57+
- v1
58+
when: openshift_release_build_name is regex('^4\\.16\\..*')
59+
60+
- name: Set consoleplugins to v1 for 4.16 upgrade
61+
ansible.builtin.shell: |
62+
oc patch crd consoleplugins.console.openshift.io --subresource='status' --type="merge" -p '{{ console_patch | to_json }}'
63+
when: openshift_release_build_name is regex('^4\\.16\\..*')
64+
changed_when: true
65+
66+
- name: Set ack data for 4.19 upgrade handholding
67+
ansible.builtin.set_fact:
68+
ack419_patch:
69+
data:
70+
ack-4.18-kube-1.32-api-removals-in-4.19: "true"
71+
when: openshift_release_build_name is regex('^4\\.19\\..*')
72+
73+
- name: Manual ack for 4.19 upgrade (k8s 1.32 API removals)
74+
ansible.builtin.shell: |
75+
oc -n openshift-config patch cm admin-acks --type="merge" -p '{{ ack419_patch | to_json }}'
76+
when: openshift_release_build_name is regex('^4\\.19\\..*')
77+
changed_when: true
78+
79+
# Get the digest ID from the release image
80+
- name: Create secret.json for OCP release info
81+
ansible.builtin.copy:
82+
content: "{{ ocp_pull_secret }}"
83+
dest: /tmp/secret.json
84+
mode: '0600'
85+
86+
- name: Get the OCP release digest
87+
ansible.builtin.shell: |
88+
set -o pipefail
89+
oc adm release info registry.ci.openshift.org/ocp/release:{{ openshift_release_build_name }} --registry-config=/tmp/secret.json | grep Digest | awk '{print $2}'
90+
args:
91+
executable: /bin/bash
92+
register: release_digest
93+
changed_when: false
94+
95+
- name: Display release digest
96+
ansible.builtin.debug:
97+
msg: "{{ openshift_release_build_name }} release digest: {{ release_digest.stdout }}"
98+
99+
- name: Run openshift upgrade command
100+
ansible.builtin.shell: |
101+
oc adm upgrade --to-image=registry.ci.openshift.org/ocp/release@{{ release_digest.stdout }} --allow-explicit-upgrade --force
102+
register: upgrade
103+
changed_when: true
104+
105+
- name: Display upgrade command output
106+
ansible.builtin.debug:
107+
var: upgrade
108+
109+
- name: Check the upgrade command returns ok
110+
ansible.builtin.assert:
111+
that:
112+
- upgrade.rc == 0
113+
fail_msg: "oc adm upgrade command returned {{ upgrade.rc }} and stderr {{ upgrade.stderr_lines }}"
114+
115+
- name: Set upgrade start datetime
116+
ansible.builtin.set_fact:
117+
start_time: "{{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%S') }}"
118+
119+
- name: Wait until the upgrade starts (Progressing=True)
120+
ansible.builtin.shell: |
121+
set -o pipefail
122+
oc get clusterversion version -o json | jq '.status.conditions[] | select(.type=="Progressing" and .status=="True")'
123+
args:
124+
executable: /bin/bash
125+
register: progressing_check
126+
until: progressing_check.stdout | length > 0
127+
retries: 20
128+
delay: 15
129+
changed_when: false
130+
131+
# Main upgrade validation block with must-gather rescue
132+
- name: Block for upgrade checks with must-gather on failure
133+
block:
134+
- name: Wait until the upgrade is completed (cluster is healthy)
135+
ansible.builtin.include_role:
136+
name: shiftstack.tools.tools_cluster_checks
137+
vars:
138+
actions:
139+
- wait_until_cluster_is_healthy
140+
wait_retries: 90
141+
wait_delay: 120
142+
143+
- name: Wait until cluster nodes are ready after OCP upgrade
144+
ansible.builtin.include_role:
145+
name: shiftstack.tools.tools_cluster_checks
146+
vars:
147+
actions:
148+
- wait_until_nodes_ready
149+
wait_retries: 80
150+
wait_delay: 30
151+
152+
- name: Wait until there are no unschedulable nodes after OCP upgrade
153+
ansible.builtin.include_role:
154+
name: shiftstack.tools.tools_cluster_checks
155+
vars:
156+
actions:
157+
- wait_until_no_unschedulable_nodes
158+
wait_retries: 80
159+
wait_delay: 30
160+
161+
- name: Set upgrade finish datetime
162+
ansible.builtin.set_fact:
163+
finish_time: "{{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%S') }}"
164+
165+
- name: Set the upgrade execution time in minutes
166+
ansible.builtin.set_fact:
167+
upgrade_time: "{{ (((finish_time | to_datetime('%Y-%m-%dT%H:%M:%S')) - (start_time | to_datetime('%Y-%m-%dT%H:%M:%S'))).total_seconds() / 60) | int }}"
168+
169+
- name: Mark the upgrade execution time
170+
ansible.builtin.debug:
171+
msg: "Build mark: upgrade_minutes={{ upgrade_time }}"
172+
173+
- name: Check there are no pods in CrashLoopBackOff
174+
ansible.builtin.include_role:
175+
name: shiftstack.tools.tools_cluster_checks
176+
vars:
177+
actions:
178+
- check_no_crashloopbackoff
179+
180+
- name: Wait until the running cluster pods are ready after OCP upgrade
181+
ansible.builtin.include_role:
182+
name: shiftstack.tools.tools_cluster_checks
183+
vars:
184+
actions:
185+
- wait_until_pods_ready
186+
wait_retries: 20
187+
wait_delay: 30
188+
189+
- name: Get the ClusterVersion object after upgrade
190+
ansible.builtin.shell: |
191+
oc get clusterversion version -o json
192+
register: cluster_version_json_after
193+
changed_when: false
194+
195+
- name: Parse ClusterVersion after upgrade
196+
ansible.builtin.set_fact:
197+
cluster_version_after: "{{ cluster_version_json_after.stdout | from_json }}"
198+
199+
- name: Check the upgraded cluster version is the desired one
200+
ansible.builtin.assert:
201+
that:
202+
- cluster_version_after.status.history[0].version == openshift_release_build_name
203+
fail_msg: "Cluster upgraded to '{{ cluster_version_after.status.history[0].version }}' instead of '{{ openshift_release_build_name }}'"
204+
success_msg: "Cluster successfully upgraded to '{{ cluster_version_after.status.history[0].version }}'"
205+
206+
- name: Check cluster operators are ready after the OCP upgrade
207+
ansible.builtin.include_role:
208+
name: shiftstack.tools.tools_cluster_checks
209+
vars:
210+
actions:
211+
- check_cluster_operators
212+
release_name: "{{ openshift_release_build_name | default('') }}"
213+
214+
rescue:
215+
- name: Run must-gather for failed upgrade
216+
ansible.builtin.include_role:
217+
name: shiftstack.tools.tools_must-gather
218+
vars:
219+
must_gather_dir_suffix: "must-gather-upgrade-{{ target_release }}"
220+
221+
- name: Fail inside rescue block
222+
ansible.builtin.fail:
223+
msg: "The cluster is not healthy after the upgrade to {{ target_release }}. Check must-gather artifacts."
224+
225+
# Post-upgrade tasks
226+
- name: Get the installer and oc client binaries for {{ openshift_release_build_name }} release
227+
ansible.builtin.include_role:
228+
name: shiftstack.tools.tools_get_openshift_release
229+
tasks_from: get_openshift_release_binaries.yml
230+
vars:
231+
binaries:
232+
- installer
233+
- client
234+
release_name: "{{ openshift_release_build_name | default('') }}"

‎jobs_definitions/upgrade_4.16_to_4.18_osp18.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
# Example job: Single-hop upgrade on OSP 18
3+
# Upgrades from 4.16 to 4.18 (skipping 4.17 which is not supported on OSP 18)
4+
openshift_release: 4.16
5+
installation_type: ipi
6+
stages:
7+
- cleanup
8+
- prepare
9+
- install
10+
- post
11+
- verification
12+
- upgrade
13+
14+
# Target version for upgrade stage
15+
upgrade_target_version: "4.18"
16+
upgrade_channel: candidate

‎jobs_definitions/upgrade_4.16_to_4.21_osp18.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
---
2+
# Example job: Full OSP 18 upgrade chain
3+
# Sequential upgrade: 4.16 → 4.18 → 4.19 → 4.20 → 4.21
4+
# Note: OSP 18 supports 4.16, 4.18, 4.19, 4.20, 4.21 (NOT 4.17)
5+
openshift_release: 4.16
6+
installation_type: ipi
7+
stages:
8+
- cleanup
9+
- prepare
10+
- install
11+
- post
12+
- verification
13+
- upgrade
14+
15+
# Target version for upgrade stage
16+
upgrade_target_version: "4.21"
17+
upgrade_channel: candidate
18+
19+
# Optional: Enable availability zones for testing CPMS updates
20+
# provision_az_enable: true

‎jobs_definitions/upgrade_4.19_to_4.21_osp18.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
# Example job: Upgrade from 4.19 to 4.21 on OSP 18
3+
# Tests the 4.19 ACK workaround when upgrading TO 4.19
4+
openshift_release: 4.19
5+
installation_type: ipi
6+
stages:
7+
- cleanup
8+
- prepare
9+
- install
10+
- post
11+
- verification
12+
- upgrade
13+
14+
# Target version for upgrade stage
15+
upgrade_target_version: "4.21"
16+
upgrade_channel: candidate

‎jobs_definitions/upgrade_4.22_to_4.23_osp18.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
# Example job: Upgrade from 4.22 to 4.23 on OSP 18
3+
# Single-hop upgrade for testing latest versions
4+
openshift_release: 4.22
5+
installation_type: ipi
6+
stages:
7+
- cleanup
8+
- prepare
9+
- install
10+
- post
11+
- verification
12+
- upgrade
13+
14+
# Target version for upgrade stage
15+
upgrade_target_version: "4.23"
16+
upgrade_channel: candidate

0 commit comments

Comments
 (0)