diff --git a/.gitignore b/.gitignore index 151070fe77..6a29f5b453 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ - *.DS_Store* .env .venv @@ -20,4 +19,4 @@ cpd-cli-workspace/* /tmp /node_modules package-lock.json -package.json +package.json \ No newline at end of file diff --git a/ibm/mas_devops/common_vars/compatibility_matrix.yml b/ibm/mas_devops/common_vars/compatibility_matrix.yml index 62656a2ef1..ac34836a09 100644 --- a/ibm/mas_devops/common_vars/compatibility_matrix.yml +++ b/ibm/mas_devops/common_vars/compatibility_matrix.yml @@ -116,4 +116,4 @@ upgrade_path: 8.9.x: 8.10.x aiservice_upgrade_path: - 9.1.x: 9.2.x-feature + 9.1.x: [9.2.x-feature, 9.2.x-jhdev] diff --git a/ibm/mas_devops/plugins/filter/filters.py b/ibm/mas_devops/plugins/filter/filters.py index 1f3654739d..669205e66e 100644 --- a/ibm/mas_devops/plugins/filter/filters.py +++ b/ibm/mas_devops/plugins/filter/filters.py @@ -7,6 +7,7 @@ # ----------------------------------------------------------- import yaml import re +import copy def private_vlan(vlans): @@ -437,6 +438,77 @@ def get_ecr_repositories(image_mirror_output): repositories.append(repo_to_add) return repositories +def is_channel_upgrade_path_valid(current: str, target: str, valid_paths: dict) -> bool: + """ + Checks if a given current channel version can be upgraded to a target channel version. + :current: The current channel version. + :target: The target channel version to upgrade to. + :valid_paths: A dictionary of supported upgrade paths. See ibm/mas_devops/common_vars/compatibility_matrix.yml. + :return: True if the upgrade path is supported, False otherwise. + """ + valid = False + if current not in valid_paths.keys(): + print(f'Current channel {current} is not supported for upgrade') + else: + allowed_targets = valid_paths[current] + if isinstance(allowed_targets, str): + if target != allowed_targets: + print(f'Upgrading from channel {current} to {target} is not supported') + else: + valid = True + elif isinstance(allowed_targets, list): + if target not in allowed_targets: + print(f'Upgrading from channel {current} to {target} is not supported') + else: + valid = True + else: + print(f'Error: channel upgrade compatibility matrix is incorrectly defined') + return valid + +def remove_dict_keys(data: dict, keys: list[str], deep_copy: bool = True) -> dict: + """ + Deletes keys from a dictionary. This has an advantage over Ansible's ansible.utils.remove_keys filter + in that nested keys are given explicitly in dot notation, for example 'a.b.c'. + :data: The input dictionary. + :keys: A list of key strings in dot notation, e.g. ['a.b', 'c.d.e']. + :deep_copy: Set to False to modify the input dictionary in-place, otherwise a copy will be modified. + :return: The dictionary with keys removed. + """ + if deep_copy: + data = copy.deepcopy(data) + for key in keys: + parts = key.split('.') + ref = 'data' + for part in parts: + ref += f'["{part}"]' + try: + exec(f'del {ref}') + except KeyError as ex: + print(f'Could not delete key from dictionary: {ex}') + return data + +def is_operator_upgraded_by_version(cr_reconciled_version: str, opcon_version: str, sub_installed_version: str) -> bool: + """ + Checks if an operator was upgraded successfully by comparing versions. Typically we just compare the version reported as + reconciled in the operator's custom resource with the OperatorCondition, however, this poses a problem for certain images + that are not tagged with a build number but the build number is used in other places within its bundle. For example, an + upgraded operator might have the CR reconciled version as "9.2.0-pre.1450" (derived from its image tag) but the version + from the OperatorCondition resource is "9.2.0-pre.1450-5075" which includes the build number. In this case where they + don't match we do our best effort by also checking the version reported as installed in the Subscription. + :cr_reconciled_version: Version number from the "versions.reconciled" field in the CR. + :opcon_version: Version derived from the operator's OperatorCondition. + :sub_installed_version: Version derived from the Subscription's "status.installedCSV" field. + :return: True if the operator was successfully upgraded (at least by checking various versions) + """ + print(f'{cr_reconciled_version} --- {opcon_version} --- {sub_installed_version}') + upgraded = False + opcon_version = opcon_version.replace('+', '-') + prefix = f'{cr_reconciled_version}-' + if cr_reconciled_version == opcon_version: + upgraded = True + elif opcon_version.startswith(prefix) and sub_installed_version.startswith(prefix) and (opcon_version == sub_installed_version): + upgraded = True + return upgraded class FilterModule(object): def filters(self): @@ -459,5 +531,8 @@ def filters(self): 'format_pre_version_without_buildid': format_pre_version_without_buildid, 'format_pre_version_with_buildid': format_pre_version_with_buildid, 'get_db2_instance_name': get_db2_instance_name, - 'get_ecr_repositories': get_ecr_repositories + 'get_ecr_repositories': get_ecr_repositories, + 'is_channel_upgrade_path_valid': is_channel_upgrade_path_valid, + 'remove_dict_keys': remove_dict_keys, + 'is_operator_upgraded_by_version': is_operator_upgraded_by_version, } diff --git a/ibm/mas_devops/roles/aiservice/tasks/aiservice/main.yml b/ibm/mas_devops/roles/aiservice/tasks/aiservice/main.yml index 3e4c11b95d..85c1faac43 100644 --- a/ibm/mas_devops/roles/aiservice/tasks/aiservice/main.yml +++ b/ibm/mas_devops/roles/aiservice/tasks/aiservice/main.yml @@ -7,8 +7,6 @@ - "Namespace ......................... {{ aiservice_namespace }}" - "Channel ........................... {{ aiservice_channel }}" - "MAS Instance Id ................... {{ aiservice_instance_id }}" - - "Channel ........................... {{ aiservice_channel }}" - - "" - "aiservice_s3_region ............... {{ aiservice_s3_region }}" - "aiservice_s3_host ................. {{ aiservice_s3_host }}" - "aiservice_s3_port ................. {{ aiservice_s3_port }}" diff --git a/ibm/mas_devops/roles/aiservice_tenant/defaults/main.yml b/ibm/mas_devops/roles/aiservice_tenant/defaults/main.yml index 981c710f2a..afc96abb3a 100644 --- a/ibm/mas_devops/roles/aiservice_tenant/defaults/main.yml +++ b/ibm/mas_devops/roles/aiservice_tenant/defaults/main.yml @@ -1,7 +1,16 @@ --- aiservice_instance_id: "{{ lookup('env', 'AISERVICE_INSTANCE_ID') }}" aiservice_namespace: "{{ lookup('env', 'AISERVICE_NAMESPACE') | default('aiservice-{}'.format(aiservice_instance_id), true) }}" -aiservice_channel: "{{ lookup('env', 'AISERVICE_CHANNEL') }}" +aiservice_channel: "{{ lookup('env', 'AISERVICE_TENANT_CHANNEL') }}" + +ibm_entitlement_username: "{{ lookup('env','IBM_ENTITLEMENT_USERNAME') }}" +mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | default('cp', true) }}" +ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" +mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" + +# Development Registry Entitlement +artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" +artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-operator-catalog', true) }}" @@ -43,6 +52,7 @@ aiservice_watsonxai_full: "{{ lookup('env', 'AISERVICE_WATSONXAI_FULL') | defaul aiservice_watsonxai_instance_id: "{{ lookup('env', 'AISERVICE_WATSONXAI_INSTANCE_ID') | default('openshift', true) }}" aiservice_watsonxai_username: "{{ lookup('env', 'AISERVICE_WATSONXAI_USERNAME') | default('', true) }}" aiservice_watsonxai_version: "{{ lookup('env', 'AISERVICE_WATSONXAI_VERSION') | default('', true) }}" + aiservice_watsonxai_verify: "{{ lookup('env', 'AISERVICE_WATSONXAI_VERIFY') | default('true', true) | bool }}" # DRO diff --git a/ibm/mas_devops/roles/aiservice_tenant/tasks/aiservice/main.yml b/ibm/mas_devops/roles/aiservice_tenant/tasks/aiservice/main.yml new file mode 100644 index 0000000000..dbdccfac33 --- /dev/null +++ b/ibm/mas_devops/roles/aiservice_tenant/tasks/aiservice/main.yml @@ -0,0 +1,63 @@ +--- +# 1. Provide Debug information +# ----------------------------------------------------------------------------- +- name: "Debug information - IBM Maximo AI Service" + debug: + msg: + - "Namespace ......................... {{ tenantNamespace }}" + - "Channel ........................... {{ aiservice_channel }}" + - "MAS Instance Id ................... {{ aiservice_instance_id }}" + + + +# 2. Install the operator & create entitlement secret +# ----------------------------------------------------------------------------- +- name: "Create IBM Entitlement Key" + ibm.mas_devops.update_ibm_entitlement: + namespace: "{{ tenantNamespace }}" + icr_username: "{{ mas_entitlement_username }}" + icr_password: "{{ mas_entitlement_key }}" + artifactory_username: "{{ artifactory_username }}" + artifactory_password: "{{ artifactory_token }}" + namespace_kyverno_label: "audit" + +- name: "Create ibm-aiservice Subscription" + ibm.mas_devops.apply_subscription: + namespace: "{{ tenantNamespace }}" + package_name: ibm-aiservice-tenant + package_channel: "{{ aiservice_channel }}" + catalog_source: "{{ mas_catalog_source }}" + register: subscription + + +# 3. Wait until the IBM Maximo AI Service CRD is available +# ----------------------------------------------------------------------------- +- name: "Wait until the IBM Maximo AI Service Operator CRD is available" + include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" + vars: + crd_name: aiservicetenants.aiservice.ibm.com + + +# 4. Lookup storage class availability +# ----------------------------------------------------------------------------- +- name: "Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" + + +# 5. Set AI Service Storage (Required) +# ----------------------------------------------------------------------------- +- name: "Default AI Service Storage if not set by user" + when: aiservice_storage_class is not defined or aiservice_storage_class == "" + set_fact: + aiservice_storage_class: "{{ defaultStorageClasses.rwx }}" + +- name: "Assert that primary storage class has been defined" + assert: + that: aiservice_storage_class is defined and aiservice_storage_class != "" + fail_msg: "aiservice_storage_class must be defined" + +- name: "Debug" + debug: + msg: "AI Service storage class ................. {{ aiservice_storage_class }}" + + diff --git a/ibm/mas_devops/roles/aiservice_tenant/tasks/config_dro/main.yml b/ibm/mas_devops/roles/aiservice_tenant/tasks/config_dro/main.yml index e309440b49..11fec155d4 100644 --- a/ibm/mas_devops/roles/aiservice_tenant/tasks/config_dro/main.yml +++ b/ibm/mas_devops/roles/aiservice_tenant/tasks/config_dro/main.yml @@ -47,5 +47,5 @@ - name: Create DRO secret kubernetes.core.k8s: state: present - namespace: "{{ aiservice_namespace }}" + namespace: "{{ tenantNamespace }}" template: "templates/dro/dro-secret.yml.j2" diff --git a/ibm/mas_devops/roles/aiservice_tenant/tasks/config_rsl/main.yml b/ibm/mas_devops/roles/aiservice_tenant/tasks/config_rsl/main.yml index 4d00b28600..4db17ef94d 100644 --- a/ibm/mas_devops/roles/aiservice_tenant/tasks/config_rsl/main.yml +++ b/ibm/mas_devops/roles/aiservice_tenant/tasks/config_rsl/main.yml @@ -16,6 +16,6 @@ - name: Create RSL secret kubernetes.core.k8s: state: present - namespace: "{{ aiservice_namespace }}" + namespace: "{{ tenantNamespace }}" template: "templates/rsl/rsl-secret.yml.j2" when: rsl_config_valid diff --git a/ibm/mas_devops/roles/aiservice_tenant/tasks/config_sls/main.yml b/ibm/mas_devops/roles/aiservice_tenant/tasks/config_sls/main.yml index 8378918f32..13e021c52a 100644 --- a/ibm/mas_devops/roles/aiservice_tenant/tasks/config_sls/main.yml +++ b/ibm/mas_devops/roles/aiservice_tenant/tasks/config_sls/main.yml @@ -47,5 +47,5 @@ - name: Create SLS secret kubernetes.core.k8s: state: present - namespace: "{{ aiservice_namespace }}" + namespace: "{{ tenantNamespace }}" template: "templates/sls/sls-secret.yml.j2" diff --git a/ibm/mas_devops/roles/aiservice_tenant/tasks/main.yml b/ibm/mas_devops/roles/aiservice_tenant/tasks/main.yml index 7be0646f88..045b402185 100644 --- a/ibm/mas_devops/roles/aiservice_tenant/tasks/main.yml +++ b/ibm/mas_devops/roles/aiservice_tenant/tasks/main.yml @@ -13,5 +13,8 @@ # create wx secret - include_tasks: tasks/watsonx/main.yml +# Install the operator +- include_tasks: tasks/aiservice/main.yml + # create AI Broker tenant - include_tasks: tasks/tenant/main.yml diff --git a/ibm/mas_devops/roles/aiservice_tenant/tasks/tenant/install/main.yml b/ibm/mas_devops/roles/aiservice_tenant/tasks/tenant/install/main.yml index a53345f07b..9028fcad7a 100644 --- a/ibm/mas_devops/roles/aiservice_tenant/tasks/tenant/install/main.yml +++ b/ibm/mas_devops/roles/aiservice_tenant/tasks/tenant/install/main.yml @@ -4,14 +4,14 @@ annotation_dict: "{{ mas_annotations | string | ibm.mas_devops.getAnnotations() }}" kubernetes.core.k8s: state: present - namespace: "{{ aiservice_namespace }}" + namespace: "{{ tenantNamespace }}" template: templates/aiservice/aiservicetenant.yml.j2 - name: "Wait for tenant CR to be ready" kubernetes.core.k8s_info: api_version: aiservice.ibm.com/v1 name: "{{ tenantNamespace }}" - namespace: "{{ aiservice_namespace }}" + namespace: "{{ tenantNamespace }}" kind: AIServiceTenant register: aiservicetenant_cr_result until: diff --git a/ibm/mas_devops/roles/aiservice_tenant/templates/aiservice/aiservicetenant.yml.j2 b/ibm/mas_devops/roles/aiservice_tenant/templates/aiservice/aiservicetenant.yml.j2 index 57dae1f9b8..1793e6715a 100644 --- a/ibm/mas_devops/roles/aiservice_tenant/templates/aiservice/aiservicetenant.yml.j2 +++ b/ibm/mas_devops/roles/aiservice_tenant/templates/aiservice/aiservicetenant.yml.j2 @@ -3,7 +3,7 @@ apiVersion: aiservice.ibm.com/v1 kind: AIServiceTenant metadata: name: "{{ tenantNamespace }}" - namespace: "{{ aiservice_namespace }}" + namespace: "{{ tenantNamespace }}" annotations: ansible.sdk.operatorframework.io/verbosity: "{{ aiservice_operator_log_level }}" labels: diff --git a/ibm/mas_devops/roles/aiservice_tenant/templates/dro/dro-secret.yml.j2 b/ibm/mas_devops/roles/aiservice_tenant/templates/dro/dro-secret.yml.j2 index 4662b0eadf..fd3bf76736 100644 --- a/ibm/mas_devops/roles/aiservice_tenant/templates/dro/dro-secret.yml.j2 +++ b/ibm/mas_devops/roles/aiservice_tenant/templates/dro/dro-secret.yml.j2 @@ -4,7 +4,7 @@ apiVersion: v1 type: Opaque metadata: name: {{ aiservice_dro_token_secret }} - namespace: {{ aiservice_namespace }} + namespace: {{ tenantNamespace }} labels: aiservice.ibm.com/instanceId: "{{ aiservice_instance_id }}" {% if custom_labels is defined and custom_labels.items() %} diff --git a/ibm/mas_devops/roles/aiservice_tenant/templates/sls/sls-secret.yml.j2 b/ibm/mas_devops/roles/aiservice_tenant/templates/sls/sls-secret.yml.j2 index d3fffa920a..33bf3c2499 100644 --- a/ibm/mas_devops/roles/aiservice_tenant/templates/sls/sls-secret.yml.j2 +++ b/ibm/mas_devops/roles/aiservice_tenant/templates/sls/sls-secret.yml.j2 @@ -4,7 +4,7 @@ apiVersion: v1 type: Opaque metadata: name: {{ aiservice_sls_secret }} - namespace: {{ aiservice_namespace }} + namespace: {{ tenantNamespace }} labels: aiservice.ibm.com/instanceId: "{{ aiservice_instance_id }}" {% if custom_labels is defined and custom_labels.items() %} diff --git a/ibm/mas_devops/roles/aiservice_upgrade/defaults/main.yml b/ibm/mas_devops/roles/aiservice_upgrade/defaults/main.yml index 08592ea31d..054c99de88 100644 --- a/ibm/mas_devops/roles/aiservice_upgrade/defaults/main.yml +++ b/ibm/mas_devops/roles/aiservice_upgrade/defaults/main.yml @@ -1,5 +1,15 @@ --- +mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-operator-catalog', true) }}" +mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | default('cp', true) }}" +mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}" + +artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}" +artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" + aiservice_upgrade_dryrun: "{{ lookup('env', 'AISERVICE_UPGRADE_DRYRUN') | default('False', True) | bool }}" aiservice_channel: "{{ lookup('env', 'AISERVICE_CHANNEL') }}" aiservice_instance_id: "{{ lookup('env', 'AISERVICE_INSTANCE_ID') }}" aiservice_namespace: "aiservice-{{ aiservice_instance_id }}" +aiservice_force_migration: "{{ lookup('env', 'AISERVICE_FORCE_MIGRATION') | default('False', True) | bool }}" +aiservice_install_num_retries: "{{ lookup('env', 'AISERVICE_INSTALL_NUM_RETRIES') | default('60', True) | int }}" +aiservice_install_wait_sec: "{{ lookup('env', 'AISERVICE_INSTALL_WAIT_SEC') | default('60', True) | int }}" diff --git a/ibm/mas_devops/roles/aiservice_upgrade/tasks/main.yml b/ibm/mas_devops/roles/aiservice_upgrade/tasks/main.yml index dc9db0f4c3..e370373786 100644 --- a/ibm/mas_devops/roles/aiservice_upgrade/tasks/main.yml +++ b/ibm/mas_devops/roles/aiservice_upgrade/tasks/main.yml @@ -27,6 +27,15 @@ - "operators.coreos.com/ibm-aiservice.{{ aiservice_namespace }}" register: aiservice_sub_info +- name: "Set current subscription channel" + set_fact: + current_aiservice_channel: "{{ aiservice_sub_info.resources[0].spec.channel }}" + +- name: Show current subscription info + ansible.builtin.debug: + msg: + - "Currently installed AI Service channel ............. {{ current_aiservice_channel }}" + - name: "Set default upgrade target based on installed version of AI Service" when: - aiservice_channel is not defined or aiservice_channel == "" @@ -37,9 +46,9 @@ - name: "Set upgrade target explicitly" when: - aiservice_channel is defined and aiservice_channel != "" - - aiservice_channel in aiservice_upgrade_path + - current_aiservice_channel | ibm.mas_devops.is_channel_upgrade_path_valid(aiservice_channel, aiservice_upgrade_path) set_fact: - target_aiservice_channel: "{{ aiservice_upgrade_path[aiservice_channel] }}" + target_aiservice_channel: "{{ aiservice_channel }}" - name: "Assert upgrade target is defined" assert: @@ -139,7 +148,10 @@ - aiservice_channel is defined and aiservice_channel != "" - aiservice_sub_info is defined and aiservice_sub_info.resources[0].spec.channel != aiservice_channel - not aiservice_upgrade_dryrun - include_tasks: tasks/upgrade.yml + include_tasks: "{{ item }}" + loop: + - tasks/upgrade_aiservice.yml + - tasks/upgrade_aiservice_tenant.yml - name: "Debug when we are already on the desired channel" when: diff --git a/ibm/mas_devops/roles/aiservice_upgrade/tasks/tenant/copy_resource.yml b/ibm/mas_devops/roles/aiservice_upgrade/tasks/tenant/copy_resource.yml new file mode 100644 index 0000000000..221dac5f8e --- /dev/null +++ b/ibm/mas_devops/roles/aiservice_upgrade/tasks/tenant/copy_resource.yml @@ -0,0 +1,8 @@ + +- name: "Copy {{ resource_description }} to namespace {{ target_namespace }}" + kubernetes.core.k8s: + state: present + namespace: "{{ target_namespace }}" + definition: > + {{ resource | ibm.mas_devops.remove_dict_keys(['status', 'metadata.namespace', 'metadata.creationTimestamp', + 'metadata.generation', 'metadata.resourceVersion', 'metadata.uid']) | to_yaml }} diff --git a/ibm/mas_devops/roles/aiservice_upgrade/tasks/tenant/migrate.yml b/ibm/mas_devops/roles/aiservice_upgrade/tasks/tenant/migrate.yml new file mode 100644 index 0000000000..e34d4db1b1 --- /dev/null +++ b/ibm/mas_devops/roles/aiservice_upgrade/tasks/tenant/migrate.yml @@ -0,0 +1,79 @@ + +- name: "Create IBM Entitlement Key in namespace {{ tenant_cr.metadata.name }}" + ibm.mas_devops.update_ibm_entitlement: + namespace: "{{ tenant_cr.metadata.name }}" + icr_username: "{{ mas_entitlement_username }}" + icr_password: "{{ mas_entitlement_key }}" + artifactory_username: "{{ artifactory_username }}" + artifactory_password: "{{ artifactory_token }}" + namespace_kyverno_label: "audit" + +- name: "Create ibm-aiservice-tenant subscription in namespace {{ tenant_cr.metadata.name }}" + ibm.mas_devops.apply_subscription: + namespace: "{{ tenant_cr.metadata.name }}" + package_name: ibm-aiservice-tenant + package_channel: "{{ aiservice_channel }}" + catalog_source: "{{ mas_catalog_source }}" + register: subscription + +- name: Find SLS secret + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: "{{ tenant_cr.metadata.name }}----sls-secret" + namespace: "{{ aiservice_namespace }}" + register: sls_secret + +# TODO copy other tenant secrets when it is safe to do so, that is, when +# API broker and model code is updated. + +- name: Set target resources + ansible.builtin.set_fact: + sls_secret: "{{ sls_secret.resources[0] }}" + +- name: "Copy AIServiceTenant CR to namespace {{ tenant_cr.metadata.name }}" + include_tasks: tenant/copy_resource.yml + vars: + resource_description: AIServiceTenant CR + resource: "{{ tenant_cr }}" + target_namespace: "{{ tenant_cr.metadata.name }}" + +- name: "Copy SLS secret to namespace {{ tenant_cr.metadata.name }}" + include_tasks: tenant/copy_resource.yml + vars: + resource_description: SLS secret + resource: "{{ sls_secret }}" + target_namespace: "{{ tenant_cr.metadata.name }}" + +# Wait until the operator's special reconciliation configmap is available which +# tells us that the operator is now running +- name: "Wait until operator reconcilation configmap is available" + kubernetes.core.k8s_info: + api_version: v1 + name: aiservice-reconcile + namespace: "{{ tenant_cr.metadata.name }}" + kind: ConfigMap + register: configmap_result + retries: "{{ aiservice_install_num_retries }}" + delay: "{{ aiservice_install_wait_sec }}" + until: configmap_result.resources | length > 0 + +- name: "Confirm AIServiceTenant CR is ready" + kubernetes.core.k8s_info: + api_version: aiservice.ibm.com/v1 + name: "{{ tenant_cr.metadata.name }}" + namespace: "{{ tenant_cr.metadata.name }}" + kind: AIServiceTenant + retries: "{{ aiservice_install_num_retries }}" + delay: "{{ aiservice_install_wait_sec }}" + register: tenant_cr_info + until: + - tenant_cr_info.resources | json_query('[*].status.conditions[?type==`Ready`][].reason') | select ('match','Ready') | list | length == 1 + +- name: "Delete AIServiceTenant CR from namespace {{ aiservice_namespace }}" + kubernetes.core.k8s: + api_version: aiservice.ibm.com/v1 + kind: AIServiceTenant + state: absent + namespace: "{{ aiservice_namespace }}" + name: "{{ tenant_cr.metadata.name }}" \ No newline at end of file diff --git a/ibm/mas_devops/roles/aiservice_upgrade/tasks/tenant/upgrade.yml b/ibm/mas_devops/roles/aiservice_upgrade/tasks/tenant/upgrade.yml new file mode 100644 index 0000000000..90ae428700 --- /dev/null +++ b/ibm/mas_devops/roles/aiservice_upgrade/tasks/tenant/upgrade.yml @@ -0,0 +1,90 @@ +--- + +# Note: same channel is used for AI Service and AI Service Tenant operators +- name: "Upgrade ibm-aiservice-tenant subscription in namespace {{ tenant_subscription.metadata.namespace }}" + kubernetes.core.k8s: + api_version: operators.coreos.com/v1alpha1 + kind: Subscription + name: "{{ tenant_subscription.metadata.name }}" + namespace: "{{ tenant_subscription.metadata.namespace }}" + definition: + spec: + channel: "{{ aiservice_channel }}" + name: "{{ tenant_subscription.spec.name }}" + source: "{{ tenant_subscription.spec.source }}" + sourceNamespace: "{{ tenant_subscription.spec.sourceNamespace }}" + apply: true + +- name: "Get upgraded subscription for ibm-aiservice-tenant in namespace {{ tenant_subscription.metadata.namespace }}" + kubernetes.core.k8s_info: + api_version: operators.coreos.com/v1alpha1 + kind: Subscription + name: "{{ tenant_subscription.metadata.name }}" + namespace: "{{ tenant_subscription.metadata.namespace }}" + register: updated_aiservice_sub_info + retries: 20 # about 10 minutes + delay: 30 # seconds + until: + - updated_aiservice_sub_info.resources[0].status.installPlanGeneration > aiservice_sub_info.resources[0].status.installPlanGeneration + - updated_aiservice_sub_info.resources[0].status.state == "AtLatestKnown" + +- name: "upgrade : Debug Subscription" + debug: + var: updated_aiservice_sub_info + +# # No easy way to determine the end of the installPlanGeneration as it depends on if we have a patch versions of the +# # new version in the catalog. No patch versions means just one installPlanGeneration increase. Catalog has patches means +# # two installPlanGenerateion increase. Wait for 5 minutes like we do for apps +- name: "Pause for 5 minutes before checking upgrade status..." + pause: + minutes: 5 + +- name: "Lookup OperatorCondition for ibm-aiservice-tenant" + kubernetes.core.k8s_info: + api_version: operators.coreos.com/v2 + kind: OperatorCondition + namespace: "{{ tenant_subscription.metadata.namespace }}" + label_selectors: + - "operators.coreos.com/ibm-aiservice-tenant.{{ tenant_subscription.metadata.namespace }}" + register: updated_opcon + retries: 10 + delay: 60 # 1 minute + until: + - updated_opcon.resources is defined + - updated_opcon.resources | length == 1 + - updated_opcon.resources[0].metadata.name is defined + +- name: "Updated OperatorCondition info" + debug: + var: updated_opcon + +# OperatorCondition names are in the format {packageName}.{packageVersion} +# We want to strip off the "v" prefix from the version while we do this +- name: "upgrade : Lookup operator version for ibm-aiservice-tenant" + set_fact: + updated_opcon_version: "{{ updated_opcon.resources[0].metadata.name.split('.v')[1] | ibm.mas_devops.format_pre_version_with_plus }}" + +- name: "Operator upgrade info" + debug: + msg: + - "Operator condition ..................... {{ updated_opcon.resources[0].metadata.name }}" + - "Operator version (before) .............. {{ opcon_version }}" + - "Operator version (after) ............... {{ updated_opcon_version }}" + +- name: "Confirm AIServiceTenant CR is ready" + kubernetes.core.k8s_info: + api_version: aiservice.ibm.com/v1 + name: "{{ tenant_subscription.metadata.name }}" + namespace: "{{ tenant_subscription.metadata.namespace }}" + kind: AIServiceTenant + retries: 20 # about 40 minutes + delay: 120 # 2 minutes + until: + - updated_aiservice_info.resources[0].status.versions.reconciled == updated_opcon_version + - updated_aiservice_info.resources | json_query('[*].status.conditions[?type==`Ready`][].reason') | select ('match','Ready') | list | length == 1 + register: updated_aiservice_info + +- name: "Updated AIServiceTenant CR info" + debug: + var: updated_aiservice_info + diff --git a/ibm/mas_devops/roles/aiservice_upgrade/tasks/upgrade.yml b/ibm/mas_devops/roles/aiservice_upgrade/tasks/upgrade_aiservice.yml similarity index 76% rename from ibm/mas_devops/roles/aiservice_upgrade/tasks/upgrade.yml rename to ibm/mas_devops/roles/aiservice_upgrade/tasks/upgrade_aiservice.yml index 104d5b70fa..4b61be69d8 100644 --- a/ibm/mas_devops/roles/aiservice_upgrade/tasks/upgrade.yml +++ b/ibm/mas_devops/roles/aiservice_upgrade/tasks/upgrade_aiservice.yml @@ -73,16 +73,37 @@ set_fact: updated_opcon_version: "{{ updated_opcon.resources[0].metadata.name.split('.v')[1] | ibm.mas_devops.format_pre_version_with_plus }}" -- name: "upgrade : Debug Operator Version" - debug: - msg: - - "Operator condition ..................... {{ updated_opcon.resources[0].metadata.name }}" - - "Operator version (before) .............. {{ opcon_version }}" - - "Operator version (after) ............... {{ updated_opcon_version }}" - - # 5. Check that the AI Service CR meets the required state # ----------------------------------------------------------------------------- +- name: "upgrade : Read subscription" + kubernetes.core.k8s_info: + api_version: operators.coreos.com/v1alpha1 + kind: Subscription + namespace: "{{ aiservice_namespace }}" + label_selectors: + - "operators.coreos.com/ibm-aiservice.{{ aiservice_namespace }}" + register: subscription + +- name: "upgrade : Read AIServiceApp CR" + kubernetes.core.k8s_info: + api_version: aiservice.ibm.com/v1 + name: "{{ aiservice_instance_id }}" + namespace: "{{ aiservice_namespace }}" + kind: AIServiceApp + register: aiserviceapp_cr + +- name: "upgrade : Set version numbers" + set_fact: + sub_installed_version: "{{ subscription.resources[0].status.installedCSV.split('.v')[1] }}" + cr_reconciled_version: "{{ aiserviceapp_cr.resources[0].status.versions.reconciled }}" + +- name: Show current version info + ansible.builtin.debug: + msg: + - "Subscription installed CSV version ............. {{ sub_installed_version }}" + - "AIServiceApp reconciled version ............. {{ cr_reconciled_version }}" + - "OperatorCondition version ............. {{ updated_opcon_version }}" + - name: "upgrade : Get Suite CR for for ibm-aiservice" kubernetes.core.k8s_info: api_version: aiservice.ibm.com/v1 @@ -92,7 +113,7 @@ retries: 20 # about 40 minutes delay: 120 # 2 minutes until: - - updated_aiservice_info.resources[0].status.versions.reconciled == updated_opcon_version + - updated_aiservice_info.resources[0].status.versions.reconciled | ibm.mas_devops.is_operator_upgraded_by_version(updated_opcon_version, sub_installed_version) - updated_aiservice_info.resources | json_query('[*].status.conditions[?type==`Ready`][].reason') | select ('match','Ready') | list | length == 1 register: updated_aiservice_info diff --git a/ibm/mas_devops/roles/aiservice_upgrade/tasks/upgrade_aiservice_tenant.yml b/ibm/mas_devops/roles/aiservice_upgrade/tasks/upgrade_aiservice_tenant.yml new file mode 100644 index 0000000000..1731de2fd3 --- /dev/null +++ b/ibm/mas_devops/roles/aiservice_upgrade/tasks/upgrade_aiservice_tenant.yml @@ -0,0 +1,84 @@ +--- + +# Release 9.2.x introduces an architectural change whereby each tenant namespace +# has a dedicated "ibm-aiservice-tenant" operator. We must migrate the environment +# to support this change: +# (1) install the operator to each tenant namespace +# (2) move the tenant CR from the main namespace into the tenant namespace +- when: current_aiservice_channel == '9.1.x' or aiservice_force_migration + block: + - name: "Delete tenant operator deployment from namespace {{ aiservice_namespace }}" + kubernetes.core.k8s: + api_version: apps/v1 + kind: Deployment + namespace: "{{ aiservice_namespace }}" + name: "{{ aiservice_instance_id }}-entitymgr-tenant" + state: absent + + - name: "Delete tenant operator secret from namespace {{ aiservice_namespace }}" + kubernetes.core.k8s: + api_version: v1 + kind: Secret + namespace: "{{ aiservice_namespace }}" + name: provision-tenant----apikey-secret + state: absent + + - name: "Delete tenant operator configmap from namespace {{ aiservice_namespace }}" + kubernetes.core.k8s: + api_version: v1 + kind: ConfigMap + namespace: "{{ aiservice_namespace }}" + name: "{{ aiservice_instance_id }}-entitymgr-tenant-reconcile" + state: absent + + - name: "Find existing AIServiceTenant CRs in namespace {{ aiservice_namespace }}" + kubernetes.core.k8s_info: + api_version: aiservice.ibm.com/v1 + kind: AIServiceTenant + namespace: "{{ aiservice_namespace }}" + register: search_result + + - name: Set target resources + ansible.builtin.set_fact: + tenant_crs: "{{ search_result.resources }}" + + - name: Migration info + ansible.builtin.debug: + msg: + - "AI Service namespace ............. {{ aiservice_namespace }}" + - "Number of tenants to migrate ............. {{ tenant_crs | length }}" + + - name: Migrate tenant + include_tasks: tenant/migrate.yml + loop: "{{ tenant_crs }}" + loop_control: + loop_var: tenant_cr + + +# For environments not on 9.1.x we can safely assume the new tenant operator exists since +# upgrade is not supported below this release, therefore the environment is on at least 9.2.x +- when: current_aiservice_channel != '9.1.x' and not aiservice_force_migration + block: + - name: Find all subscriptions for ibm-aiservice-tenant + kubernetes.core.k8s_info: + api_version: operators.coreos.com/v1alpha1 + kind: Subscription + name: ibm-aiservice-tenant + label_selectors: + - "aiservice.ibm.com/instanceId = {{ aiservice_instance_id }}" + register: aiservice_tenant_sub_info + + - name: Set target resources + ansible.builtin.set_fact: + tenant_subscriptions: "{{ aiservice_tenant_sub_info.resources }}" + + - name: Subscription info + ansible.builtin.debug: + msg: + - "Number of tenant subscriptions ............. {{ tenant_subscriptions | length }}" + + - name: Upgrade tenant operator subscriptions + include_tasks: tenant/upgrade.yml + loop: "{{ tenant_subscriptions }}" + loop_control: + loop_var: tenant_subscription