[minor] support new AI Service tenant operator

.gitignore

@@ -1,4 +1,3 @@
-
 *.DS_Store*
 .env
 .venv
@@ -20,4 +19,4 @@ cpd-cli-workspace/*
 /tmp
 /node_modules
 package-lock.json
-package.json
+package.json

ibm/mas_devops/common_vars/compatibility_matrix.yml

@@ -116,4 +116,4 @@ upgrade_path:
   8.9.x: 8.10.x
 
 aiservice_upgrade_path:
-  9.1.x: 9.2.x-feature
+  9.1.x: [9.2.x-feature, 9.2.x-jhdev]

ibm/mas_devops/plugins/filter/filters.py

@@ -7,6 +7,7 @@
 # -----------------------------------------------------------
 import yaml
 import re
+import copy
 
 
 def private_vlan(vlans):
@@ -437,6 +438,77 @@ def get_ecr_repositories(image_mirror_output):
         repositories.append(repo_to_add)
   return repositories
 
+def is_channel_upgrade_path_valid(current: str, target: str, valid_paths: dict) -> bool:
+  """
+    Checks if a given current channel version can be upgraded to a target channel version.
+    :current: The current channel version.
+    :target: The target channel version to upgrade to.
+    :valid_paths: A dictionary of supported upgrade paths. See ibm/mas_devops/common_vars/compatibility_matrix.yml.
+    :return: True if the upgrade path is supported, False otherwise.
+  """
+  valid = False
+  if current not in valid_paths.keys():
+      print(f'Current channel {current} is not supported for upgrade')
+  else:
+      allowed_targets = valid_paths[current]
+      if isinstance(allowed_targets, str):
+          if target != allowed_targets:
+              print(f'Upgrading from channel {current} to {target} is not supported')
+          else:
+              valid = True
+      elif isinstance(allowed_targets, list):
+          if target not in allowed_targets:
+              print(f'Upgrading from channel {current} to {target} is not supported')
+          else:
+              valid = True
+      else:
+          print(f'Error: channel upgrade compatibility matrix is incorrectly defined')
+  return valid
+
+def remove_dict_keys(data: dict, keys: list[str], deep_copy: bool = True) -> dict:
+  """
+    Deletes keys from a dictionary. This has an advantage over Ansible's ansible.utils.remove_keys filter
+    in that nested keys are given explicitly in dot notation, for example 'a.b.c'.
+    :data: The input dictionary.
+    :keys: A list of key strings in dot notation, e.g. ['a.b', 'c.d.e'].
+    :deep_copy: Set to False to modify the input dictionary in-place, otherwise a copy will be modified.
+    :return: The dictionary with keys removed.
+  """
+  if deep_copy:
+    data = copy.deepcopy(data)
+  for key in keys:
+    parts = key.split('.')
+    ref = 'data'
+    for part in parts:
+      ref += f'["{part}"]'
+    try:
+      exec(f'del {ref}')
+    except KeyError as ex:
+      print(f'Could not delete key from dictionary: {ex}')
+  return data
+
+def is_operator_upgraded_by_version(cr_reconciled_version: str, opcon_version: str, sub_installed_version: str) -> bool:
+  """
+    Checks if an operator was upgraded successfully by comparing versions. Typically we just compare the version reported as
+    reconciled in the operator's custom resource with the OperatorCondition, however, this poses a problem for certain images
+    that are not tagged with a build number but the build number is used in other places within its bundle. For example, an
+    upgraded operator might have the CR reconciled version as "9.2.0-pre.1450" (derived from its image tag) but the version
+    from the OperatorCondition resource is "9.2.0-pre.1450-5075" which includes the build number. In this case where they
+    don't match we do our best effort by also checking the version reported as installed in the Subscription.
+    :cr_reconciled_version: Version number from the "versions.reconciled" field in the CR.
+    :opcon_version: Version derived from the operator's OperatorCondition.
+    :sub_installed_version: Version derived from the Subscription's "status.installedCSV" field.
+    :return: True if the operator was successfully upgraded (at least by checking various versions)
+  """
+  print(f'{cr_reconciled_version} --- {opcon_version} --- {sub_installed_version}')
+  upgraded = False
+  opcon_version = opcon_version.replace('+', '-')
+  prefix = f'{cr_reconciled_version}-'
+  if cr_reconciled_version == opcon_version:
+    upgraded = True
+  elif opcon_version.startswith(prefix) and sub_installed_version.startswith(prefix) and (opcon_version == sub_installed_version):
+      upgraded = True
+  return upgraded
 
 class FilterModule(object):
   def filters(self):
@@ -459,5 +531,8 @@ def filters(self):
       'format_pre_version_without_buildid': format_pre_version_without_buildid,
       'format_pre_version_with_buildid': format_pre_version_with_buildid,
       'get_db2_instance_name': get_db2_instance_name,
-      'get_ecr_repositories': get_ecr_repositories
+      'get_ecr_repositories': get_ecr_repositories,
+      'is_channel_upgrade_path_valid': is_channel_upgrade_path_valid,
+      'remove_dict_keys': remove_dict_keys,
+      'is_operator_upgraded_by_version': is_operator_upgraded_by_version,
     }

ibm/mas_devops/roles/aiservice/tasks/aiservice/main.yml

@@ -7,8 +7,6 @@
       - "Namespace ......................... {{ aiservice_namespace }}"
       - "Channel ........................... {{ aiservice_channel }}"
       - "MAS Instance Id ................... {{ aiservice_instance_id }}"
-      - "Channel ........................... {{ aiservice_channel }}"
-      - ""
       - "aiservice_s3_region ............... {{ aiservice_s3_region }}"
       - "aiservice_s3_host ................. {{ aiservice_s3_host }}"
       - "aiservice_s3_port ................. {{ aiservice_s3_port }}"

ibm/mas_devops/roles/aiservice_tenant/defaults/main.yml

@@ -1,7 +1,16 @@
 ---
 aiservice_instance_id: "{{ lookup('env', 'AISERVICE_INSTANCE_ID') }}"
 aiservice_namespace: "{{ lookup('env', 'AISERVICE_NAMESPACE') | default('aiservice-{}'.format(aiservice_instance_id), true) }}"
-aiservice_channel: "{{ lookup('env', 'AISERVICE_CHANNEL') }}"
+aiservice_channel: "{{ lookup('env', 'AISERVICE_TENANT_CHANNEL') }}"
+
+ibm_entitlement_username: "{{ lookup('env','IBM_ENTITLEMENT_USERNAME') }}"
+mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | default('cp', true) }}"
+ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}"
+mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}"
+
+# Development Registry Entitlement
+artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}"
+artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}"
 
 mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}"
 mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-operator-catalog', true) }}"
@@ -43,6 +52,7 @@ aiservice_watsonxai_full: "{{ lookup('env', 'AISERVICE_WATSONXAI_FULL') | defaul
 aiservice_watsonxai_instance_id: "{{ lookup('env', 'AISERVICE_WATSONXAI_INSTANCE_ID') | default('openshift', true) }}"
 aiservice_watsonxai_username: "{{ lookup('env', 'AISERVICE_WATSONXAI_USERNAME') | default('', true) }}"
 aiservice_watsonxai_version: "{{ lookup('env', 'AISERVICE_WATSONXAI_VERSION') | default('', true) }}"
+
 aiservice_watsonxai_verify: "{{ lookup('env', 'AISERVICE_WATSONXAI_VERIFY') | default('true', true) | bool }}"
 
 #  DRO

ibm/mas_devops/roles/aiservice_tenant/tasks/aiservice/main.yml

@@ -0,0 +1,63 @@
+---
+# 1. Provide Debug information
+# -----------------------------------------------------------------------------
+- name: "Debug information - IBM Maximo AI Service"
+  debug:
+    msg:
+      - "Namespace ......................... {{ tenantNamespace }}"
+      - "Channel ........................... {{ aiservice_channel }}"
+      - "MAS Instance Id ................... {{ aiservice_instance_id }}"
+
+
+
+# 2. Install the operator & create entitlement secret
+# -----------------------------------------------------------------------------
+- name: "Create IBM Entitlement Key"
+  ibm.mas_devops.update_ibm_entitlement:
+    namespace: "{{ tenantNamespace }}"
+    icr_username: "{{ mas_entitlement_username }}"
+    icr_password: "{{ mas_entitlement_key }}"
+    artifactory_username: "{{ artifactory_username }}"
+    artifactory_password: "{{ artifactory_token }}"
+    namespace_kyverno_label: "audit"
+
+- name: "Create ibm-aiservice Subscription"
+  ibm.mas_devops.apply_subscription:
+    namespace: "{{ tenantNamespace }}"
+    package_name: ibm-aiservice-tenant
+    package_channel: "{{ aiservice_channel }}"
+    catalog_source: "{{ mas_catalog_source }}"
+  register: subscription
+
+
+# 3. Wait until the IBM Maximo AI Service CRD is available
+# -----------------------------------------------------------------------------
+- name: "Wait until the IBM Maximo AI Service Operator CRD is available"
+  include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml"
+  vars:
+    crd_name: aiservicetenants.aiservice.ibm.com
+
+
+# 4. Lookup storage class availability
+# -----------------------------------------------------------------------------
+- name: "Load default storage class information"
+  include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml"
+
+
+# 5. Set AI Service Storage (Required)
+# -----------------------------------------------------------------------------
+- name: "Default AI Service Storage if not set by user"
+  when: aiservice_storage_class is not defined or aiservice_storage_class == ""
+  set_fact:
+    aiservice_storage_class: "{{ defaultStorageClasses.rwx }}"
+
+- name: "Assert that primary storage class has been defined"
+  assert:
+    that: aiservice_storage_class is defined and aiservice_storage_class != ""
+    fail_msg: "aiservice_storage_class must be defined"
+
+- name: "Debug"
+  debug:
+    msg: "AI Service storage class ................. {{ aiservice_storage_class }}"
+
+

ibm/mas_devops/roles/aiservice_tenant/tasks/config_dro/main.yml

@@ -47,5 +47,5 @@
 - name: Create DRO secret
   kubernetes.core.k8s:
     state: present
-    namespace: "{{ aiservice_namespace }}"
+    namespace: "{{ tenantNamespace }}"
     template: "templates/dro/dro-secret.yml.j2"

ibm/mas_devops/roles/aiservice_tenant/tasks/config_rsl/main.yml

@@ -16,6 +16,6 @@
 - name: Create RSL secret
   kubernetes.core.k8s:
     state: present
-    namespace: "{{ aiservice_namespace }}"
+    namespace: "{{ tenantNamespace }}"
     template: "templates/rsl/rsl-secret.yml.j2"
   when: rsl_config_valid

ibm/mas_devops/roles/aiservice_tenant/tasks/config_sls/main.yml

@@ -47,5 +47,5 @@
 - name: Create SLS secret
   kubernetes.core.k8s:
     state: present
-    namespace: "{{ aiservice_namespace }}"
+    namespace: "{{ tenantNamespace }}"
     template: "templates/sls/sls-secret.yml.j2"

ibm/mas_devops/roles/aiservice_tenant/tasks/main.yml

@@ -13,5 +13,8 @@
 # create wx secret
 - include_tasks: tasks/watsonx/main.yml
 
+# Install the operator
+- include_tasks: tasks/aiservice/main.yml
+
 # create AI Broker tenant
 - include_tasks: tasks/tenant/main.yml

ibm/mas_devops/roles/aiservice_tenant/tasks/tenant/install/main.yml

@@ -4,14 +4,14 @@
     annotation_dict: "{{ mas_annotations | string | ibm.mas_devops.getAnnotations() }}"
   kubernetes.core.k8s:
     state: present
-    namespace: "{{ aiservice_namespace }}"
+    namespace: "{{ tenantNamespace }}"
     template: templates/aiservice/aiservicetenant.yml.j2
 
 - name: "Wait for tenant CR to be ready"
   kubernetes.core.k8s_info:
     api_version: aiservice.ibm.com/v1
     name: "{{ tenantNamespace }}"
-    namespace: "{{ aiservice_namespace }}"
+    namespace: "{{ tenantNamespace }}"
     kind: AIServiceTenant
   register: aiservicetenant_cr_result
   until:

ibm/mas_devops/roles/aiservice_tenant/templates/aiservice/aiservicetenant.yml.j2

@@ -3,7 +3,7 @@ apiVersion: aiservice.ibm.com/v1
 kind: AIServiceTenant
 metadata:
     name: "{{ tenantNamespace }}"
-    namespace: "{{ aiservice_namespace }}"
+    namespace: "{{ tenantNamespace }}"
     annotations:
         ansible.sdk.operatorframework.io/verbosity: "{{ aiservice_operator_log_level }}"
     labels:

ibm/mas_devops/roles/aiservice_tenant/templates/dro/dro-secret.yml.j2

@@ -4,7 +4,7 @@ apiVersion: v1
 type: Opaque
 metadata:
   name: {{ aiservice_dro_token_secret }}
-  namespace: {{ aiservice_namespace }}
+  namespace: {{ tenantNamespace }}
   labels:
     aiservice.ibm.com/instanceId: "{{ aiservice_instance_id }}"
 {% if custom_labels is defined and custom_labels.items() %}

ibm/mas_devops/roles/aiservice_tenant/templates/sls/sls-secret.yml.j2

@@ -4,7 +4,7 @@ apiVersion: v1
 type: Opaque
 metadata:
   name: {{ aiservice_sls_secret }}
-  namespace: {{ aiservice_namespace }}
+  namespace: {{ tenantNamespace }}
   labels:
     aiservice.ibm.com/instanceId: "{{ aiservice_instance_id }}"
 {% if custom_labels is defined and custom_labels.items() %}

ibm/mas_devops/roles/aiservice_upgrade/defaults/main.yml

@@ -1,5 +1,15 @@
 ---
+mas_catalog_source: "{{ lookup('env', 'MAS_CATALOG_SOURCE') | default('ibm-operator-catalog', true) }}"
+mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | default('cp', true) }}"
+mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}"
+
+artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') | lower }}"
+artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}"
+
 aiservice_upgrade_dryrun: "{{ lookup('env', 'AISERVICE_UPGRADE_DRYRUN') | default('False', True) | bool }}"
 aiservice_channel: "{{ lookup('env', 'AISERVICE_CHANNEL') }}"
 aiservice_instance_id: "{{ lookup('env', 'AISERVICE_INSTANCE_ID') }}"
 aiservice_namespace: "aiservice-{{ aiservice_instance_id }}"
+aiservice_force_migration: "{{ lookup('env', 'AISERVICE_FORCE_MIGRATION') | default('False', True) | bool }}"
+aiservice_install_num_retries: "{{ lookup('env', 'AISERVICE_INSTALL_NUM_RETRIES') | default('60', True) | int }}"
+aiservice_install_wait_sec: "{{ lookup('env', 'AISERVICE_INSTALL_WAIT_SEC') | default('60', True) | int }}"

ibm/mas_devops/roles/aiservice_upgrade/tasks/main.yml

@@ -27,6 +27,15 @@
       - "operators.coreos.com/ibm-aiservice.{{ aiservice_namespace }}"
   register: aiservice_sub_info
 
+- name: "Set current subscription channel"
+  set_fact:
+    current_aiservice_channel: "{{ aiservice_sub_info.resources[0].spec.channel }}"
+
+- name: Show current subscription info
+  ansible.builtin.debug:
+    msg:
+      - "Currently installed AI Service channel ............. {{ current_aiservice_channel }}"
+
 - name: "Set default upgrade target based on installed version of AI Service"
   when:
     - aiservice_channel is not defined or aiservice_channel == ""
@@ -37,9 +46,9 @@
 - name: "Set upgrade target explicitly"
   when:
     - aiservice_channel is defined and aiservice_channel != ""
-    - aiservice_channel in aiservice_upgrade_path
+    - current_aiservice_channel | ibm.mas_devops.is_channel_upgrade_path_valid(aiservice_channel, aiservice_upgrade_path)
   set_fact:
-    target_aiservice_channel: "{{ aiservice_upgrade_path[aiservice_channel] }}"
+    target_aiservice_channel: "{{ aiservice_channel }}"
 
 - name: "Assert upgrade target is defined"
   assert:
@@ -139,7 +148,10 @@
     - aiservice_channel is defined and aiservice_channel != ""
     - aiservice_sub_info is defined and aiservice_sub_info.resources[0].spec.channel != aiservice_channel
     - not aiservice_upgrade_dryrun
-  include_tasks: tasks/upgrade.yml
+  include_tasks: "{{ item  }}"
+  loop:
+    - tasks/upgrade_aiservice.yml
+    - tasks/upgrade_aiservice_tenant.yml
 
 - name: "Debug when we are already on the desired channel"
   when:

ibm/mas_devops/roles/aiservice_upgrade/tasks/tenant/copy_resource.yml

@@ -0,0 +1,8 @@
+
+- name: "Copy {{ resource_description }} to namespace {{ target_namespace }}"
+  kubernetes.core.k8s:
+    state: present
+    namespace: "{{ target_namespace }}"
+    definition: >
+      {{ resource | ibm.mas_devops.remove_dict_keys(['status', 'metadata.namespace', 'metadata.creationTimestamp',
+      'metadata.generation', 'metadata.resourceVersion', 'metadata.uid']) | to_yaml }}