btl/uct: complete rework of descovery and initialization code

There is an issue with btl/uct which prevents the usage of the standard
btl_uct_ MCA variables (eager limit, flags, etc). Because of the way the btl was
written these values are all determined directly from UCT and can not be changed
using the MCA variable interface. To address this issue this commit breaks apart
the initialization code and separates out the pieces that are necessary for
discovery only. The discovery pieces now use a new set of variables that include
the memory domain name and directly control the behavior for BTLs on that
memory domain as well as enabling the usage of the btl_uct variable to control
the defaults for these variables.

Example, using memory domain irdma0 will create variables:
btl_uct_irdma0_eager_limit, btl_uct_irdma0_max_send_size, etc.

The defaults will be based on what is reported by UCT and the user can set the
values to a subset of what UCT reports. For example, if the max send size for
the hardware is 8192B then it can be set to anything up to and including that
value. The same is true for feature flags, if the hardware supports only some
btl atomics or operations the user can specify a subset of them (others will
be ignored).

Signed-off-by: Nathan Hjelm <hjelmn@google.com>

Author: hjelmn

opal/mca/btl/uct/Makefile.am

@@ -13,6 +13,7 @@
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
 # Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights
 #                         reserved.
+# Copyright (c) 2025      Google, LLC. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -24,22 +25,30 @@ AM_CPPFLAGS = $(btl_uct_CPPFLAGS)
 
 amca_paramdir = $(AMCA_PARAM_SETS_DIR)
 
-sources = \
+headers = \
     btl_uct.h \
+    btl_uct_rdma.h \
+    btl_uct_endpoint.h \
+    btl_uct_am.h \
+    btl_uct_frag.h \
+    btl_uct_types.h \
+    btl_uct_device_context.h \
+    btl_uct_discover.h \
+    btl_uct_modex.h \
+    btl_uct_include_list.h
+
+sources = \
     btl_uct_module.c \
     btl_uct_component.c \
-    btl_uct_rdma.h \
     btl_uct_rdma.c \
-    btl_uct_endpoint.h \
     btl_uct_endpoint.c \
     btl_uct_amo.c \
-    btl_uct_am.h \
     btl_uct_am.c \
-    btl_uct_frag.h \
     btl_uct_frag.c \
     btl_uct_tl.c \
-    btl_uct_types.h \
-    btl_uct_device_context.h
+    btl_uct_discover.c \
+    btl_uct_modex.c \
+    btl_uct_include_list.c
 
 # Make the output library in this directory, and name it either
 # mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
@@ -50,20 +59,22 @@ lib =
 lib_sources =
 component = mca_btl_uct.la
 component_sources = $(sources)
+component_headers = $(headers)
 else
 lib = libmca_btl_uct.la
 lib_sources = $(sources)
+lib_headers = ${headers}
 component =
 component_sources =
 endif
 
 mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component)
-mca_btl_uct_la_SOURCES = $(component_sources)
+mca_btl_uct_la_SOURCES = $(component_sources) $(component_headers)
 mca_btl_uct_la_LDFLAGS = -module -avoid-version $(btl_uct_LDFLAGS)
 mca_btl_uct_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la $(btl_uct_LIBS)
 
 noinst_LTLIBRARIES = $(lib)
-libmca_btl_uct_la_SOURCES = $(lib_sources)
+libmca_btl_uct_la_SOURCES = $(lib_sources) $(lib_headers)
 libmca_btl_uct_la_LDFLAGS = -module -avoid-version $(btl_uct_LDFLAGS)
 libmca_btl_uct_la_LIBADD = $(btl_uct_LIBS)

opal/mca/btl/uct/btl_uct.h

@@ -85,12 +85,6 @@ struct mca_btl_uct_module_t {
     /** transport for RDMA/AMOs */
     mca_btl_uct_tl_t *rdma_tl;
 
-    /** transport for forming connections (if needed) */
-    mca_btl_uct_tl_t *conn_tl;
-
-    /** array containing the am_tl and rdma_tl */
-    mca_btl_uct_tl_t *comm_tls[2];
-
     /** registration cache */
     mca_rcache_base_module_t *rcache;
 
@@ -111,6 +105,10 @@ struct mca_btl_uct_module_t {
 
     /** frags that were waiting on connections that are now ready to send */
     opal_list_t pending_frags;
+
+    /** allowed transports */
+    char *allowed_transports;
+    mca_btl_uct_include_list_t allowed_transport_list;
 };
 typedef struct mca_btl_uct_module_t mca_btl_uct_module_t;
 
@@ -123,6 +121,9 @@ struct mca_btl_uct_component_t {
     /** base BTL component */
     mca_btl_base_component_3_0_0_t super;
 
+    /** whether the component is initialized. controls cleanup. */
+    bool initialized;
+
     /** number of TL modules */
     int module_count;
 
@@ -135,7 +136,6 @@ struct mca_btl_uct_component_t {
 
     /** allowed transports */
     char *allowed_transports;
-    mca_btl_uct_include_list_t allowed_transport_list;
 
     /** transports to consider for forming connections */
     char *connection_domains;
@@ -155,14 +155,16 @@ struct mca_btl_uct_component_t {
     /** connection retry timeout */
     unsigned int connection_retry_timeout;
 
-    /** alternate connection-only module that can be used if no suitable
-     * connection tl is found. this is usually a tcp tl. */
-    mca_btl_uct_module_t *conn_module;
-
 #if UCT_API >= UCT_VERSION(1, 7)
     uct_component_h *uct_components;
     unsigned num_uct_components;
 #endif
+
+    /** list of memory domains (btl_uct_md_t) */
+    opal_list_t md_list;
+
+    /** connection transport (if needed). reference is owned by conn_md */
+    mca_btl_uct_tl_t *conn_tl;
 };
 typedef struct mca_btl_uct_component_t mca_btl_uct_component_t;
 
@@ -298,12 +300,16 @@ ucs_status_t mca_btl_uct_am_handler(void *arg, void *data, size_t length, unsign
 struct mca_btl_base_endpoint_t *mca_btl_uct_get_ep(struct mca_btl_base_module_t *module,
                                                    opal_proc_t *proc);
 
-int mca_btl_uct_query_tls(mca_btl_uct_module_t *module, mca_btl_uct_md_t *md,
-                          uct_tl_resource_desc_t *tl_descs, unsigned tl_count,
-                          bool evaluate_for_conn_only);
+int mca_btl_uct_populate_tls(mca_btl_uct_md_t *md, uct_tl_resource_desc_t *tl_descs, unsigned tl_count);
 int mca_btl_uct_process_connection_request(mca_btl_uct_module_t *module,
                                            mca_btl_uct_conn_req_t *req);
 
+mca_btl_uct_module_t *mca_btl_uct_alloc_module(mca_btl_uct_md_t *md,
+                                               size_t registration_size);
+
+int mca_btl_uct_evaluate_tl(mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl);
+int mca_btl_uct_enable_tl_conn(mca_btl_uct_tl_t *tl);
+
 /**
  * @brief Checks if a tl is suitable for using for RDMA
  *
@@ -344,18 +350,12 @@ static inline bool mca_btl_uct_tl_supports_conn(mca_btl_uct_tl_t *tl)
  */
 static inline bool mca_btl_uct_tl_requires_connection_tl(mca_btl_uct_tl_t *tl)
 {
+    if (NULL == tl) {
+        return false;
+    }
+
     return !(tl->uct_iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE);
 }
 
-/**
- * @brief Find the rank of `name` in the include list `list`.
- *
- * @param[in] name   name to find
- * @param[in] list   list to search
- *
- * A negative result means the name is not present or the list is negated.
- */
-int mca_btl_uct_include_list_rank (const char *name, const mca_btl_uct_include_list_t *list);
-
 END_C_DECLS
 #endif

opal/mca/btl/uct/btl_uct_am.c

@@ -233,7 +233,7 @@ int mca_btl_uct_send_frag(mca_btl_uct_module_t *uct_btl, mca_btl_uct_base_frag_t
     }
 
     OPAL_THREAD_LOCK(&uct_btl->lock);
-    mca_btl_uct_append_pending_frag(uct_btl, frag, context, true);
+    mca_btl_uct_append_pending_frag(uct_btl, frag, context, /*ready=*/true);
     OPAL_THREAD_UNLOCK(&uct_btl->lock);
 
     return OPAL_SUCCESS;
@@ -260,14 +260,14 @@ int mca_btl_uct_send(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoi
         OPAL_THREAD_LOCK(&uct_btl->lock);
         /* check one more time in case another thread is completing the connection now */
         if (OPAL_SUCCESS != mca_btl_uct_endpoint_test_am(uct_btl, endpoint, context, &ep_handle)) {
-            mca_btl_uct_append_pending_frag(uct_btl, frag, context, false);
+            mca_btl_uct_append_pending_frag(uct_btl, frag, context, /*ready=*/false);
             OPAL_THREAD_UNLOCK(&uct_btl->lock);
             return OPAL_SUCCESS;
         }
         OPAL_THREAD_UNLOCK(&uct_btl->lock);
     }
 
-    return mca_btl_uct_send_frag(uct_btl, frag, true);
+    return mca_btl_uct_send_frag(uct_btl, frag, /*append=*/true);
 }
 
 struct mca_btl_uct_sendi_pack_args_t {