btl/uct: remove the need for a BTL module for connection TLs

Connection TLs are only used to form connections for connect-to-endpoint TLs.
They do not need to belong to the same memory domain as the one they are used
with so there is no need to rely on a BTL module. This commit moves the
pending_connection_reqs to the tl and changes the code to support a NULL
module for the connection tl.

Signed-off-by: Nathan Hjelm <hjelmn@google.com>

Author: hjelmn

opal/mca/btl/uct/btl_uct.h

@@ -118,9 +118,6 @@ struct mca_btl_uct_module_t {
 
     /** frags that were waiting on connections that are now ready to send */
     opal_list_t pending_frags;
-
-    /** pending connection requests */
-    opal_fifo_t pending_connection_reqs;
 };
 typedef struct mca_btl_uct_module_t mca_btl_uct_module_t;
 

opal/mca/btl/uct/btl_uct_component.c

@@ -387,7 +387,6 @@ static mca_btl_uct_module_t *mca_btl_uct_alloc_module(const char *md_name, mca_b
     OBJ_CONSTRUCT(&module->max_frags, opal_free_list_t);
     OBJ_CONSTRUCT(&module->pending_frags, opal_list_t);
     OBJ_CONSTRUCT(&module->lock, opal_recursive_mutex_t);
-    OBJ_CONSTRUCT(&module->pending_connection_reqs, opal_fifo_t);
 
     module->md = md;
     module->md_name = strdup(md_name);
@@ -798,19 +797,19 @@ static int mca_btl_uct_component_progress_pending(mca_btl_uct_module_t *uct_btl)
     return completed;
 }
 
-static int mca_btl_uct_component_progress_connections (mca_btl_uct_module_t *module) {
+static int mca_btl_uct_component_progress_connections (mca_btl_uct_tl_t *conn_tl) {
     mca_btl_uct_pending_connection_request_t *request;
     int ret;
 
-    if (module->conn_tl == NULL) {
+    if (conn_tl == NULL) {
         return 0;
     }
 
-    ret = mca_btl_uct_tl_progress(module->conn_tl, 0);
+    ret = mca_btl_uct_tl_progress(conn_tl, 0);
 
     while (NULL
            != (request = (mca_btl_uct_pending_connection_request_t *) opal_fifo_pop_atomic(
-                                                                                           &module->pending_connection_reqs))) {
+                                                                                           &conn_tl->pending_connection_reqs))) {
         mca_btl_uct_conn_req_t *conn_req = (mca_btl_uct_conn_req_t *) request->request_data;
         BTL_VERBOSE(("processing connection request...."));
         if (conn_req->module_index >= mca_btl_uct_component.module_count) {
@@ -819,7 +818,7 @@ static int mca_btl_uct_component_progress_connections (mca_btl_uct_module_t *mod
         }
         int rc = mca_btl_uct_process_connection_request(mca_btl_uct_component.modules[conn_req->module_index], conn_req);
         if (rc != OPAL_SUCCESS) {
-            opal_fifo_push_atomic(&module->pending_connection_reqs, &request->super);
+            opal_fifo_push_atomic(&conn_tl->pending_connection_reqs, &request->super);
             break;
         }
         OBJ_RELEASE(request);
@@ -849,15 +848,15 @@ static int mca_btl_uct_component_progress(void)
             ret += mca_btl_uct_tl_progress(module->am_tl, starting_index);
         }
 
-        mca_btl_uct_component_progress_connections (module);
+        mca_btl_uct_component_progress_connections (module->conn_tl);
 
         if (0 != opal_list_get_size(&module->pending_frags)) {
             mca_btl_uct_component_progress_pending(module);
         }
     }
 
     if (NULL != mca_btl_uct_component.conn_module) {
-        ret += mca_btl_uct_component_progress_connections (mca_btl_uct_component.conn_module);
+        ret += mca_btl_uct_component_progress_connections (mca_btl_uct_component.conn_module->conn_tl);
     }
 
     return (int) ret;

opal/mca/btl/uct/btl_uct_device_context.h

@@ -94,14 +94,14 @@ mca_btl_uct_module_get_tl_context_specific(mca_btl_uct_module_t *module, mca_btl
     mca_btl_uct_device_context_t *context = tl->uct_dev_contexts[context_id];
 
     if (OPAL_UNLIKELY(NULL == context)) {
-        OPAL_THREAD_LOCK(&module->lock);
+        OPAL_THREAD_LOCK(&tl->tl_lock);
         context = tl->uct_dev_contexts[context_id];
         if (OPAL_UNLIKELY(NULL == context)) {
             context = tl->uct_dev_contexts[context_id] = mca_btl_uct_context_create(module, tl,
                                                                                     context_id,
                                                                                     true);
         }
-        OPAL_THREAD_UNLOCK(&module->lock);
+        OPAL_THREAD_UNLOCK(&tl->tl_lock);
     }
 
     return context;

opal/mca/btl/uct/btl_uct_module.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2020-2025 Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -284,7 +284,6 @@ int mca_btl_uct_finalize(mca_btl_base_module_t *btl)
     OBJ_DESTRUCT(&uct_module->max_frags);
     OBJ_DESTRUCT(&uct_module->pending_frags);
     OBJ_DESTRUCT(&uct_module->lock);
-    OBJ_DESTRUCT(&uct_module->pending_connection_reqs);
 
     if (uct_module->rcache) {
         mca_rcache_base_module_destroy(uct_module->rcache);

opal/mca/btl/uct/btl_uct_tl.c

@@ -144,6 +144,7 @@ static void mca_btl_uct_tl_constructor(mca_btl_uct_tl_t *tl)
 {
     memset((void *) ((uintptr_t) tl + sizeof(tl->super)), 0, sizeof(*tl) - sizeof(tl->super));
     OBJ_CONSTRUCT(&tl->tl_lock, opal_mutex_t);
+    OBJ_CONSTRUCT(&tl->pending_connection_reqs, opal_fifo_t);
 }
 
 static void mca_btl_uct_tl_destructor(mca_btl_uct_tl_t *tl)
@@ -172,21 +173,22 @@ static void mca_btl_uct_tl_destructor(mca_btl_uct_tl_t *tl)
     }
 
     OBJ_DESTRUCT(&tl->tl_lock);
+    OBJ_DESTRUCT(&tl->pending_connection_reqs);
 }
 
 OBJ_CLASS_INSTANCE(mca_btl_uct_tl_t, opal_list_item_t, mca_btl_uct_tl_constructor,
                    mca_btl_uct_tl_destructor);
 
 static ucs_status_t mca_btl_uct_conn_req_cb(void *arg, void *data, size_t length, unsigned flags)
 {
-    mca_btl_uct_module_t *module = (mca_btl_uct_module_t *) arg;
+    mca_btl_uct_tl_t *tl = (mca_btl_uct_tl_t *) arg;
     mca_btl_uct_pending_connection_request_t *request = calloc(1, length + sizeof(request->super));
 
     /* it is not safe to process the connection request from the callback so just save it for
      * later processing */
     OBJ_CONSTRUCT(request, mca_btl_uct_pending_connection_request_t);
     memcpy(&request->request_data, (void *) ((intptr_t) data + 8), length);
-    opal_fifo_push_atomic(&module->pending_connection_reqs, &request->super);
+    opal_fifo_push_atomic(&tl->pending_connection_reqs, &request->super);
 
     return UCS_OK;
 }
@@ -241,20 +243,21 @@ int mca_btl_uct_process_connection_request(mca_btl_uct_module_t *module,
     return OPAL_SUCCESS;
 }
 
-static int mca_btl_uct_setup_connection_tl(mca_btl_uct_module_t *module)
+static int mca_btl_uct_setup_connection_tl(mca_btl_uct_tl_t *tl)
 {
     ucs_status_t ucs_status;
 
-    if (NULL == module->conn_tl) {
+    if (NULL == tl) {
         return OPAL_ERR_NOT_SUPPORTED;
     }
 
-    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_tl_context_specific(module, module->conn_tl,
-                                                                                       /*context_id=*/0);
+    mca_btl_uct_device_context_t *context =
+        mca_btl_uct_module_get_tl_context_specific(/*module=*/NULL, tl,
+                                                   /*context_id=*/0);
 
     ucs_status = uct_iface_set_am_handler(context->uct_iface,
-                                          MCA_BTL_UCT_CONNECT_RDMA, mca_btl_uct_conn_req_cb, module,
-                                          UCT_CB_FLAG_ASYNC);
+                                          MCA_BTL_UCT_CONNECT_RDMA, mca_btl_uct_conn_req_cb,
+                                          tl, UCT_CB_FLAG_ASYNC);
     if (UCS_OK != ucs_status) {
         BTL_ERROR(("could not set active message handler for uct tl"));
     }
@@ -383,7 +386,7 @@ mca_btl_uct_device_context_t *mca_btl_uct_context_create(mca_btl_uct_module_t *m
         return NULL;
     }
 
-    if (tl == module->am_tl) {
+    if (module != NULL && tl == module->am_tl) {
         BTL_VERBOSE(("installing AM handler for tl %p context id %d", (void *) tl, context_id));
         uct_iface_set_am_handler(context->uct_iface, MCA_BTL_UCT_FRAG, mca_btl_uct_am_handler,
                                  context, MCA_BTL_UCT_CB_FLAG_SYNC);
@@ -524,7 +527,7 @@ static int mca_btl_uct_set_tl_conn(mca_btl_uct_module_t *module, mca_btl_uct_tl_
     BTL_VERBOSE(("tl %s is suitable for making connections", tl->uct_tl_name));
 
     module->conn_tl = tl;
-    rc = mca_btl_uct_setup_connection_tl(module);
+    rc = mca_btl_uct_setup_connection_tl(tl);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }

opal/mca/btl/uct/btl_uct_types.h

@@ -341,6 +341,9 @@ struct mca_btl_uct_tl_t {
 
     /** async context */
     ucs_async_context_t *ucs_async;
+
+    /** pending connection requests */
+    opal_fifo_t pending_connection_reqs;
 };
 
 typedef struct mca_btl_uct_tl_t mca_btl_uct_tl_t;