diff --git a/src/container/srv_container.c b/src/container/srv_container.c index bf9ebdeddaa..aca2121457f 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -1712,10 +1712,12 @@ cont_ec_agg_alloc(struct cont_svc *cont_svc, uuid_t cont_uuid, ec_agg->ea_servers_num = rank_nr; ec_agg->ea_current_eph = 0; ec_agg->ea_rdb_eph = 0; + ec_agg->ea_start_ts = daos_gettime_coarse(); + ec_agg->ea_warn_slug_ts = ec_agg->ea_start_ts; for (i = 0; i < rank_nr; i++) { ec_agg->ea_server_ephs[i].rank = doms[i].do_comp.co_rank; ec_agg->ea_server_ephs[i].eph = 0; - ec_agg->ea_server_ephs[i].ee_update_ts = daos_gettime_coarse(); + ec_agg->ea_server_ephs[i].ee_update_ts = ec_agg->ea_start_ts; } d_list_add(&ec_agg->ea_list, &cont_svc->cs_ec_agg_list); *ec_aggp = ec_agg; @@ -1818,9 +1820,25 @@ cont_refresh_vos_agg_eph_one(void *data) int rc; rc = ds_cont_child_lookup(arg->pool_uuid, arg->cont_uuid, &cont_child); - if (rc) { - DL_CDEBUG(rc != 0 && rc != -DER_SHUTDOWN, DLOG_ERR, DB_MD, rc, - DF_CONT " lookup cont failed", DP_CONT(arg->pool_uuid, arg->cont_uuid)); + if (rc == -DER_NONEXIST) { + /* + * The VOS container can be absent on this target when the container was + * created while the target was excluded, and the following reintegration + * (or extend) did not migrate any record for it. Create it here, otherwise + * this target never reports EC aggregation epoch to the container service + * leader, and the leader keeps the EC aggregation boundary of the whole + * container pinned. The container service leader only refreshes containers + * that still exist in RDB, so it is safe to create the VOS container here. + */ + rc = ds_cont_child_open_create(arg->pool_uuid, arg->cont_uuid, &cont_child); + DL_CDEBUG(rc != 0 && rc != -DER_SHUTDOWN, DLOG_ERR, DLOG_INFO, rc, + DF_CONT " create missing vos container", + DP_CONT(arg->pool_uuid, arg->cont_uuid)); + if (rc != 0) + return rc; + } else if (rc != 0) { + DL_CDEBUG(rc != -DER_SHUTDOWN, DLOG_ERR, DB_MD, rc, DF_CONT " lookup cont failed", + DP_CONT(arg->pool_uuid, arg->cont_uuid)); return rc; } @@ -1850,9 +1868,13 @@ ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, uuid_copy(arg.cont_uuid, cont_uuid); arg.min_eph = eph; - rc = ds_pool_task_collective(pool_uuid, PO_COMP_ST_NEW | PO_COMP_ST_DOWN | - PO_COMP_ST_DOWNOUT, cont_refresh_vos_agg_eph_one, - &arg, DSS_ULT_FL_PERIODIC); + /* + * NB: cont_refresh_vos_agg_eph_one() may create the VOS container, which can yield + * and needs a deep stack, so a thread (ULT) collective must be used here. + */ + rc = ds_pool_thread_collective( + pool_uuid, PO_COMP_ST_NEW | PO_COMP_ST_DOWN | PO_COMP_ST_DOWNOUT, + cont_refresh_vos_agg_eph_one, &arg, DSS_ULT_DEEP_STACK | DSS_ULT_FL_PERIODIC); if (rc) { DL_ERROR(rc, DF_CONT ": refresh ec_agg_eph " DF_X64 " failed.", DP_CONT(pool_uuid, cont_uuid), eph); @@ -2094,6 +2116,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) ec_agg->ea_warn_slug_ts = cur_ts; } else if (cur_eph && new_eph > cur_eph && (new_eph - cur_eph) >= 600 && + (cur_ts - ec_agg->ea_start_ts) >= (new_eph - cur_eph) && (cur_ts - ec_agg->ea_warn_slug_ts) >= 600) { ec_agg->ea_warn_slug_ts = cur_ts; D_WARN(DF_CONT ": Sluggish EC boundary reporting. " diff --git a/src/container/srv_internal.h b/src/container/srv_internal.h index d53205a4be1..593fbc7d3fd 100644 --- a/src/container/srv_internal.h +++ b/src/container/srv_internal.h @@ -69,6 +69,11 @@ struct cont_ec_agg { uuid_t ea_cont_uuid; daos_epoch_t ea_current_eph; daos_epoch_t ea_rdb_eph; + /* time when this leader started tracking the container. A boundary jump larger + * than the tracking age cannot have been observed by this leader, it comes from + * the stale epoch loaded from rdb at leader step-up, so it is not a real stall. + */ + uint64_t ea_start_ts; uint64_t ea_warn_slug_ts; struct ec_eph *ea_server_ephs; d_list_t ea_list;