diff --git a/src/container/srv_container.c b/src/container/srv_container.c index bf9ebdeddaa..9f194969fa8 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -1712,10 +1712,12 @@ cont_ec_agg_alloc(struct cont_svc *cont_svc, uuid_t cont_uuid, ec_agg->ea_servers_num = rank_nr; ec_agg->ea_current_eph = 0; ec_agg->ea_rdb_eph = 0; + ec_agg->ea_start_ts = daos_gettime_coarse(); + ec_agg->ea_warn_slug_ts = ec_agg->ea_start_ts; for (i = 0; i < rank_nr; i++) { ec_agg->ea_server_ephs[i].rank = doms[i].do_comp.co_rank; ec_agg->ea_server_ephs[i].eph = 0; - ec_agg->ea_server_ephs[i].ee_update_ts = daos_gettime_coarse(); + ec_agg->ea_server_ephs[i].ee_update_ts = ec_agg->ea_start_ts; } d_list_add(&ec_agg->ea_list, &cont_svc->cs_ec_agg_list); *ec_aggp = ec_agg; @@ -1776,6 +1778,22 @@ ds_cont_leader_update_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, if (ec_agg->ea_server_ephs[i].eph < eph) { ec_agg->ea_server_ephs[i].eph = eph; ec_agg->ea_server_ephs[i].ee_update_ts = daos_gettime_coarse(); + } else if (eph == 0 && ec_agg->ea_current_eph != 0) { + /* + * A rank reports zero only while its EC aggregation is paused + * waiting for the boundary (sc_ec_agg_eph_valid == 0), e.g. a + * reintegrated/extended rank that got the VOS container after the + * last refresh. Ask cont_agg_eph_sync() to re-send the current + * boundary even if the minimum epoch did not change, otherwise + * the rank never starts and pins the boundary forever. + * (ea_current_eph == 0 refreshes every round anyway.) + */ + if (!ec_agg->ea_need_refresh) + D_INFO(DF_CONT ": rank %u waits for EC agg boundary, " + "current " DF_X64 "\n", + DP_CONT(pool_uuid, cont_uuid), rank, + ec_agg->ea_current_eph); + ec_agg->ea_need_refresh = 1; } break; } @@ -1850,9 +1868,9 @@ ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, uuid_copy(arg.cont_uuid, cont_uuid); arg.min_eph = eph; - rc = ds_pool_task_collective(pool_uuid, PO_COMP_ST_NEW | PO_COMP_ST_DOWN | - PO_COMP_ST_DOWNOUT, cont_refresh_vos_agg_eph_one, - &arg, DSS_ULT_FL_PERIODIC); + rc = ds_pool_task_collective(pool_uuid, + PO_COMP_ST_NEW | PO_COMP_ST_DOWN | PO_COMP_ST_DOWNOUT, + cont_refresh_vos_agg_eph_one, &arg, DSS_ULT_FL_PERIODIC); if (rc) { DL_ERROR(rc, DF_CONT ": refresh ec_agg_eph " DF_X64 " failed.", DP_CONT(pool_uuid, cont_uuid), eph); @@ -2074,7 +2092,14 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) if (min_eph < ec_agg->ea_rdb_eph) min_eph = ec_agg->ea_rdb_eph; - if (min_eph == ec_agg->ea_current_eph && ec_agg->ea_current_eph != 0) + /* + * Skip the refresh when nothing changed, unless some rank asked for the + * boundary (see ds_cont_leader_update_agg_eph()). Such a rank has eph 0, + * so min_eph is pinned to ea_rdb_eph == ea_current_eph and it would never + * get the boundary it needs to start reporting. + */ + if (min_eph == ec_agg->ea_current_eph && ec_agg->ea_current_eph != 0 && + !ec_agg->ea_need_refresh) continue; /** @@ -2094,6 +2119,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) ec_agg->ea_warn_slug_ts = cur_ts; } else if (cur_eph && new_eph > cur_eph && (new_eph - cur_eph) >= 600 && + (cur_ts - ec_agg->ea_start_ts) >= (new_eph - cur_eph) && (cur_ts - ec_agg->ea_warn_slug_ts) >= 600) { ec_agg->ea_warn_slug_ts = cur_ts; D_WARN(DF_CONT ": Sluggish EC boundary reporting. " @@ -2120,6 +2146,10 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) } } + /* clear before the refresh (which yields), so a request arriving meanwhile + * triggers another refresh in the next round. + */ + ec_agg->ea_need_refresh = 0; rc = cont_iv_ec_agg_eph_refresh(pool->sp_iv_ns, ec_agg->ea_cont_uuid, min_eph, svc->cs_ec_leader_ephs_req); if (rc) { diff --git a/src/container/srv_internal.h b/src/container/srv_internal.h index d53205a4be1..bd27d232c0a 100644 --- a/src/container/srv_internal.h +++ b/src/container/srv_internal.h @@ -69,11 +69,20 @@ struct cont_ec_agg { uuid_t ea_cont_uuid; daos_epoch_t ea_current_eph; daos_epoch_t ea_rdb_eph; + /* time when this leader started tracking the container. A boundary jump larger + * than the tracking age cannot have been observed by this leader, it comes from + * the stale epoch loaded from rdb at leader step-up, so it is not a real stall. + */ + uint64_t ea_start_ts; uint64_t ea_warn_slug_ts; struct ec_eph *ea_server_ephs; d_list_t ea_list; int ea_servers_num; - uint32_t ea_deleted : 1; + uint32_t ea_deleted : 1, + /* some rank reported eph 0 (its EC aggregation waits for the boundary), re-send the + * boundary in the next cont_agg_eph_sync() even if the minimum epoch did not change. + */ + ea_need_refresh : 1; }; /*