diff --git a/src/container/container_iv.c b/src/container/container_iv.c index b61bd28e59a..aea055c0a27 100644 --- a/src/container/container_iv.c +++ b/src/container/container_iv.c @@ -1,6 +1,6 @@ /** * (C) Copyright 2019-2023 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -194,6 +194,10 @@ cont_iv_ent_copy(struct ds_iv_entry *entry, struct cont_iv_key *key, break; case IV_CONT_AGG_EPOCH_BOUNDRY: dst->iv_agg_eph.eph = src->iv_agg_eph.eph; + /* Fetches are not tracked by ea_refreshing: do not propagate the push's + * creation marker, or a late fetch could recreate a destroyed container. + */ + dst->iv_agg_eph.rank = 0; break; default: rc = -DER_INVAL; @@ -571,7 +575,7 @@ cont_iv_ent_fetch(struct ds_iv_entry *entry, struct ds_iv_key *key, */ rc = ds_cont_tgt_refresh_agg_eph(entry->ns->iv_pool_uuid, civ_key->cont_uuid, - ec_agg_eph); + ec_agg_eph, false); if (rc == 0) goto again; } @@ -622,9 +626,9 @@ cont_iv_ent_agg_eph_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key, struct cont_iv_key *civ_key = key2priv(key); int rc; - rc = ds_cont_tgt_refresh_agg_eph(entry->ns->iv_pool_uuid, - civ_key->cont_uuid, - civ_ent->iv_agg_eph.eph); + rc = ds_cont_tgt_refresh_agg_eph(entry->ns->iv_pool_uuid, civ_key->cont_uuid, + civ_ent->iv_agg_eph.eph, + civ_ent->iv_agg_eph.rank == CRT_NO_RANK); return rc; } @@ -1144,6 +1148,8 @@ cont_iv_ec_agg_eph_update_internal(void *ns, uuid_t cont_uuid, DP_UUID(cont_uuid), op, DP_RC(rc)); return rc; } + if (op == IV_CONT_AGG_EPOCH_BOUNDRY) + iv_entry.iv_agg_eph.rank = CRT_NO_RANK; rc = cont_iv_update(ns, op, cont_uuid, &iv_entry, sizeof(iv_entry), shortcut, sync_mode, false /* retry */); diff --git a/src/container/srv_container.c b/src/container/srv_container.c index bf9ebdeddaa..1909f05e2b1 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -1541,7 +1541,7 @@ evict_hdls(struct rdb_tx *tx, struct cont *cont, bool force, struct ds_pool_hdl return rc; } -static void +static int cont_ec_agg_delete(struct cont_svc *svc, uuid_t cont_uuid); static int @@ -1591,28 +1591,30 @@ cont_destroy(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *cont, if (rc != 0) goto out_prop; + rc = cont_ec_agg_delete(cont->c_svc, cont->c_uuid); + if (rc != 0) + goto out_prop; + rc = cont_destroy_bcast(rpc->cr_ctx, cont->c_svc, cont->c_uuid); if (rc != 0) goto out_prop; - cont_ec_agg_delete(cont->c_svc, cont->c_uuid); - - if (pool_hdl->sph_global_ver >= DAOS_POOL_GLOBAL_VERSION_WITH_OIT_OID_KVS) { - need_destroy_oid_oit_kvs = true; - } else { - d_iov_t value; - - d_iov_set(&value, NULL, 0); - rc = rdb_tx_lookup(tx, &cont->c_prop, &ds_cont_prop_oit_oids, &value); - if (rc && rc != -DER_NONEXIST) { - DL_ERROR(rc, "failed to lookup oit oid kvs pool/cont: " DF_CONTF, - DP_CONT(pool_hdl->sph_pool->sp_uuid, cont->c_uuid)); - goto out_prop; - } - /* There was a bug that oit oids might be created already see DAOS-14799 */ + if (pool_hdl->sph_global_ver >= DAOS_POOL_GLOBAL_VERSION_WITH_OIT_OID_KVS) { + need_destroy_oid_oit_kvs = true; + } else { + d_iov_t value; + + d_iov_set(&value, NULL, 0); + rc = rdb_tx_lookup(tx, &cont->c_prop, &ds_cont_prop_oit_oids, &value); + if (rc && rc != -DER_NONEXIST) { + DL_ERROR(rc, "failed to lookup oit oid kvs pool/cont: " DF_CONTF, + DP_CONT(pool_hdl->sph_pool->sp_uuid, cont->c_uuid)); + goto out_prop; + } + /* There was a bug that oit oids might be created already see DAOS-14799 */ if (rc == 0) need_destroy_oid_oit_kvs = true; - } + } /* Destroy oit oids index KVS. */ if (need_destroy_oid_oit_kvs) { @@ -1712,10 +1714,12 @@ cont_ec_agg_alloc(struct cont_svc *cont_svc, uuid_t cont_uuid, ec_agg->ea_servers_num = rank_nr; ec_agg->ea_current_eph = 0; ec_agg->ea_rdb_eph = 0; + ec_agg->ea_start_ts = daos_gettime_coarse(); + ec_agg->ea_warn_slug_ts = ec_agg->ea_start_ts; for (i = 0; i < rank_nr; i++) { ec_agg->ea_server_ephs[i].rank = doms[i].do_comp.co_rank; ec_agg->ea_server_ephs[i].eph = 0; - ec_agg->ea_server_ephs[i].ee_update_ts = daos_gettime_coarse(); + ec_agg->ea_server_ephs[i].ee_update_ts = ec_agg->ea_start_ts; } d_list_add(&ec_agg->ea_list, &cont_svc->cs_ec_agg_list); *ec_aggp = ec_agg; @@ -1729,19 +1733,35 @@ cont_ec_agg_alloc(struct cont_svc *cont_svc, uuid_t cont_uuid, return rc; } -static void +static int cont_ec_agg_delete(struct cont_svc *svc, uuid_t cont_uuid) { - struct cont_ec_agg *ec_agg; - - ec_agg = cont_ec_agg_lookup(svc, cont_uuid); - if (ec_agg == NULL) - return; + struct cont_ec_agg *ec_agg; + int retry = 0; - /* Set ea_deleted flag to destroy it inside cont_agg_eph_leader_ult() - * to avoid list iteration broken. + /* A boundary push may create VOS containers. Do not destroy until it completes, + * but bound the wait (cs_lock may held) in case the push keeps retrying on an + * unreachable rank. */ - ec_agg->ea_deleted = 1; +again: + d_list_for_each_entry(ec_agg, &svc->cs_ec_agg_list, ea_list) { + if (uuid_compare(ec_agg->ea_cont_uuid, cont_uuid) == 0 && ec_agg->ea_refreshing) { + if (retry++ < 30) { + dss_sleep(300 /* ms, 10s in total */); + goto again; + } + D_DEBUG(DB_MD, DF_CONT ": EC boundary refresh in flight\n", + DP_CONT(svc->cs_pool_uuid, cont_uuid)); + return -DER_BUSY; + } + } + + /* Defer freeing to cont_agg_eph_sync() to preserve its list iteration. */ + d_list_for_each_entry(ec_agg, &svc->cs_ec_agg_list, ea_list) { + if (uuid_compare(ec_agg->ea_cont_uuid, cont_uuid) == 0) + ec_agg->ea_deleted = 1; + } + return 0; } /** @@ -1776,6 +1796,9 @@ ds_cont_leader_update_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, if (ec_agg->ea_server_ephs[i].eph < eph) { ec_agg->ea_server_ephs[i].eph = eph; ec_agg->ea_server_ephs[i].ee_update_ts = daos_gettime_coarse(); + } else if (eph == 0 && ec_agg->ea_current_eph != 0) { + /* A recovered target of this rank waits for the boundary. */ + ec_agg->ea_need_refresh = 1; } break; } @@ -1808,6 +1831,7 @@ struct refresh_vos_agg_eph_arg { uuid_t pool_uuid; uuid_t cont_uuid; daos_epoch_t min_eph; + bool create; }; int @@ -1818,9 +1842,19 @@ cont_refresh_vos_agg_eph_one(void *data) int rc; rc = ds_cont_child_lookup(arg->pool_uuid, arg->cont_uuid, &cont_child); - if (rc) { - DL_CDEBUG(rc != 0 && rc != -DER_SHUTDOWN, DLOG_ERR, DB_MD, rc, - DF_CONT " lookup cont failed", DP_CONT(arg->pool_uuid, arg->cont_uuid)); + if (rc == -DER_NONEXIST && arg->create) { + /* Reintegration may not migrate any records for this container. Only a + * leader push, serialized with destroy, may create the missing container. + */ + rc = ds_cont_child_open_create(arg->pool_uuid, arg->cont_uuid, &cont_child); + DL_CDEBUG(rc != 0 && rc != -DER_SHUTDOWN, DLOG_ERR, DLOG_INFO, rc, + DF_CONT " create missing vos container", + DP_CONT(arg->pool_uuid, arg->cont_uuid)); + if (rc != 0) + return rc; + } else if (rc != 0) { + DL_CDEBUG(rc != -DER_SHUTDOWN, DLOG_ERR, DB_MD, rc, DF_CONT " lookup cont failed", + DP_CONT(arg->pool_uuid, arg->cont_uuid)); return rc; } @@ -1838,8 +1872,7 @@ cont_refresh_vos_agg_eph_one(void *data) } int -ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, - daos_epoch_t eph) +ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, daos_epoch_t eph, bool create) { struct refresh_vos_agg_eph_arg arg; static uint64_t cnt; @@ -1849,10 +1882,12 @@ ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, uuid_copy(arg.pool_uuid, pool_uuid); uuid_copy(arg.cont_uuid, cont_uuid); arg.min_eph = eph; + arg.create = create; - rc = ds_pool_task_collective(pool_uuid, PO_COMP_ST_NEW | PO_COMP_ST_DOWN | - PO_COMP_ST_DOWNOUT, cont_refresh_vos_agg_eph_one, - &arg, DSS_ULT_FL_PERIODIC); + /* VOS container creation may yield and needs a deep stack. */ + rc = ds_pool_thread_collective( + pool_uuid, PO_COMP_ST_NEW | PO_COMP_ST_DOWN | PO_COMP_ST_DOWNOUT, + cont_refresh_vos_agg_eph_one, &arg, DSS_ULT_DEEP_STACK | DSS_ULT_FL_PERIODIC); if (rc) { DL_ERROR(rc, DF_CONT ": refresh ec_agg_eph " DF_X64 " failed.", DP_CONT(pool_uuid, cont_uuid), eph); @@ -1996,6 +2031,7 @@ cont_agg_eph_store(struct cont_svc *svc, uuid_t cont_uuid, uint64_t ec_agg_eph, return rc; } +#define EC_AGG_EPH_INTV (10ULL * 1000) /* seconds interval to check*/ static void cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) { @@ -2030,12 +2066,15 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) rc = cont_agg_eph_load(svc, ec_agg->ea_cont_uuid, &ec_agg->ea_rdb_eph); if (rc) { if (rc == -DER_NONEXIST) { + /* destroyed, free the entry in the next round */ + ec_agg->ea_deleted = 1; DL_INFO(rc, DF_CONT " container skipped", DP_CONT(svc->cs_pool_uuid, ec_agg->ea_cont_uuid)); continue; } DL_ERROR(rc, DF_CONT ": cont_agg_eph_load failed.", DP_CONT(svc->cs_pool_uuid, ec_agg->ea_cont_uuid)); + continue; } } @@ -2070,11 +2109,21 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) min_eph = ec_agg->ea_server_ephs[i].eph; } + /* some rank reported nothing, it may lack the VOS container */ + if (min_eph == 0) + ec_agg->ea_need_refresh = 1; + /* for reboot case the ea_rdb_eph possibly higher than min_eph */ if (min_eph < ec_agg->ea_rdb_eph) min_eph = ec_agg->ea_rdb_eph; - if (min_eph == ec_agg->ea_current_eph && ec_agg->ea_current_eph != 0) + /* A recovered target may report nothing, or its zero epoch may be suppressed + * by the rank's last report. Periodically resend even an unchanged boundary + * while some rank waits for it, once rebuild is done and reclaim is enabled. + */ + if (min_eph == ec_agg->ea_current_eph && ec_agg->ea_current_eph != 0 && + (!warn_sluggish || !ec_agg->ea_need_refresh || + cur_ts - ec_agg->ea_refresh_ts < EC_AGG_EPH_INTV * 3 / 1000)) continue; /** @@ -2094,6 +2143,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) ec_agg->ea_warn_slug_ts = cur_ts; } else if (cur_eph && new_eph > cur_eph && (new_eph - cur_eph) >= 600 && + (cur_ts - ec_agg->ea_start_ts) >= (new_eph - cur_eph) && (cur_ts - ec_agg->ea_warn_slug_ts) >= 600) { ec_agg->ea_warn_slug_ts = cur_ts; D_WARN(DF_CONT ": Sluggish EC boundary reporting. " @@ -2110,6 +2160,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) &ec_agg->ea_rdb_eph); if (rc) { if (rc == -DER_NONEXIST) { + ec_agg->ea_deleted = 1; DL_INFO(rc, DF_CONT " container skipped", DP_CONT(svc->cs_pool_uuid, ec_agg->ea_cont_uuid)); continue; @@ -2117,11 +2168,19 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) DL_ERROR(rc, DF_CONT ": rdb_tx_update ec_agg_eph " DF_X64 " failed.", DP_CONT(svc->cs_pool_uuid, ec_agg->ea_cont_uuid), min_eph); + continue; } } + if (ec_agg->ea_deleted) + continue; + + /* clear before the push (which yields), a request arriving meanwhile is kept */ + ec_agg->ea_need_refresh = 0; + ec_agg->ea_refreshing = 1; rc = cont_iv_ec_agg_eph_refresh(pool->sp_iv_ns, ec_agg->ea_cont_uuid, min_eph, svc->cs_ec_leader_ephs_req); + ec_agg->ea_refreshing = 0; if (rc) { DL_CDEBUG(rc == -DER_NONEXIST, DLOG_INFO, DLOG_ERR, rc, DF_CONT ": refresh failed", @@ -2144,6 +2203,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) continue; } ec_agg->ea_current_eph = min_eph; + ec_agg->ea_refresh_ts = daos_gettime_coarse(); } ABT_mutex_unlock(svc->cs_ec_agg_mutex); @@ -2166,7 +2226,6 @@ ds_cont_svc_refresh_agg_eph(uuid_t pool_uuid) return 0; } -#define EC_AGG_EPH_INTV (10ULL * 1000) /* seconds interval to check*/ static void cont_agg_eph_leader_ult(void *arg) { @@ -6278,9 +6337,9 @@ ds_cont_destroy_orphan(struct cont_svc *svc, uuid_t uuid) /* Forbid to destroy non-orphan container. */ D_GOTO(out, rc = -DER_BUSY); - rc = cont_destroy_bcast(dss_get_module_info()->dmi_ctx, svc, uuid); + rc = cont_ec_agg_delete(svc, uuid); if (rc == 0) - cont_ec_agg_delete(svc, uuid); + rc = cont_destroy_bcast(dss_get_module_info()->dmi_ctx, svc, uuid); out: D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, diff --git a/src/container/srv_internal.h b/src/container/srv_internal.h index d53205a4be1..42998c67a68 100644 --- a/src/container/srv_internal.h +++ b/src/container/srv_internal.h @@ -69,11 +69,16 @@ struct cont_ec_agg { uuid_t ea_cont_uuid; daos_epoch_t ea_current_eph; daos_epoch_t ea_rdb_eph; + /* Ignore boundary jumps predating this leader's tracking. */ + uint64_t ea_start_ts; + uint64_t ea_refresh_ts; uint64_t ea_warn_slug_ts; struct ec_eph *ea_server_ephs; d_list_t ea_list; int ea_servers_num; - uint32_t ea_deleted : 1; + uint32_t ea_deleted : 1, ea_refreshing : 1, + /* some rank waits for the boundary (reported 0 or nothing), see cont_agg_eph_sync() */ + ea_need_refresh : 1; }; /* @@ -165,6 +170,10 @@ struct cont_iv_prop { struct cont_iv_agg_eph { daos_epoch_t eph; + /* Reports carry the reporter rank. For boundaries, CRT_NO_RANK allows a push to create + * missing VOS containers; fetch replies use 0 to disallow creation, not as + * a reporter rank. Only leader pushes are tracked against container destroy. + */ d_rank_t rank; }; @@ -275,8 +284,8 @@ int ds_cont_tgt_snapshots_update(uuid_t pool_uuid, uuid_t cont_uuid, uint64_t *snapshots, int snap_count); int ds_cont_tgt_snapshots_refresh(uuid_t pool_uuid, uuid_t cont_uuid); int ds_cont_tgt_close(uuid_t pool_uuid, uuid_t cont_hdl_uuid); -int ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, - daos_epoch_t eph); +int + ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, daos_epoch_t eph, bool create); int ds_cont_tgt_prop_update(uuid_t pool_uuid, uuid_t cont_uuid, daos_prop_t *prop); /* oid_iv.c */ diff --git a/src/container/srv_target.c b/src/container/srv_target.c index ef05ad89e3b..598bded6da9 100644 --- a/src/container/srv_target.c +++ b/src/container/srv_target.c @@ -1428,6 +1428,7 @@ cont_child_create_start(uuid_t pool_uuid, uuid_t cont_uuid, uint32_t pm_ver, bool *started, struct ds_cont_child **cont_out) { struct ds_pool_child *pool_child; + bool created = true; int rc; pool_child = ds_pool_child_lookup(pool_uuid); @@ -1451,11 +1452,16 @@ cont_child_create_start(uuid_t pool_uuid, uuid_t cont_uuid, uint32_t pm_ver, DP_CONT(pool_uuid, cont_uuid)); rc = vos_cont_create(pool_child->spc_hdl, cont_uuid); + if (unlikely(rc == -DER_EXIST)) { + /* Creation may yield; do not roll back another sponsor's container. */ + rc = 0; + created = false; + } if (!rc) { rc = cont_child_start(pool_child, cont_uuid, started, cont_out); if (rc == 0) { (*cont_out)->sc_status_pm_ver = pm_ver; - } else { + } else if (created) { int rc_tmp; rc_tmp = vos_cont_destroy(pool_child->spc_hdl, cont_uuid); @@ -1466,7 +1472,7 @@ cont_child_create_start(uuid_t pool_uuid, uuid_t cont_uuid, uint32_t pm_ver, } ds_pool_child_put(pool_child); - return rc == 0 ? 1 : rc; + return rc == 0 ? (created ? 1 : 0) : rc; } int @@ -2563,10 +2569,13 @@ ds_cont_tgt_ec_eph_query_ult(void *data) continue; /* if aggregation enabled, make sure to report ec_agg_eph at the start phase - * when min_eph and ce_last_eph are both zero. + * when min_eph and ce_last_eph are both zero. Also report min_eph 0 after + * non-zero reports: a reintegrated target has no boundary yet and waits + * for the leader to resend it, see ds_cont_leader_update_agg_eph(). */ if (min_eph == DAOS_EPOCH_MAX || - (ec_eph->ce_last_eph != 0 && min_eph <= ec_eph->ce_last_eph)) { + (ec_eph->ce_last_eph != 0 && min_eph != 0 && + min_eph <= ec_eph->ce_last_eph)) { if (min_eph > 0 && min_eph < ec_eph->ce_last_eph) D_ERROR("ignore for now "DF_X64" < "DF_X64 " "DF_UUID"\n", min_eph, ec_eph->ce_last_eph,