From 3585d8d970f1b73869a762de990e71b06a03f033 Mon Sep 17 00:00:00 2001 From: Liu Xuezhao Date: Wed, 15 Oct 2025 15:43:13 +0800 Subject: [PATCH 01/43] DAOS-17589 pool: not changing fseq for exclude during drain (#16753) If the target is being drained, then exclude should not change its fseq number, otherwise it may get wrong object layout and interfere the drain process. Signed-off-by: Di Wang Co-authored-by: Xuezhao Liu --- src/pool/srv_pool_map.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pool/srv_pool_map.c b/src/pool/srv_pool_map.c index 0807a87e048..32f0710102c 100644 --- a/src/pool/srv_pool_map.c +++ b/src/pool/srv_pool_map.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2021-2024 Intel Corporation. + * (C) Copyright 2025 Google LLC * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent @@ -72,13 +73,13 @@ update_one_tgt(uuid_t pool_uuid, struct pool_map *map, struct pool_target *targe switch (target->ta_comp.co_status) { case PO_COMP_ST_DOWN: case PO_COMP_ST_DOWNOUT: + case PO_COMP_ST_DRAIN: /* Nothing to do, already excluded */ D_INFO(DF_MAP ": Skip exclude down " DF_TARGET "\n", DP_MAP(pool_uuid, map), DP_TARGET(target)); break; case PO_COMP_ST_UP: case PO_COMP_ST_UPIN: - case PO_COMP_ST_DRAIN: D_DEBUG(DB_MD, DF_MAP ": change " DF_TARGET " to DOWN\n", DP_MAP(pool_uuid, map), DP_TARGET(target)); target->ta_comp.co_status = PO_COMP_ST_DOWN; From b9c6229997182ddd550cd93e7e4089e80f6402d8 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Wed, 15 Oct 2025 14:05:30 -0700 Subject: [PATCH 02/43] DAOS-18029 cq: do not run trivy on all prs (#16935) (#16957) Trivy often fails and is unrelated to most PRs. Only run it on PRs if relevant files are touched. Signed-off-by: Dalton Bohning --- .github/workflows/trivy.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 92f84cd4b65..1f4b2f52a28 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -11,6 +11,16 @@ on: branches: ["master", "release/**"] pull_request: branches: ["master", "release/**"] + paths: + - .github/workflows/trivy.yml + - requirements-*.txt + - src/client/java/**/pom.xml + - src/control/go.mod + - src/control/vendor/** + - utils/build.config + - utils/rpms/** + - utils/scripts/install-*.sh + - utils/trivy/** # Declare default permissions as nothing. permissions: {} @@ -23,7 +33,7 @@ jobs: security-events: write steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Run Trivy vulnerability scanner in filesystem mode (table format) uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # 0.33.1 From 599f078dd78c9ecefc0dfb02d98e844ad6440fe0 Mon Sep 17 00:00:00 2001 From: Wang Shilong Date: Thu, 16 Oct 2025 23:40:25 +0800 Subject: [PATCH 03/43] DAOS-18103: container: fix compatibility for newer clients with older servers (#16981) Now the pool connection will return the server's maximum supported layout version, allowing the client to determine which version to use for container creation/open. Signed-off-by: Wang Shilong Co-authored-by: Mohamad Chaarawi --- src/client/pydaos/raw/daos_cref.py | 4 +++- src/container/cli.c | 20 ++++++++++++-------- src/include/daos/pool.h | 3 +++ src/include/daos_pool.h | 7 ++++++- src/pool/cli.c | 21 ++++++++++++++++----- src/pool/srv_pool.c | 2 ++ src/rebuild/README.md | 6 ++++-- src/tests/ftest/util/test_utils_pool.py | 4 ++-- 8 files changed, 48 insertions(+), 19 deletions(-) diff --git a/src/client/pydaos/raw/daos_cref.py b/src/client/pydaos/raw/daos_cref.py index ceb2a5b6885..3878912b698 100644 --- a/src/client/pydaos/raw/daos_cref.py +++ b/src/client/pydaos/raw/daos_cref.py @@ -1,5 +1,6 @@ """ (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -51,7 +52,8 @@ class RebuildStatus(ctypes.Structure): ("rs_seconds", ctypes.c_uint32), ("rs_errno", ctypes.c_uint32), ("rs_state", ctypes.c_uint32), - ("rs_padding32", ctypes.c_uint32), + ("rs_max_supported_layout_ver", ctypes.c_uint16), + ("rs_padding16", ctypes.c_uint16), ("rs_fail_rank", ctypes.c_uint32), ("rs_toberb_obj_nr", ctypes.c_uint64), ("rs_obj_nr", ctypes.c_uint64), diff --git a/src/container/cli.c b/src/container/cli.c index 18b6a75a590..70287280cd0 100644 --- a/src/container/cli.c +++ b/src/container/cli.c @@ -178,8 +178,8 @@ daos_prop_has_entry(daos_prop_t *prop, uint32_t entry_type) * The newly allocated prop is expected to be freed by the cont create callback. */ static int -dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, - daos_prop_t *prop_in) +dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, daos_prop_t *prop_in, + uint32_t co_ver) { char *owner = NULL; char *owner_grp = NULL; @@ -189,8 +189,7 @@ dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, uint32_t entries; int rc = 0; uid_t uid = geteuid(); - gid_t gid = getegid(); - uint32_t co_ver = 0; + gid_t gid = getegid(); entries = (prop_in == NULL) ? 0 : prop_in->dpp_nr; @@ -226,8 +225,10 @@ dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, entries++; } if (!daos_prop_has_entry(prop_in, DAOS_PROP_CO_OBJ_VERSION)) { - co_ver = DAOS_POOL_OBJ_VERSION_2; - entries++; + if (co_ver >= DAOS_POOL_OBJ_VERSION_2) + entries++; + else /* don't specify version if server is old */ + co_ver = 0; } if (!daos_prop_has_entry(prop_in, DAOS_PROP_CO_ROOTS)) { @@ -351,7 +352,8 @@ dc_cont_create(tse_task_t *task) D_GOTO(err_task, rc = -DER_NO_HDL); /** duplicate properties and add extra ones (e.g. ownership) */ - rc = dup_cont_create_props(args->poh, &rpc_prop, args->prop); + rc = dup_cont_create_props(args->poh, &rpc_prop, args->prop, + pool->dp_max_supported_layout_ver); if (rc != 0) D_GOTO(err_pool, rc); @@ -1009,7 +1011,9 @@ dc_cont_open(tse_task_t *task) if (tpriv->cont == NULL) D_GOTO(err_pool, rc = -DER_NOMEM); uuid_generate(tpriv->cont->dc_cont_hdl); - tpriv->cont->dc_capas = args->flags | DAOS_COO_NEW_LAYOUT; + tpriv->cont->dc_capas = args->flags; + if (pool->dp_max_supported_layout_ver >= DAOS_POOL_OBJ_VERSION_2) + tpriv->cont->dc_capas |= DAOS_COO_NEW_LAYOUT; } D_DEBUG(DB_MD, DF_UUID ":%s: opening: hdl=" DF_UUIDF " flags=%x\n", DP_UUID(pool->dp_pool), diff --git a/src/include/daos/pool.h b/src/include/daos/pool.h index 2747b8c3432..78e628b0bf6 100644 --- a/src/include/daos/pool.h +++ b/src/include/daos/pool.h @@ -34,6 +34,7 @@ /** pool query request bits */ #define DAOS_PO_QUERY_SPACE (1ULL << 0) #define DAOS_PO_QUERY_REBUILD_STATUS (1ULL << 1) +#define DAOS_PO_QUERY_REBULD_MAX_LAYOUT_VER (1ULL << 2) #define PROP_BIT_START 16 #define DAOS_PO_QUERY_PROP_BIT_START PROP_BIT_START #define DAOS_PO_QUERY_PROP_LABEL (1ULL << (PROP_BIT_START + 0)) @@ -135,6 +136,8 @@ struct dc_pool { /* pool redunc factor */ uint32_t dp_rf; + /* Maximum supported layout version */ + uint16_t dp_max_supported_layout_ver; }; static inline unsigned int diff --git a/src/include/daos_pool.h b/src/include/daos_pool.h index 24746a70ff7..a41aba71ac6 100644 --- a/src/include/daos_pool.h +++ b/src/include/daos_pool.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2020-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -121,8 +122,10 @@ struct daos_rebuild_status { int32_t rs_state; int32_t rs_done; }; + /** Maximum supported layout version */ + uint16_t rs_max_supported_layout_ver; /** padding of rebuild status */ - int32_t rs_padding32; + int16_t rs_padding16; /** Failure on which rank */ int32_t rs_fail_rank; @@ -164,6 +167,8 @@ enum daos_pool_info_bit { DPI_ENGINES_DISABLED = 1ULL << 3, /** true to include (in \a ranks) engines marked DEAD by SWIM. */ DPI_ENGINES_DEAD = 1ULL << 4, + /** true to query pool's maximum supported rebuild layout version */ + DPI_REBUILD_MAX_LAYOUT_VER = 1ULL << 5, /** query all above optional info */ DPI_ALL = -1, }; diff --git a/src/pool/cli.c b/src/pool/cli.c index 49d9b4155ca..09baee2204d 100644 --- a/src/pool/cli.c +++ b/src/pool/cli.c @@ -861,6 +861,7 @@ pool_connect_cp(tse_task_t *task, void *data) d_iov_t *credp; bool free_tpriv = true; int rc = task->dt_result; + struct daos_rebuild_status *rs; rc = pool_rsvc_client_complete_rpc(tpriv->pool, &arg->rpc->cr_ep, rc, &pco->pco_op, task); if (rc < 0) { @@ -892,9 +893,14 @@ pool_connect_cp(tse_task_t *task, void *data) D_GOTO(out, rc); } + rs = &pco->pco_rebuild_st; rc = process_query_reply(tpriv->pool, map_buf, pco->pco_op.po_map_version, - pco->pco_op.po_hint.sh_rank, &pco->pco_space, &pco->pco_rebuild_st, - NULL /* tgts */, info, NULL, NULL, true); + pco->pco_op.po_hint.sh_rank, &pco->pco_space, rs, NULL /* tgts */, + info, NULL, NULL, true); + + tpriv->pool->dp_max_supported_layout_ver = rs->rs_max_supported_layout_ver + ? rs->rs_max_supported_layout_ver + : DAOS_POOL_OBJ_VERSION_1; if (rc != 0) { if (rc == -DER_AGAIN) { rc = tse_task_reinit(task); @@ -1036,8 +1042,9 @@ dc_pool_connect_internal(tse_task_t *task, daos_pool_info_t *info, const char *l D_GOTO(out_cred, rc); /** fill in request buffer */ - pool_connect_in_set_data(rpc, pool->dp_capas, pool_query_bits(info, NULL), bulk, - DAOS_POOL_GLOBAL_VERSION); + pool_connect_in_set_data(rpc, pool->dp_capas, + pool_query_bits(info, NULL) | DAOS_PO_QUERY_REBULD_MAX_LAYOUT_VER, + bulk, DAOS_POOL_GLOBAL_VERSION); /** Prepare "con_args" for pool_connect_cp(). */ con_args.pca_info = info; @@ -1299,7 +1306,8 @@ dc_pool_disconnect(tse_task_t *task) struct dc_pool_glob { /* magic number, DC_POOL_GLOB_MAGIC */ uint32_t dpg_magic; - uint32_t dpg_padding; + uint16_t dpg_max_supported_layout_ver; + uint16_t dpg_padding; /* pool UUID, pool handle UUID, and capas */ uuid_t dpg_pool; uuid_t dpg_pool_hdl; @@ -1354,6 +1362,7 @@ swap_pool_glob(struct dc_pool_glob *pool_glob) D_SWAP32S(&pool_glob->dpg_magic); /* skip pool_glob->dpg_padding */ + D_SWAP16S(&pool_glob->dpg_max_supported_layout_ver); /* skip pool_glob->dpg_pool (uuid_t) */ /* skip pool_glob->dpg_pool_hdl (uuid_t) */ D_SWAP64S(&pool_glob->dpg_capas); @@ -1421,6 +1430,7 @@ dc_pool_l2g(daos_handle_t poh, d_iov_t *glob) /* init pool global handle */ pool_glob = (struct dc_pool_glob *)glob->iov_buf; pool_glob->dpg_magic = DC_POOL_GLOB_MAGIC; + pool_glob->dpg_max_supported_layout_ver = pool->dp_max_supported_layout_ver; uuid_copy(pool_glob->dpg_pool, pool->dp_pool); uuid_copy(pool_glob->dpg_pool_hdl, pool->dp_pool_hdl); pool_glob->dpg_capas = pool->dp_capas; @@ -1491,6 +1501,7 @@ dc_pool_g2l(struct dc_pool_glob *pool_glob, size_t len, daos_handle_t *poh) if (pool == NULL) D_GOTO(out, rc = -DER_NOMEM); + pool->dp_max_supported_layout_ver = pool_glob->dpg_max_supported_layout_ver; uuid_copy(pool->dp_pool, pool_glob->dpg_pool); uuid_copy(pool->dp_pool_hdl, pool_glob->dpg_pool_hdl); pool->dp_capas = pool_glob->dpg_capas; diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 64582d9daeb..5e066cbf0d5 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -3941,6 +3941,8 @@ ds_pool_connect_handler(crt_rpc_t *rpc, int handler_version) if (rc != 0) D_GOTO(out_svc, rc); } + if (query_bits & DAOS_PO_QUERY_REBULD_MAX_LAYOUT_VER) + out->pco_rebuild_st.rs_max_supported_layout_ver = DAOS_POOL_OBJ_VERSION; rc = rdb_tx_begin(svc->ps_rsvc.s_db, svc->ps_rsvc.s_term, &tx); if (rc != 0) diff --git a/src/rebuild/README.md b/src/rebuild/README.md index 47827b62bfd..33f6c32d7ca 100644 --- a/src/rebuild/README.md +++ b/src/rebuild/README.md @@ -204,8 +204,10 @@ struct daos_rebuild_status { int32_t rs_done; }; - /* padding of rebuild status */ - int32_t rs_padding32; + /** Maximum supported layout version */ + uint16_t rs_max_supported_layout_ver; + /** padding of rebuild status */ + int16_t rs_padding16; /* Failure on which rank */ int32_t rs_fail_rank; diff --git a/src/tests/ftest/util/test_utils_pool.py b/src/tests/ftest/util/test_utils_pool.py index cd50f144e35..1b41c8c8b57 100644 --- a/src/tests/ftest/util/test_utils_pool.py +++ b/src/tests/ftest/util/test_utils_pool.py @@ -959,7 +959,7 @@ def check_free_space(self, expected_scm=None, expected_nvme=None, timeout=30, in return False def check_rebuild_status(self, rs_version=None, rs_seconds=None, - rs_errno=None, rs_state=None, rs_padding32=None, + rs_errno=None, rs_state=None, rs_padding16=None, rs_fail_rank=None, rs_toberb_obj_nr=None, rs_obj_nr=None, rs_rec_nr=None, rs_size=None): # pylint: disable=unused-argument @@ -976,7 +976,7 @@ def check_rebuild_status(self, rs_version=None, rs_seconds=None, rs_seconds (int, optional): rebuild seconds. Defaults to None. rs_errno (int, optional): rebuild error number. Defaults to None. rs_state (int, optional): rebuild state flag. Defaults to None. - rs_padding32 (int, optional): padding. Defaults to None. + rs_padding16 (int, optional): padding. Defaults to None. rs_fail_rank (int, optional): rebuild fail target. Defaults to None. rs_toberb_obj_nr (int, optional): number of objects to be rebuilt. Defaults to None. From c979f32c347de9a77810b86ceeae328d74012fd1 Mon Sep 17 00:00:00 2001 From: Liang Zhen Date: Sat, 18 Oct 2025 03:06:02 +0800 Subject: [PATCH 04/43] DAOS-17772 rebuild: avoid potential races between rebuild and EC aggregation (#16945) Increment the map_version for reclaim tasks to ensure they operate on the latest pool map. Without this, reclaim tasks might act on an outdated pool map, potentially causing data corruption. Always recalculate parity for partial overwrites, disabling the optimized path to ensure data consistency. Ignore parity update errors during EC aggregation. This is a temporary workaround because DAOS does not use DTX to update parities during EC aggregation. Without DTX, parity shards may become inconsistent if an engine failure occurs alongside a rebuild. This approach significantly increases the likelihood of parity shards remaining consistent when EC aggregation interacts with rebuild operations. Standardize the parity target status checks for both full stripe and hole processing during EC aggregation. Introduce additional barriers to minimize the chances of overlapping EC aggregation and rebuild operations. Signed-off-by: Liang Zhen --- src/container/srv_target.c | 64 +++- src/include/daos_srv/container.h | 12 + src/include/daos_srv/pool.h | 18 +- src/include/daos_srv/rebuild.h | 1 + src/object/cli_obj.c | 4 +- src/object/srv_ec_aggregate.c | 397 ++++++++++---------- src/object/srv_obj.c | 27 +- src/object/srv_obj_migrate.c | 6 +- src/pool/srv_pool.c | 7 +- src/rebuild/rebuild_internal.h | 1 + src/rebuild/scan.c | 18 +- src/rebuild/srv.c | 25 +- src/tests/ftest/util/server_utils_params.py | 1 + src/tests/suite/daos_container.c | 8 +- 14 files changed, 354 insertions(+), 235 deletions(-) diff --git a/src/container/srv_target.c b/src/container/srv_target.c index c86c273916a..e30738e1696 100644 --- a/src/container/srv_target.c +++ b/src/container/srv_target.c @@ -61,7 +61,7 @@ agg_rate_ctl(void *arg) * XXX temporary workaround: EC aggregation needs to be paused during rebuilding * to avoid the race between EC rebuild and EC aggregation. **/ - if (pool->sp_rebuilding && cont->sc_ec_agg_active && !param->ap_vos_agg) + if (ds_pool_is_rebuilding(pool) && cont->sc_ec_agg_active && !param->ap_vos_agg) return -1; /* When system is idle or under space pressure, let aggregation run in tight mode */ @@ -181,10 +181,10 @@ cont_aggregate_runnable(struct ds_cont_child *cont, struct sched_request *req, return false; } - if (pool->sp_rebuilding && !vos_agg) { - D_DEBUG(DB_EPC, DF_CONT": skip EC aggregation during rebuild %d.\n", - DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), - pool->sp_rebuilding); + if (ds_pool_is_rebuilding(pool) && !vos_agg) { + D_DEBUG(DB_EPC, DF_CONT ": skip EC aggregation during rebuild %d, %d.\n", + DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), pool->sp_rebuilding, + pool->sp_rebuild_scan); return false; } @@ -499,7 +499,8 @@ cont_aggregate_interval(struct ds_cont_child *cont, cont_aggregate_cb_t cb, /* sleep 18 seconds for EC aggregation ULT if the pool is in rebuilding, * if no space pressure. */ - if (cont->sc_pool->spc_pool->sp_rebuilding && !param->ap_vos_agg && msecs != 200) + if (ds_pool_is_rebuilding(cont->sc_pool->spc_pool) && !param->ap_vos_agg && + msecs != 200) msecs = 18000; sched_req_sleep(req, msecs); @@ -917,6 +918,57 @@ ds_cont_child_reset_ec_agg_eph_all(struct ds_pool_child *pool_child) cont_child->sc_ec_agg_eph = cont_child->sc_ec_agg_eph_boundary; } +#define WAIT_EC_PAUSE_MAX 600 + +void +ds_cont_child_wait_ec_agg_pause(struct ds_pool_child *pool_child, int wait_timeout) +{ + uint64_t start_time = daos_wallclock_secs(); + int wait_intv = 10; + int waited = 0; + + D_DEBUG(DB_MD, DF_UUID "[%d]: wait for pausing EC aggregation\n", + DP_UUID(pool_child->spc_uuid), dss_get_module_info()->dmi_tgt_id); + + if (wait_timeout == 0 || wait_timeout > WAIT_EC_PAUSE_MAX) + wait_timeout = WAIT_EC_PAUSE_MAX; /* 10 minutes by default */ + + while (1) { + struct ds_cont_child *coc; + bool paused = true; + + /* Wait for pausing aggregation + * XXX: There is no global barrier so we always wait for at least 10 seconds to + * lower the chance that remote targets are still running EC aggregation. + */ + if (wait_intv > wait_timeout - waited) + wait_intv = wait_timeout - waited; + + dss_sleep(wait_intv * 1000); + d_list_for_each_entry(coc, &pool_child->spc_cont_list, sc_link) { + if (ds_cont_child_ec_aggregating(coc)) { + /* Aggregation is active on this container */ + paused = false; + break; + } + } + if (paused) + return; + + waited = daos_wallclock_secs() - start_time; + if (waited >= wait_timeout) { + D_WARN("can't pause EC aggregation after %d seconds\n", waited); + return; /* XXX what can I do? */ + } + + if (waited % 60 == 0) { + D_WARN(DF_UUID "[%d]: waited %d secs for EC aggregation to pause\n", + DP_UUID(pool_child->spc_uuid), dss_get_module_info()->dmi_tgt_id, + waited); + } + } +} + static int cont_child_start(struct ds_pool_child *pool_child, const uuid_t co_uuid, bool *started, struct ds_cont_child **cont_out) diff --git a/src/include/daos_srv/container.h b/src/include/daos_srv/container.h index 0853196b1f8..5cb2d466027 100644 --- a/src/include/daos_srv/container.h +++ b/src/include/daos_srv/container.h @@ -136,6 +136,7 @@ struct ds_cont_child { d_list_t sc_dtx_batched_list; /* the pool map version of updating DAOS_PROP_CO_STATUS prop */ uint32_t sc_status_pm_ver; + int sc_ec_agg_updates; }; struct agg_param { @@ -148,6 +149,14 @@ struct agg_param { typedef int (*cont_aggregate_cb_t)(struct ds_cont_child *cont, daos_epoch_range_t *epr, uint32_t flags, struct agg_param *param); + +static inline bool +ds_cont_child_ec_aggregating(struct ds_cont_child *cont) +{ + /* either I'm running aggregation, or someone else is writing to me for aggregation */ + return cont->sc_ec_agg_updates || cont->sc_ec_agg_active; +} + void cont_aggregate_interval(struct ds_cont_child *cont, cont_aggregate_cb_t cb, struct agg_param *param); @@ -198,6 +207,9 @@ ds_cont_child_destroy(uuid_t pool_uuid, uuid_t cont_uuid); void ds_cont_child_reset_ec_agg_eph_all(struct ds_pool_child *pool_child); +void + ds_cont_child_wait_ec_agg_pause(struct ds_pool_child *pool_child, int wait_timeout); + /** initialize a csummer based on container properties. Will retrieve the * checksum related properties from IV */ diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index 7cf244e3cb0..e97c1fcad05 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -90,9 +90,13 @@ struct ds_pool { /* pool_uuid + map version + leader term + rebuild generation define a * rebuild job. */ - uint32_t sp_rebuild_gen; - + uint32_t sp_rebuild_gen; int sp_rebuilding; + /** + * someone has already messaged this pool to for rebuild scan, + * NB: all xstreams can do lockless-write on it but it's OK + */ + int sp_rebuild_scan; int sp_discard_status; /** path to ephemeral metrics */ @@ -206,6 +210,12 @@ struct ds_pool_svc_op_val { char ov_resvd[60]; }; +static inline bool +ds_pool_is_rebuilding(struct ds_pool *pool) +{ + return (pool->sp_rebuilding > 0 || pool->sp_rebuild_scan > 0); +} + /* encode metadata RPC operation key: HLC time first, in network order, for keys sorted by time. * allocates the byte-stream, caller must free with D_FREE(). */ @@ -273,7 +283,9 @@ int int ds_pool_tgt_add_in(uuid_t pool_uuid, struct pool_target_id_list *list); int ds_pool_tgt_revert_rebuild(uuid_t pool_uuid, struct pool_target_id_list *list); -int ds_pool_tgt_finish_rebuild(uuid_t pool_uuid, struct pool_target_id_list *list); +int + ds_pool_tgt_finish_rebuild(uuid_t pool_uuid, struct pool_target_id_list *list, + uint32_t *reclaim_ver); int ds_pool_tgt_map_update(struct ds_pool *pool, struct pool_buf *buf, unsigned int map_version); diff --git a/src/include/daos_srv/rebuild.h b/src/include/daos_srv/rebuild.h index bbf7abac479..72ad383405c 100644 --- a/src/include/daos_srv/rebuild.h +++ b/src/include/daos_srv/rebuild.h @@ -16,6 +16,7 @@ #define REBUILD_ENV "DAOS_REBUILD" #define REBUILD_ENV_DISABLED "no" +#define REBUILD_WAIT_EC_PAUSE_ENV "DAOS_REBUILD_WAIT_EC_PAUSE" /** * Enum values to indicate the rebuild operation that should be applied to the diff --git a/src/object/cli_obj.c b/src/object/cli_obj.c index 75089cb1ae1..45a1e447e12 100644 --- a/src/object/cli_obj.c +++ b/src/object/cli_obj.c @@ -3771,9 +3771,9 @@ obj_shard_comp_cb(tse_task_t *task, struct shard_auxi_args *shard_auxi, * 2) if any shard failed, store it in obj_auxi->result, the * un-retryable error with higher priority. */ + if (obj_auxi->map_ver_reply < shard_auxi->map_ver) + obj_auxi->map_ver_reply = shard_auxi->map_ver; if (ret == 0) { - if (obj_auxi->map_ver_reply < shard_auxi->map_ver) - obj_auxi->map_ver_reply = shard_auxi->map_ver; if (obj_req_is_ec_cond_fetch(obj_auxi)) { iter_arg->cond_fetch_exist = true; if (obj_auxi->result == -DER_NONEXIST) diff --git a/src/object/srv_ec_aggregate.c b/src/object/srv_ec_aggregate.c index ed30865f61d..bb05d449616 100644 --- a/src/object/srv_ec_aggregate.c +++ b/src/object/srv_ec_aggregate.c @@ -132,6 +132,7 @@ struct ec_agg_param { daos_handle_t ap_cont_handle; /* VOS container handle */ int (*ap_yield_func)(void *arg); /* yield function*/ void *ap_yield_arg; /* yield argument */ + struct ds_cont_child *ap_cont; uint32_t ap_credits_max; /* # of tight loops to yield */ uint32_t ap_credits; /* # of tight loops */ uint32_t ap_initialized:1; /* initialized flag */ @@ -145,8 +146,7 @@ struct ec_agg_stripe_ud { daos_recx_t *asu_recxs; /* For re-replicate */ unsigned int asu_cell_cnt; /* Count of cells */ bool asu_recalc; /* Should recalc parity */ - bool asu_write_par; /* Should write parity */ - struct daos_csummer *asu_csummer; + bool asu_write_par; /* Should write parity */ daos_iod_t asu_iod; d_iov_t asu_csum_iov; struct dcs_iod_csums *asu_iod_csums; /* iod csums */ @@ -786,12 +786,12 @@ agg_update_vos(struct ec_agg_param *agg_param, struct ec_agg_entry *entry, { daos_recx_t recx = { 0 }; daos_epoch_range_t epoch_range = { 0 }; - struct ec_agg_param *ap; - struct ec_agg_extent *ext; + struct ec_agg_param *ap; uint32_t len = ec_age2cs(entry); uint32_t pidx = ec_age2pidx(entry); struct daos_csummer *csummer; struct dcs_iod_csums *iod_csums = NULL; + int err; int rc = 0; ap = container_of(entry, struct ec_agg_param, ap_agg_entry); @@ -837,21 +837,16 @@ agg_update_vos(struct ec_agg_param *agg_param, struct ec_agg_entry *entry, } } - d_list_for_each_entry(ext, &entry->ae_cur_stripe.as_dextents, ae_link) { - int err; - - epoch_range.epr_lo = epoch_range.epr_hi = ext->ae_epoch; - err = vos_obj_array_remove(ap->ap_cont_handle, - entry->ae_oid, - &epoch_range, - &entry->ae_dkey, - &entry->ae_akey, - &ext->ae_recx); - if (err) - D_ERROR("array_remove fails: "DF_RC"\n", DP_RC(err)); - if (!rc && err) - rc = err; - } + recx.rx_idx = entry->ae_cur_stripe.as_stripenum * ec_age2ss(entry); + recx.rx_nr = ec_age2ss(entry); + epoch_range.epr_lo = ap->ap_epr.epr_lo; + epoch_range.epr_hi = entry->ae_cur_stripe.as_hi_epoch; + err = vos_obj_array_remove(ap->ap_cont_handle, entry->ae_oid, &epoch_range, &entry->ae_dkey, + &entry->ae_akey, &recx); + if (err) + D_ERROR("array_remove fails: " DF_RC "\n", DP_RC(err)); + if (!rc && err) + rc = err; out: return rc; } @@ -1224,6 +1219,7 @@ agg_process_partial_stripe(struct ec_agg_entry *entry) entry->ae_cur_stripe.as_stripenum, &full_cell_cnt); + has_old_replicas = true; /* XXX force full recalculation */ if (full_cell_cnt >= k / 2 || cell_cnt == k || has_old_replicas) { stripe_ud.asu_recalc = true; cell_cnt = full_cell_cnt; @@ -1282,6 +1278,62 @@ agg_process_partial_stripe(struct ec_agg_entry *entry) return rc; } +int +agg_peer_check_avail(struct ec_agg_param *agg_param, struct ec_agg_entry *entry) +{ + struct pool_target *targets = NULL; + struct daos_shard_loc *peer_loc; + uint32_t failed_tgts_cnt = 0; + uint32_t p = ec_age2p(entry); + uint32_t pidx = ec_age2pidx(entry); + int peer; + int i; + int rc; + + if (ds_pool_is_rebuilding(agg_param->ap_cont->sc_pool->spc_pool)) { + /* We currently pause EC aggregation for rebuild, so just cancel the + * aggregation for the current stripe. It means the following peer status + * check may not be checked at all, but let's keep the code because it could + * be useful in the future. + */ + D_ERROR(DF_UOID " pauses EC aggregation for rebuild\n", DP_UOID(entry->ae_oid)); + return -DER_OP_CANCELED; + } + + rc = pool_map_find_failed_tgts(agg_param->ap_pool_info.api_pool->sp_map, &targets, + &failed_tgts_cnt); + if (rc) { + D_ERROR(DF_UOID " pool_map_find_failed_tgts failed: " DF_RC "\n", + DP_UOID(entry->ae_oid), DP_RC(rc)); + return rc; + } + + if (targets == NULL) + return 0; + + for (peer = 0; peer < p; peer++) { + if (peer == pidx) + continue; + + peer_loc = &entry->ae_peer_pshards[peer]; + for (i = 0; i < failed_tgts_cnt; i++) { + if (peer_loc->sd_rank == DAOS_TGT_IGNORE || + (targets[i].ta_comp.co_rank == peer_loc->sd_rank && + targets[i].ta_comp.co_index == peer_loc->sd_tgt_idx)) { + D_DEBUG(DB_EPC, + DF_UOID " peer parity tgt failed rank %d, " + "tgt_idx %d.\n", + DP_UOID(entry->ae_oid), peer_loc->sd_rank, + peer_loc->sd_tgt_idx); + D_GOTO(out, rc = -DER_OP_CANCELED); + } + } + } +out: + D_FREE(targets); + return rc; +} + /* Sends the generated parity and the stripe number to the peer * parity target. Handler writes the parity and deletes the replicas * for the stripe. @@ -1299,7 +1351,7 @@ agg_peer_update_ult(void *arg) crt_endpoint_t tgt_ep = { 0 }; unsigned char *buf = NULL; struct obj_ec_agg_in *ec_agg_in = NULL; - struct obj_ec_agg_out *ec_agg_out; + struct obj_ec_agg_out *ec_agg_out = NULL; struct ec_agg_param *agg_param; crt_bulk_t bulk_hdl = NULL; struct daos_csummer *csummer = NULL; @@ -1311,34 +1363,42 @@ agg_peer_update_ult(void *arg) uint32_t peer, peer_shard; struct dc_object *obj = NULL; crt_rpc_t *rpc = NULL; - int rc = 0; - uint64_t enqueue_id; + int peer_rc = 0, rc = 0; + int peer_updated = 0; uint32_t max_delay = 0; if (unlikely(DAOS_FAIL_CHECK(DAOS_FORCE_EC_AGG_PEER_FAIL))) D_GOTO(out, rc = -DER_TIMEDOUT); agg_param = container_of(entry, struct ec_agg_param, ap_agg_entry); + rc = agg_peer_check_avail(agg_param, entry); + if (rc != 0) + D_GOTO(out, rc); + csummer = ec_agg_param2csummer(agg_param); iod.iod_type = DAOS_IOD_ARRAY; iod.iod_name = entry->ae_akey; iod.iod_size = entry->ae_rsize; obj = obj_hdl2ptr(entry->ae_obj_hdl); for (peer = 0; peer < p; peer++) { + uint64_t enqueue_id = 0; + bool overloaded; + if (peer == pidx) continue; D_ASSERT(entry->ae_peer_pshards[peer].sd_rank != DAOS_TGT_IGNORE); tgt_ep.ep_rank = entry->ae_peer_pshards[peer].sd_rank; - tgt_ep.ep_tag = entry->ae_peer_pshards[peer].sd_tgt_idx; - enqueue_id = 0; + tgt_ep.ep_tag = entry->ae_peer_pshards[peer].sd_tgt_idx; retry: + overloaded = false; rc = obj_req_create(dss_get_module_info()->dmi_ctx, &tgt_ep, DAOS_OBJ_RPC_EC_AGGREGATE, &rpc); if (rc) { D_ERROR(DF_UOID" pidx %d to peer %d, rank %d tag %d obj_req_create " DF_RC"\n", DP_UOID(entry->ae_oid), pidx, peer, tgt_ep.ep_rank, tgt_ep.ep_tag, DP_RC(rc)); - goto out; + if (peer_updated == 0) + goto next; } ec_agg_in = crt_req_get(rpc); uuid_copy(ec_agg_in->ea_pool_uuid, @@ -1379,26 +1439,21 @@ agg_peer_update_ult(void *arg) rc = crt_bulk_create(dss_get_module_info()->dmi_ctx, &sgl, CRT_BULK_RW, &bulk_hdl); if (rc) { - D_ERROR(DF_UOID" pidx %d to peer %d, " - "crt_bulk_create "DF_RC"\n", - DP_UOID(entry->ae_oid), pidx, peer, - DP_RC(rc)); - goto out_rpc; + D_ERROR(DF_UOID " pidx %d to peer %d, crt_bulk_create " DF_RC "\n", + DP_UOID(entry->ae_oid), pidx, peer, DP_RC(rc)); + goto next; } ec_agg_in->ea_bulk = bulk_hdl; if (csummer != NULL) { - rc = daos_csummer_calc_iods(csummer, &sgl, &iod, - NULL, 1, false, - NULL, 0, - &iod_csums); + rc = daos_csummer_calc_iods(csummer, &sgl, &iod, NULL, 1, false, + NULL, 0, &iod_csums); if (rc) { - D_ERROR("daos_csummer_calc_iods " - "failed: "DF_RC"\n", DP_RC(rc)); - goto out; + D_ERROR("daos_csummer_calc_iods failed: " DF_RC "\n", + DP_RC(rc)); + goto next; } D_ASSERT(iod_csums != NULL); - ec_agg_in->ea_iod_csums.ca_arrays = iod_csums; ec_agg_in->ea_iod_csums.ca_count = 1; } @@ -1407,46 +1462,47 @@ agg_peer_update_ult(void *arg) rc = dss_rpc_send(rpc); if (rc) { - D_ERROR(DF_UOID" pidx %d to peer %d, dss_rpc_send " - DF_RC"\n", DP_UOID(entry->ae_oid), pidx, peer, - DP_RC(rc)); - goto out_bulk; + D_ERROR(DF_UOID " pidx %d to parity[%d], dss_rpc_send " DF_RC "\n", + DP_UOID(entry->ae_oid), pidx, peer, DP_RC(rc)); + } else { + crt_req_get_timeout(rpc, &max_delay); + ec_agg_out = crt_reply_get(rpc); + rc = ec_agg_out->ea_status; + if (rc == -DER_OVERLOAD_RETRY) { + enqueue_id = ec_agg_out->ea_comm_out.req_out_enqueue_id; + overloaded = true; + } + D_CDEBUG(rc == 0, DB_TRACE, DLOG_ERR, + "update parity[%d] to %d:%d, status = " DF_RC "\n", peer, + tgt_ep.ep_rank, tgt_ep.ep_tag, DP_RC(rc)); + peer_updated += rc == 0; } - crt_req_get_timeout(rpc, &max_delay); - ec_agg_out = crt_reply_get(rpc); - rc = ec_agg_out->ea_status; - if (bulk_hdl) { +next: + if (bulk_hdl) crt_bulk_free(bulk_hdl); - bulk_hdl = NULL; - } - D_DEBUG(DB_TRACE, "send DAOS_OBJ_RPC_EC_AGGREGATE to %d:%d, peer %d, rc %d\n", - tgt_ep.ep_rank, tgt_ep.ep_tag, peer, rc); - if (csummer != NULL && iod_csums != NULL) + if (rpc) + crt_req_decref(rpc); + if (iod_csums != NULL) daos_csummer_free_ic(csummer, &iod_csums); - crt_req_decref(rpc); rpc = NULL; - if (rc) { - D_ERROR(DF_UOID" pidx %d to peer %d, ea_status " - DF_RC"\n", DP_UOID(entry->ae_oid), pidx, peer, - DP_RC(rc)); - if (rc == -DER_OVERLOAD_RETRY) { - enqueue_id = ec_agg_out->ea_comm_out.req_out_enqueue_id; - dss_sleep(daos_rpc_rand_delay(max_delay) << 10); - goto retry; - } - break; + bulk_hdl = NULL; + iod_csums = NULL; + if (overloaded) { + dss_sleep(daos_rpc_rand_delay(max_delay) << 10); + goto retry; } + peer_rc = peer_rc ?: rc; + if (peer_updated == 0 && peer_rc != 0) + break; /* otherwise continue because no DTX no rollback */ } - -out_bulk: - if (bulk_hdl) - crt_bulk_free(bulk_hdl); -out_rpc: - if (rpc) - crt_req_decref(rpc); out: if (obj) obj_decref(obj); + /* NB: before switching to DTX, any successful parity write is deemed as success + * of all parity shards. + * NB: it's OK if the only remote parity peer failed. + */ + rc = peer_updated > 0 ? 0 : peer_rc; ABT_eventual_set(stripe_ud->asu_eventual, (void *)&rc, sizeof(rc)); } @@ -1456,28 +1512,11 @@ agg_peer_update_ult(void *arg) static int agg_peer_update(struct ec_agg_entry *entry, bool write_parity) { - struct ec_agg_stripe_ud stripe_ud = { 0 }; - struct ec_agg_param *agg_param; - struct pool_target *targets; + struct ec_agg_stripe_ud stripe_ud = {0}; int *status; - struct daos_shard_loc *peer_loc; - uint32_t failed_tgts_cnt = 0; - uint32_t p = ec_age2p(entry); - uint32_t pidx = ec_age2pidx(entry); - uint32_t peer; - int i, tid, rc = 0; - - D_ASSERT(!write_parity || - entry->ae_sgl.sg_iovs[AGG_IOV_PARITY].iov_buf); - agg_param = container_of(entry, struct ec_agg_param, ap_agg_entry); + int tid, rc = 0; - /* If rebuild started, abort it before sending RPC to save conflict window with rebuild - * (see obj_inflight_io_check()). - */ - if (agg_param->ap_pool_info.api_pool->sp_rebuilding > 0) { - D_DEBUG(DB_EPC, DF_UOID" abort as rebuild started\n", DP_UOID(entry->ae_oid)); - return -1; - } + D_ASSERT(!write_parity || entry->ae_sgl.sg_iovs[AGG_IOV_PARITY].iov_buf); rc = agg_get_obj_handle(entry, false); if (rc) { @@ -1485,56 +1524,28 @@ agg_peer_update(struct ec_agg_entry *entry, bool write_parity) return rc; } - rc = pool_map_find_failed_tgts(agg_param->ap_pool_info.api_pool->sp_map, - &targets, &failed_tgts_cnt); - if (rc) { - D_ERROR(DF_UOID" pool_map_find_failed_tgts failed: "DF_RC"\n", - DP_UOID(entry->ae_oid), DP_RC(rc)); - return rc; - } - - if (targets != NULL) { - for (peer = 0; peer < p; peer++) { - if (peer == pidx) - continue; - peer_loc = &entry->ae_peer_pshards[peer]; - for (i = 0; i < failed_tgts_cnt; i++) { - if (peer_loc->sd_rank == DAOS_TGT_IGNORE || - (targets[i].ta_comp.co_rank == peer_loc->sd_rank && - targets[i].ta_comp.co_index == peer_loc->sd_tgt_idx)) { - D_DEBUG(DB_EPC, DF_UOID" peer parity tgt failed rank %d, " - "tgt_idx %d.\n", DP_UOID(entry->ae_oid), - peer_loc->sd_rank, peer_loc->sd_tgt_idx); - D_GOTO(out, rc = -1); - } - } - } - } - stripe_ud.asu_write_par = write_parity; stripe_ud.asu_agg_entry = entry; rc = ABT_eventual_create(sizeof(*status), &stripe_ud.asu_eventual); if (rc != ABT_SUCCESS) { rc = dss_abterr2der(rc); - goto out; + return rc; } tid = dss_get_module_info()->dmi_tgt_id; rc = dss_ult_create(agg_peer_update_ult, &stripe_ud, DSS_XS_IOFW, tid, 0, NULL); if (rc) - goto ev_out; + goto out; rc = ABT_eventual_wait(stripe_ud.asu_eventual, (void **)&status); if (rc != ABT_SUCCESS) { rc = dss_abterr2der(rc); - goto ev_out; + goto out; } if (*status != 0) rc = *status; -ev_out: - ABT_eventual_free(&stripe_ud.asu_eventual); out: - D_FREE(targets); + ABT_eventual_free(&stripe_ud.asu_eventual); return rc; } @@ -1546,8 +1557,7 @@ agg_process_holes_ult(void *arg) struct ec_agg_stripe_ud *stripe_ud = (struct ec_agg_stripe_ud *)arg; daos_iod_t *iod = &stripe_ud->asu_iod; struct ec_agg_entry *entry = stripe_ud->asu_agg_entry; - struct ec_agg_extent *agg_extent; - struct pool_target *targets = NULL; + struct ec_agg_extent *agg_extent; struct ec_agg_param *agg_param; struct obj_ec_rep_in *ec_rep_in = NULL; struct obj_ec_rep_out *ec_rep_out = NULL; @@ -1562,13 +1572,12 @@ agg_process_holes_ult(void *arg) k * len; uint64_t last_ext_end = 0; uint64_t ext_cnt = 0; - uint64_t ext_tot_len = 0; - uint32_t failed_tgts_cnt = 0; + uint64_t ext_tot_len = 0; uint32_t pidx = ec_age2pidx(entry); uint32_t peer; - int i, rc = 0; - uint32_t max_delay = 0; - uint64_t enqueue_id; + int rc = 0, peer_rc = 0; + int peer_updated = 0; + uint32_t max_delay = 0; stripe_ud->asu_valid_hole = false; /* Process extent list to find what to re-replicate -- build recx array @@ -1580,13 +1589,10 @@ agg_process_holes_ult(void *arg) if (agg_extent->ae_hole) stripe_ud->asu_valid_hole = true; if (agg_extent->ae_recx.rx_idx - ss > last_ext_end) { - stripe_ud->asu_recxs[ext_cnt].rx_idx = - ss + last_ext_end; + stripe_ud->asu_recxs[ext_cnt].rx_idx = ss + last_ext_end; stripe_ud->asu_recxs[ext_cnt].rx_nr = - agg_extent->ae_recx.rx_idx - ss - - last_ext_end; - ext_tot_len += - stripe_ud->asu_recxs[ext_cnt++].rx_nr; + agg_extent->ae_recx.rx_idx - ss - last_ext_end; + ext_tot_len += stripe_ud->asu_recxs[ext_cnt++].rx_nr; } last_ext_end = agg_extent->ae_recx.rx_idx + agg_extent->ae_recx.rx_nr - ss; @@ -1597,6 +1603,11 @@ agg_process_holes_ult(void *arg) if (!stripe_ud->asu_valid_hole) goto out; + agg_param = container_of(entry, struct ec_agg_param, ap_agg_entry); + rc = agg_peer_check_avail(agg_param, entry); + if (rc != 0) + goto out; + obj = obj_hdl2ptr(entry->ae_obj_hdl); if (last_ext_end < k * len) { stripe_ud->asu_recxs[ext_cnt].rx_idx = ss + last_ext_end; @@ -1613,7 +1624,6 @@ agg_process_holes_ult(void *arg) entry->ae_sgl.sg_iovs[AGG_IOV_DATA].iov_len = ext_tot_len * entry->ae_rsize; D_ASSERT(entry->ae_sgl.sg_iovs[AGG_IOV_DATA].iov_len <= k * cell_b); - agg_param = container_of(entry, struct ec_agg_param, ap_agg_entry); /* Pull data via dsc_obj_fetch */ if (ext_cnt) { struct daos_csummer *csummer; @@ -1623,6 +1633,7 @@ agg_process_holes_ult(void *arg) NULL, DIOF_FOR_EC_AGG, NULL, NULL); if (rc) { D_ERROR("dsc_obj_fetch failed: "DF_RC"\n", DP_RC(rc)); + D_GOTO(out, rc); goto out; } @@ -1631,6 +1642,7 @@ agg_process_holes_ult(void *arg) NULL, -1, &stripe_ud->asu_iod_csums); if (rc != 0) { D_ERROR("setting up iods csums failed: "DF_RC"\n", DP_RC(rc)); + D_GOTO(out, rc); goto out; } } @@ -1644,39 +1656,22 @@ agg_process_holes_ult(void *arg) if (rc) { D_ERROR(DF_UOID" crt_bulk_create returned: "DF_RC"\n", DP_UOID(entry->ae_oid), DP_RC(rc)); + D_GOTO(out, rc); goto out; } } - rc = pool_map_find_failed_tgts(agg_param->ap_pool_info.api_pool->sp_map, - &targets, &failed_tgts_cnt); - if (rc) { - D_ERROR(DF_UOID" pool_map_find_failed_tgts failed: " - DF_RC"\n", DP_UOID(entry->ae_oid), DP_RC(rc)); - goto out; - } - /* Invoke peer re-replicate */ for (peer = 0; peer < p; peer++) { + uint64_t enqueue_id = 0; uint32_t peer_shard; + bool overloaded; if (pidx == peer) continue; - for (i = 0; targets && i < failed_tgts_cnt; i++) { - if (entry->ae_peer_pshards[peer].sd_rank == DAOS_TGT_IGNORE || - (targets[i].ta_comp.co_rank == entry->ae_peer_pshards[peer].sd_rank && - targets[i].ta_comp.co_index == - entry->ae_peer_pshards[peer].sd_tgt_idx)) { - D_ERROR(DF_UOID" peer %d parity tgt failed\n", - DP_UOID(entry->ae_oid), peer); - rc = -1; - goto out; - } - } - - enqueue_id = 0; retry: + overloaded = false; D_ASSERT(entry->ae_peer_pshards[peer].sd_rank != DAOS_TGT_IGNORE); tgt_ep.ep_rank = entry->ae_peer_pshards[peer].sd_rank; tgt_ep.ep_tag = entry->ae_peer_pshards[peer].sd_tgt_idx; @@ -1685,7 +1680,11 @@ agg_process_holes_ult(void *arg) if (rc) { D_ERROR(DF_UOID" obj_req_create failed: "DF_RC"\n", DP_UOID(entry->ae_oid), DP_RC(rc)); - goto out; + peer_rc = peer_rc ?: rc; + if (peer_updated == 0) + break; + else + continue; } ec_rep_in = crt_req_get(rpc); uuid_copy(ec_rep_in->er_pool_uuid, @@ -1710,38 +1709,43 @@ agg_process_holes_ult(void *arg) ec_rep_in->er_map_ver = agg_param->ap_pool_info.api_pool->sp_map_version; ec_rep_in->er_bulk = bulk_hdl; ec_rep_in->er_comm_in.req_in_enqueue_id = enqueue_id; - crt_req_get_timeout(rpc, &max_delay); rc = dss_rpc_send(rpc); if (rc) { D_ERROR(DF_UOID" peer %d dss_rpc_send failed "DF_RC"\n", DP_UOID(entry->ae_oid), peer, DP_RC(rc)); - goto out; - } - ec_rep_out = crt_reply_get(rpc); - rc = ec_rep_out->er_status; - crt_req_decref(rpc); - rpc = NULL; - if (rc) { - D_ERROR(DF_UOID" peer %d er_status failed "DF_RC"\n", - DP_UOID(entry->ae_oid), peer, DP_RC(rc)); + } else { + crt_req_get_timeout(rpc, &max_delay); + ec_rep_out = crt_reply_get(rpc); + rc = ec_rep_out->er_status; if (rc == -DER_OVERLOAD_RETRY) { enqueue_id = ec_rep_out->er_comm_out.req_out_enqueue_id; - dss_sleep(daos_rpc_rand_delay(max_delay) << 10); - goto retry; + overloaded = true; } - break; + D_CDEBUG(rc == 0, DB_TRACE, DLOG_ERR, + DF_UOID " parity[%d] er_status = " DF_RC "\n", + DP_UOID(entry->ae_oid), peer, DP_RC(rc)); + peer_updated += rc == 0; + } + crt_req_decref(rpc); + rpc = NULL; + if (overloaded) { + dss_sleep(daos_rpc_rand_delay(max_delay) << 10); + goto retry; } + peer_rc = peer_rc ?: rc; + if (peer_updated == 0 && peer_rc != 0) + break; /* otherwise continue because no DTX no rollback */ } - out: - D_FREE(targets); - if (rpc) - crt_req_decref(rpc); if (bulk_hdl) crt_bulk_free(bulk_hdl); if (obj) obj_decref(obj); entry->ae_sgl.sg_nr = AGG_IOV_CNT; + /* NB: before switching to DTX, any successful parity write is deemed as success + * of all parity shards. + */ + rc = peer_updated > 0 ? 0 : peer_rc; ABT_eventual_set(stripe_ud->asu_eventual, (void *)&rc, sizeof(rc)); } @@ -1760,11 +1764,6 @@ agg_process_holes(struct ec_agg_entry *entry) /* If rebuild started, abort it before sending RPC to save conflict window with rebuild * (see obj_inflight_io_check()). */ - if (agg_param->ap_pool_info.api_pool->sp_rebuilding > 0) { - D_DEBUG(DB_EPC, DF_UOID" abort as rebuild started\n", DP_UOID(entry->ae_oid)); - return -1; - } - D_ALLOC_ARRAY(stripe_ud.asu_recxs, entry->ae_cur_stripe.as_extent_cnt + 1); if (stripe_ud.asu_recxs == NULL) { @@ -1788,6 +1787,7 @@ agg_process_holes(struct ec_agg_entry *entry) goto out; } tid = dss_get_module_info()->dmi_tgt_id; + rc = dss_ult_create(agg_process_holes_ult, &stripe_ud, DSS_XS_IOFW, tid, 0, NULL); if (rc) @@ -1842,11 +1842,9 @@ agg_process_holes(struct ec_agg_entry *entry) ABT_eventual_free(&stripe_ud.asu_eventual); out: - D_FREE(stripe_ud.asu_recxs); - daos_csummer_free_ic(stripe_ud.asu_csummer, &stripe_ud.asu_iod_csums); + daos_csummer_free_ic(ec_agg_param2csummer(agg_param), &stripe_ud.asu_iod_csums); D_FREE(stripe_ud.asu_csum_iov.iov_buf); - daos_csummer_destroy(&stripe_ud.asu_csummer); - + D_FREE(stripe_ud.asu_recxs); return rc; } @@ -1862,6 +1860,15 @@ agg_process_stripe(struct ec_agg_param *agg_param, struct ec_agg_entry *entry) bool write_parity = true; int rc = 0; + /* avoid race between EC aggregation and rebuild scanner */ + agg_param->ap_cont->sc_ec_agg_updates++; + if (ds_pool_is_rebuilding(agg_param->ap_cont->sc_pool->spc_pool)) { + D_DEBUG(DB_EPC, DF_UOID " abort as rebuild started\n", DP_UOID(entry->ae_oid)); + update_vos = false; + rc = -1; + goto out; + } + if (DAOS_FAIL_CHECK(DAOS_FORCE_EC_AGG_FAIL)) D_GOTO(out, rc = -DER_DATA_LOSS); @@ -1939,12 +1946,13 @@ agg_process_stripe(struct ec_agg_param *agg_param, struct ec_agg_entry *entry) if (ec_age_with_hole(entry)) { entry->ae_process_partial = 0; rc = agg_process_holes(entry); - if (rc != 0 || !entry->ae_process_partial) - goto clear_exts; + if (rc != 0 || !entry->ae_process_partial) { + update_vos = false; + goto out; + } } rc = agg_process_partial_stripe(entry); - out: if (update_vos && rc == 0) { if (ec_age2p(entry) > 1) { @@ -1962,8 +1970,7 @@ agg_process_stripe(struct ec_agg_param *agg_param, struct ec_agg_entry *entry) write_parity); } } - -clear_exts: + agg_param->ap_cont->sc_ec_agg_updates--; agg_clear_extents(entry); return rc; } @@ -2280,7 +2287,7 @@ ec_aggregate_yield(struct ec_agg_param *agg_param) { int rc; - if (agg_param->ap_pool_info.api_pool->sp_rebuilding > 0) { + if (ds_pool_is_rebuilding(agg_param->ap_pool_info.api_pool)) { D_INFO(DF_UUID": abort ec aggregation, sp_rebuilding %d\n", DP_UUID(agg_param->ap_pool_info.api_pool->sp_uuid), agg_param->ap_pool_info.api_pool->sp_rebuilding); @@ -2486,7 +2493,7 @@ agg_iterate_pre_cb(daos_handle_t ih, vos_iter_entry_t *entry, /* If rebuild started, abort it to save conflict window with rebuild * (see obj_inflight_io_check()). */ - if (agg_param->ap_pool_info.api_pool->sp_rebuilding > 0) { + if (ds_pool_is_rebuilding(agg_param->ap_pool_info.api_pool)) { D_INFO(DF_CONT" abort as rebuild started, sp_rebuilding %d\n", DP_CONT(agg_param->ap_pool_info.api_pool_uuid, agg_param->ap_pool_info.api_cont_uuid), @@ -2647,6 +2654,7 @@ ec_agg_param_init(struct ds_cont_child *cont, struct agg_param *param) uuid_copy(info->api_cont_uuid, cont->sc_uuid); info->api_pool = cont->sc_pool->spc_pool; + agg_param->ap_cont = cont; agg_param->ap_cont_handle = cont->sc_hdl; agg_param->ap_yield_func = agg_rate_ctl; agg_param->ap_yield_arg = param; @@ -2787,12 +2795,7 @@ cont_ec_aggregate_cb(struct ds_cont_child *cont, daos_epoch_range_t *epr, agg_clear_extents(&ec_agg_param->ap_agg_entry); agg_reset_entry(&ec_agg_param->ap_agg_entry, NULL, NULL); - if (daos_handle_is_valid(ec_agg_param->ap_agg_entry.ae_obj_hdl)) { - dsc_obj_close(ec_agg_param->ap_agg_entry.ae_obj_hdl); - ec_agg_param->ap_agg_entry.ae_obj_hdl = DAOS_HDL_INVAL; - } - - if (rc == -DER_BUSY && cont->sc_pool->spc_pool->sp_rebuilding == 0) { + if (rc == -DER_BUSY && !ds_pool_is_rebuilding(cont->sc_pool->spc_pool)) { /** Hit an object conflict VOS aggregation or discard. Rather than exiting, let's * yield and try again. */ @@ -2811,7 +2814,7 @@ cont_ec_aggregate_cb(struct ds_cont_child *cont, daos_epoch_range_t *epr, /* clear the flag before next turn's cont_aggregate_runnable(), to save conflict * window with rebuild (see obj_inflight_io_check()). */ - if (cont->sc_pool->spc_pool->sp_rebuilding > 0) + if (ds_pool_is_rebuilding(cont->sc_pool->spc_pool)) cont->sc_ec_agg_active = 0; if (rc == 0) { diff --git a/src/object/srv_obj.c b/src/object/srv_obj.c index e883f5cb7e1..68512bb653f 100644 --- a/src/object/srv_obj.c +++ b/src/object/srv_obj.c @@ -2423,7 +2423,8 @@ obj_inflight_io_check(struct ds_cont_child *child, uint32_t opc, uint32_t rpc_map_ver, uint32_t flags) { if (opc == DAOS_OBJ_RPC_ENUMERATE && flags & ORF_FOR_MIGRATION) { - if (child->sc_ec_agg_active) { + /* EC aggregation is still inflight, rebuild should wait until it's paused */ + if (ds_cont_child_ec_aggregating(child)) { D_ERROR(DF_CONT" ec aggregate still active, rebuilding %d\n", DP_CONT(child->sc_pool->spc_uuid, child->sc_uuid), child->sc_pool->spc_pool->sp_rebuilding); @@ -2539,6 +2540,8 @@ ds_obj_ec_rep_handler(crt_rpc_t *rpc) } D_ASSERT(ioc.ioc_coc != NULL); + ioc.ioc_coc->sc_ec_agg_updates++; + dkey = (daos_key_t *)&oer->er_dkey; iod = (daos_iod_t *)&oer->er_iod; if (iod->iod_nr == 0) /* nothing to replicate, directly remove parity */ @@ -2549,7 +2552,7 @@ ds_obj_ec_rep_handler(crt_rpc_t *rpc) if (rc) { D_ERROR(DF_UOID" Update begin failed: "DF_RC"\n", DP_UOID(oer->er_oid), DP_RC(rc)); - goto out; + goto out_agg; } biod = vos_ioh2desc(ioh); rc = bio_iod_prep(biod, BIO_CHK_TYPE_IO, rpc->cr_ctx, CRT_BULK_RW); @@ -2573,13 +2576,15 @@ ds_obj_ec_rep_handler(crt_rpc_t *rpc) if (rc) { D_ERROR(DF_UOID " vos_update_end failed: " DF_RC "\n", DP_UOID(oer->er_oid), DP_RC(rc)); - goto out; + goto out_agg; } remove_parity: recx.rx_nr = obj_ioc2ec_cs(&ioc); recx.rx_idx = (oer->er_stripenum * recx.rx_nr) | PARITY_INDICATOR; rc = vos_obj_array_remove(ioc.ioc_coc->sc_hdl, oer->er_oid, &oer->er_epoch_range, dkey, &iod->iod_name, &recx); +out_agg: + ioc.ioc_coc->sc_ec_agg_updates--; out: obj_rw_reply(rpc, rc, 0, false, &ioc); obj_ioc_end(&ioc, rc); @@ -2619,6 +2624,8 @@ ds_obj_ec_agg_handler(crt_rpc_t *rpc) } D_ASSERT(ioc.ioc_coc != NULL); + ioc.ioc_coc->sc_ec_agg_updates++; + dkey = (daos_key_t *)&oea->ea_dkey; if (parity_bulk != CRT_BULK_NULL) { rc = vos_update_begin(ioc.ioc_coc->sc_hdl, oea->ea_oid, oea->ea_epoch_range.epr_hi, @@ -2626,7 +2633,7 @@ ds_obj_ec_agg_handler(crt_rpc_t *rpc) if (rc) { D_ERROR(DF_UOID" Update begin failed: "DF_RC"\n", DP_UOID(oea->ea_oid), DP_RC(rc)); - goto out; + goto out_agg; } biod = vos_ioh2desc(ioh); rc = bio_iod_prep(biod, BIO_CHK_TYPE_IO, rpc->cr_ctx, @@ -2659,7 +2666,7 @@ ds_obj_ec_agg_handler(crt_rpc_t *rpc) } else { D_ERROR(DF_UOID " vos_update_end failed: " DF_RC "\n", DP_UOID(oea->ea_oid), DP_RC(rc)); - D_GOTO(out, rc); + D_GOTO(out_agg, rc); } } } @@ -2676,6 +2683,8 @@ ds_obj_ec_agg_handler(crt_rpc_t *rpc) if (rc1) D_ERROR(DF_UOID ": array_remove failed: " DF_RC "\n", DP_UOID(oea->ea_oid), DP_RC(rc1)); +out_agg: + ioc.ioc_coc->sc_ec_agg_updates--; out: obj_rw_reply(rpc, rc, 0, false, &ioc); obj_ioc_end(&ioc, rc); @@ -3363,8 +3372,14 @@ obj_local_enum(struct obj_io_context *ioc, crt_rpc_t *rpc, goto failed; } - if (oei->oei_flags & ORF_FOR_MIGRATION) + if (oei->oei_flags & ORF_FOR_MIGRATION) { + /* just in case ds_pool::sp_rebuilding is not set, pause my local EC aggregation + * by setting this flag. + * NB: it's a lockess write to shared data structure and it's harmless. + */ + ioc->ioc_coc->sc_pool->spc_pool->sp_rebuild_scan = 1; flags = DTX_FOR_MIGRATION; + } rc = dtx_begin(ioc->ioc_vos_coh, &oei->oei_dti, &epoch, 0, oei->oei_map_ver, &oei->oei_oid, NULL, 0, flags, diff --git a/src/object/srv_obj_migrate.c b/src/object/srv_obj_migrate.c index 2a91c7f1722..1d5715260eb 100644 --- a/src/object/srv_obj_migrate.c +++ b/src/object/srv_obj_migrate.c @@ -2865,6 +2865,7 @@ migrate_one_epoch_object(daos_epoch_range_t *epr, struct migrate_pool_tls *tls, uint32_t minimum_nr; uint32_t enum_flags; uint32_t num; + int waited = 0; int rc = 0; D_DEBUG(DB_REBUILD, "migrate obj "DF_UOID" for shard %u eph " @@ -3028,7 +3029,10 @@ migrate_one_epoch_object(daos_epoch_range_t *epr, struct migrate_pool_tls *tls, /* -DER_UPDATE_AGAIN means the remote target does not parse EC * aggregation yet, so let's retry. */ - D_DEBUG(DB_REBUILD, DF_UOID "retry with %d \n", DP_UOID(arg->oid), rc); + waited++; + dss_sleep(5000); + D_DEBUG(DB_REBUILD, DF_UOID "retry %d secs with %d \n", DP_UOID(arg->oid), + waited * 5, rc); rc = 0; continue; } else if (rc) { diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 5e066cbf0d5..57df9b94590 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -7308,10 +7308,11 @@ ds_pool_tgt_add_in(uuid_t pool_uuid, struct pool_target_id_list *list) } int -ds_pool_tgt_finish_rebuild(uuid_t pool_uuid, struct pool_target_id_list *list) +ds_pool_tgt_finish_rebuild(uuid_t pool_uuid, struct pool_target_id_list *list, + uint32_t *reclaim_ver) { - return pool_update_map_internal(pool_uuid, MAP_FINISH_REBUILD, false, list, - NULL, NULL, NULL, NULL, NULL, NULL); + return pool_update_map_internal(pool_uuid, MAP_FINISH_REBUILD, false, list, NULL, NULL, + NULL, NULL, reclaim_ver, NULL); } int diff --git a/src/rebuild/rebuild_internal.h b/src/rebuild/rebuild_internal.h index 0510a32f0f8..4eb7f8ef2b5 100644 --- a/src/rebuild/rebuild_internal.h +++ b/src/rebuild/rebuild_internal.h @@ -217,6 +217,7 @@ struct rebuild_global { /* Per target structure to track the rebuild status */ extern struct rebuild_global rebuild_gst; +extern unsigned int rebuild_wait_ec_pause; struct rebuild_task { d_list_t dst_list; diff --git a/src/rebuild/scan.c b/src/rebuild/scan.c index a41d6e253a8..1b4de302cc6 100644 --- a/src/rebuild/scan.c +++ b/src/rebuild/scan.c @@ -985,7 +985,7 @@ rebuild_scanner(void *data) { struct rebuild_scan_arg arg = { 0 }; struct rebuild_tgt_pool_tracker *rpt = data; - struct ds_pool_child *child; + struct ds_pool_child *child = NULL; struct rebuild_pool_tls *tls; vos_iter_param_t param = { 0 }; struct vos_iter_anchors anchor = { 0 }; @@ -998,6 +998,12 @@ rebuild_scanner(void *data) if (tls == NULL) return 0; + child = ds_pool_child_lookup(rpt->rt_pool_uuid); + if (child == NULL) + D_GOTO(out, rc = -DER_NONEXIST); + + ds_cont_child_wait_ec_agg_pause(child, rebuild_wait_ec_pause); + /* There maybe orphan DTX entries after DTX resync, let's cleanup before rebuild scan. */ rc = dtx_cleanup_orphan(rpt->rt_pool_uuid, rpt->rt_pool->sp_dtx_resync_version); if (rc != 0) @@ -1038,10 +1044,6 @@ rebuild_scanner(void *data) } } - child = ds_pool_child_lookup(rpt->rt_pool_uuid); - if (child == NULL) - D_GOTO(out, rc = -DER_NONEXIST); - param.ip_hdl = child->spc_hdl; param.ip_flags = VOS_IT_FOR_MIGRATION; arg.rpt = rpt; @@ -1049,11 +1051,11 @@ rebuild_scanner(void *data) rc = vos_iterate(¶m, VOS_ITER_COUUID, false, &anchor, rebuild_container_scan_cb, NULL, &arg, NULL); if (rc < 0) - D_GOTO(put, rc); + D_GOTO(out, rc); rc = 0; /* rc might be 1 if rebuild is aborted */ -put: - ds_pool_child_put(child); out: + if (child) + ds_pool_child_put(child); tls->rebuild_pool_scan_done = 1; if (ult_send != ABT_THREAD_NULL) ABT_thread_free(&ult_send); diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c index 817dda33ac4..c0ebf709453 100644 --- a/src/rebuild/srv.c +++ b/src/rebuild/srv.c @@ -25,6 +25,7 @@ #define RBLD_CHECK_INTV 2000 /* milliseconds interval to check*/ struct rebuild_global rebuild_gst; +unsigned int rebuild_wait_ec_pause = 0; struct pool_map * rebuild_pool_map_get(struct ds_pool *pool) @@ -1472,7 +1473,8 @@ retry_rebuild_task(struct rebuild_task *task, int error, daos_rebuild_opc_t *opc static int rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, - struct rebuild_global_pool_tracker *rgt, int ret) + struct rebuild_global_pool_tracker *rgt, uint32_t obj_reclaim_ver, + int ret) { int rc = 0; int rc1; @@ -1561,9 +1563,12 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, D_DEBUG(DB_REBUILD, DF_UUID" opc %u/%u, error %d, schedule RECLAIM.\n", DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, task->dst_map_ver, rgt->rgt_status.rs_errno); - rc = ds_rebuild_schedule(pool, task->dst_map_ver, rgt->rgt_reclaim_epoch, - task->dst_new_layout_version, - &task->dst_tgts, RB_OP_RECLAIM, 5); + + /* NB: if pool map remains unchanged, ds_pool_tgt_finish_rebuild sets it to 0 */ + obj_reclaim_ver = obj_reclaim_ver > 0 ? obj_reclaim_ver : task->dst_map_ver; + rc = ds_rebuild_schedule(pool, obj_reclaim_ver, rgt->rgt_reclaim_epoch, + task->dst_new_layout_version, &task->dst_tgts, + RB_OP_RECLAIM, 5); if (rc != 0) D_ERROR("reschedule reclaim, "DF_UUID" failed: "DF_RC"\n", DP_UUID(task->dst_pool_uuid), DP_RC(rc)); @@ -1596,6 +1601,7 @@ rebuild_task_ult(void *arg) struct rebuild_global_pool_tracker *rgt = NULL; d_rank_t myrank; uint64_t cur_ts = 0; + uint32_t obj_reclaim_ver = 0; int rc; cur_ts = daos_gettime_coarse(); @@ -1724,7 +1730,8 @@ rebuild_task_ult(void *arg) goto iv_stop; if (task->dst_rebuild_op == RB_OP_REBUILD) - rc = ds_pool_tgt_finish_rebuild(pool->sp_uuid, &task->dst_tgts); + rc = ds_pool_tgt_finish_rebuild(pool->sp_uuid, &task->dst_tgts, + &obj_reclaim_ver); D_INFO("finish rebuild %d of " DF_UUID " "DF_RC"\n", task->dst_tgts.pti_ids[0].pti_id, DP_UUID(task->dst_pool_uuid), DP_RC(rc)); } @@ -1748,7 +1755,7 @@ rebuild_task_ult(void *arg) } try_reschedule: - rebuild_task_complete_schedule(task, pool, rgt, rc); + rebuild_task_complete_schedule(task, pool, rgt, obj_reclaim_ver, rc); output: rc = rebuild_notify_ras_end(&task->dst_pool_uuid, task->dst_map_ver, RB_OP_STR(task->dst_rebuild_op), @@ -2282,7 +2289,6 @@ rebuild_fini_one(void *arg) } ds_pool_child_put(dpc); - return 0; } @@ -2297,6 +2303,7 @@ rebuild_tgt_fini(struct rebuild_tgt_pool_tracker *rpt) D_ASSERT(rpt->rt_pool->sp_rebuilding > 0); rpt->rt_pool->sp_rebuilding--; + rpt->rt_pool->sp_rebuild_scan = 0; ABT_mutex_lock(rpt->rt_lock); ABT_cond_signal(rpt->rt_global_dtx_wait_cond); @@ -2737,6 +2744,10 @@ init(void) { int rc; + rc = d_getenv_uint(REBUILD_WAIT_EC_PAUSE_ENV, &rebuild_wait_ec_pause); + if (rc == 0) + D_DEBUG(DB_REBUILD, "Set REBUILD_WAIT_EC_PAUSE to %u\n", rebuild_wait_ec_pause); + D_INIT_LIST_HEAD(&rebuild_gst.rg_tgt_tracker_list); D_INIT_LIST_HEAD(&rebuild_gst.rg_global_tracker_list); D_INIT_LIST_HEAD(&rebuild_gst.rg_completed_list); diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index 032a8183659..6ed19436d97 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -436,6 +436,7 @@ class EngineYamlParameters(YamlParameters): "common": [ "D_LOG_FILE_APPEND_PID=1", "DAOS_POOL_RF=4", + "DAOS_REBUILD_WAIT_EC_PAUSE=1", "CRT_EVENT_DELAY=1", # pylint: disable-next=fixme # FIXME disable space cache since some tests need to verify instant pool space diff --git a/src/tests/suite/daos_container.c b/src/tests/suite/daos_container.c index 243a829c317..d7151722061 100644 --- a/src/tests/suite/daos_container.c +++ b/src/tests/suite/daos_container.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -2676,7 +2677,9 @@ co_rf_simple(void **state) daos_prop_val_2_co_status(entry->dpe_val, &stat); assert_int_equal(stat.dcs_status, DAOS_PROP_CO_HEALTHY); - test_set_engine_fail_loc(arg, CRT_NO_RANK, DAOS_REBUILD_DELAY | DAOS_FAIL_ALWAYS); + /* Hang the rebuild */ + test_set_engine_fail_loc(arg, CRT_NO_RANK, + DAOS_REBUILD_TGT_REBUILD_HANG | DAOS_FAIL_ALWAYS); if (arg->myrank == 0) { unsigned int ranks[2]; @@ -2698,7 +2701,8 @@ co_rf_simple(void **state) assert_rc_equal(rc, 0); /* Hang the rebuild */ - test_set_engine_fail_loc(arg, CRT_NO_RANK, DAOS_REBUILD_TGT_REBUILD_HANG | DAOS_FAIL_ALWAYS); + test_set_engine_fail_loc(arg, CRT_NO_RANK, + DAOS_REBUILD_TGT_REBUILD_HANG | DAOS_FAIL_ALWAYS); /* IO testing */ io_oid = daos_test_oid_gen(arg->coh, OC_RP_4G1, 0, 0, arg->myrank); rc = daos_obj_open(arg->coh, io_oid, DAOS_OO_RW, &io_oh, NULL); From e8544113e74267495b8647e2ff3c193730774d5b Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 17 Oct 2025 16:44:36 -0400 Subject: [PATCH 05/43] DAOS-18050 build: Tag 2.6.4 rc3 (#16994) Tag third release candidate for 2.6.4. Signed-off-by: Phil Henderson --- TAG | 2 +- debian/changelog | 6 ++++++ utils/rpms/daos.spec | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/TAG b/TAG index ca1fdbfd4c1..251785bc13e 100644 --- a/TAG +++ b/TAG @@ -1 +1 @@ -2.6.4-rc2 +2.6.4-rc3 \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 5e918af031e..c8d33052123 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +daos (2.6.4-4) unstable; urgency=medium + [ Phillip Henderson ] + * Third release candidate for 2.6.4 + + -- Phillip Henderson Fri, 17 Oct 2025 15:09:00 -0500 + daos (2.6.4-2) unstable; urgency=medium [ Dalton Bohning ] * Second release candidate for 2.6.4 diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 5f88446f481..394091a2172 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -23,7 +23,7 @@ Name: daos Version: 2.6.4 -Release: 3%{?relval}%{?dist} +Release: 4%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -633,6 +633,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Fri Oct 17 2025 Phillip Henderson 2.6.4-4 +- Third release candidate for 2.6.4 + * Wed Sep 10 2025 Jeff Olivier 2.6.4-3 - Ensure daos-server installs spdk From 5fde7ff21c2fea56d7e90941ac090cb29e8903c1 Mon Sep 17 00:00:00 2001 From: Liu Xuezhao Date: Wed, 22 Oct 2025 07:52:03 +0800 Subject: [PATCH 06/43] DAOS-18114 container: add mutex to protect cs_ec_agg_list (#17000) Signed-off-by: Xuezhao Liu --- src/container/srv_container.c | 15 +++++++++++++-- src/container/srv_internal.h | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/container/srv_container.c b/src/container/srv_container.c index 0971752888e..42f2a43bd8f 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -101,10 +101,16 @@ cont_svc_init(struct cont_svc *svc, const uuid_t pool_uuid, uint64_t id, D_GOTO(err, rc = dss_abterr2der(rc)); } + rc = ABT_mutex_create(&svc->cs_ec_agg_mutex); + if (rc != ABT_SUCCESS) { + D_ERROR("failed to create cs_ec_agg_mutex: %d\n", rc); + D_GOTO(err_rwlock, rc = dss_abterr2der(rc)); + } + /* cs_root */ rc = rdb_path_init(&svc->cs_root); if (rc != 0) - goto err_lock; + goto err_mutex; rc = rdb_path_push(&svc->cs_root, &rdb_path_root_key); if (rc != 0) goto err_root; @@ -143,7 +149,9 @@ cont_svc_init(struct cont_svc *svc, const uuid_t pool_uuid, uint64_t id, rdb_path_fini(&svc->cs_uuids); err_root: rdb_path_fini(&svc->cs_root); -err_lock: +err_mutex: + ABT_mutex_free(&svc->cs_ec_agg_mutex); +err_rwlock: ABT_rwlock_free(&svc->cs_lock); err: return rc; @@ -157,6 +165,7 @@ cont_svc_fini(struct cont_svc *svc) rdb_path_fini(&svc->cs_uuids); rdb_path_fini(&svc->cs_root); ABT_rwlock_free(&svc->cs_lock); + ABT_mutex_free(&svc->cs_ec_agg_mutex); } int @@ -1986,6 +1995,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) return; } + ABT_mutex_lock(svc->cs_ec_agg_mutex); d_list_for_each_entry_safe(ec_agg, tmp, &svc->cs_ec_agg_list, ea_list) { if (ec_agg->ea_deleted) { d_list_del(&ec_agg->ea_list); @@ -2068,6 +2078,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) } ec_agg->ea_current_eph = min_eph; } + ABT_mutex_unlock(svc->cs_ec_agg_mutex); map_ranks_fini(&fail_ranks); } diff --git a/src/container/srv_internal.h b/src/container/srv_internal.h index bb257a73212..aaae6d772bf 100644 --- a/src/container/srv_internal.h +++ b/src/container/srv_internal.h @@ -93,6 +93,7 @@ struct cont_svc { /* Manage the EC aggregation epoch */ struct sched_request *cs_ec_leader_ephs_req; d_list_t cs_ec_agg_list; /* link cont_ec_agg */ + ABT_mutex cs_ec_agg_mutex; /* protect cs_ec_agg_list */ }; /* Container descriptor */ From 35c1183446c8f68e8c4d0172fd8f789a9026d7d5 Mon Sep 17 00:00:00 2001 From: Liu Xuezhao Date: Thu, 23 Oct 2025 03:14:36 +0800 Subject: [PATCH 07/43] DAOS-18110 test: refine RF 2 test cases (#17008) remove the inflight update test to avoid DER_UPDATE_AGAIN trouble, it is not easy to control the timing for that case. Signed-off-by: Xuezhao Liu --- src/tests/suite/daos_container.c | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/tests/suite/daos_container.c b/src/tests/suite/daos_container.c index d7151722061..8f61d42c63c 100644 --- a/src/tests/suite/daos_container.c +++ b/src/tests/suite/daos_container.c @@ -2677,9 +2677,7 @@ co_rf_simple(void **state) daos_prop_val_2_co_status(entry->dpe_val, &stat); assert_int_equal(stat.dcs_status, DAOS_PROP_CO_HEALTHY); - /* Hang the rebuild */ - test_set_engine_fail_loc(arg, CRT_NO_RANK, - DAOS_REBUILD_TGT_REBUILD_HANG | DAOS_FAIL_ALWAYS); + test_set_engine_fail_loc(arg, CRT_NO_RANK, DAOS_REBUILD_DELAY | DAOS_FAIL_ALWAYS); if (arg->myrank == 0) { unsigned int ranks[2]; @@ -2720,11 +2718,7 @@ co_rf_simple(void **state) iod.iod_size = 1; iod.iod_nr = 1; iod.iod_recxs = &recx; - iod.iod_type = DAOS_IOD_ARRAY; - print_message("obj update should success before RF broken\n"); - rc = daos_obj_update(io_oh, DAOS_TX_NONE, 0, &dkey, 1, &iod, &sgl, - NULL); - assert_rc_equal(rc, 0); + iod.iod_type = DAOS_IOD_ARRAY; if (arg->myrank == 0) { arg->no_rebuild = 1; @@ -3077,7 +3071,7 @@ co_redun_lvl(void **state) goto out; rc = ranks_on_same_node(poolmap, 7, ranks); assert_rc_equal(rc, 0); - for (i = 5; i > 0; i++) { + for (i = 5; i > 0; i--) { if (i != ranks[0] && i != ranks[1]) { ranks[2] = i; break; @@ -3152,16 +3146,12 @@ co_redun_lvl(void **state) assert_rc_equal(rc, -DER_INVAL); } - test_set_engine_fail_loc(arg, CRT_NO_RANK, DAOS_REBUILD_TGT_REBUILD_HANG | DAOS_FAIL_ALWAYS); - print_message("obj update should success before RF broken\n"); + test_set_engine_fail_loc(arg, CRT_NO_RANK, + DAOS_REBUILD_TGT_REBUILD_HANG | DAOS_FAIL_ALWAYS); io_oid = daos_test_oid_gen(arg->coh, OC_EC_2P2G1, 0, 0, arg->myrank); rc = daos_obj_open(arg->coh, io_oid, DAOS_OO_RW, &io_oh, NULL); assert_rc_equal(rc, 0); - rc = daos_obj_update(io_oh, DAOS_TX_NONE, 0, &dkey, 1, &iod, &sgl, - NULL); - assert_rc_equal(rc, 0); - /* exclude one more rank on another NODE dom */ if (arg->myrank == 0) rebuild_single_pool_rank(arg, ranks[2], false); From 06acdf464039705f239ef3252e7f068b4fc2a2d1 Mon Sep 17 00:00:00 2001 From: Wang Shilong Date: Thu, 23 Oct 2025 23:12:40 +0800 Subject: [PATCH 08/43] DAOS-18127 container: use VERSION 1 for backward compatibility in container creation (#17018) New container creation by clients will specify the object version. If not specified (dpe_val == 0), it indicates a client from before DAOS 2.6.4, so use VERSION 1 for backward compatibility instead of inheriting from pool object version. Signed-off-by: Wang Shilong --- src/container/srv_container.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/container/srv_container.c b/src/container/srv_container.c index 42f2a43bd8f..b2319271fb6 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -714,7 +714,6 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl, D_ERROR("container global %u version could be not set\n", entry->dpe_type); return -DER_INVAL; case DAOS_PROP_CO_OBJ_VERSION: - /* this is a walkaround for 2.6 only */ entry_def->dpe_val = entry->dpe_val; break; default: @@ -769,10 +768,14 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl, if (entry_def) entry_def->dpe_val = pool_hdl->sph_global_ver; - /* inherit object version from pool*/ + /* + * New container creation by clients will specify the object version. + * If not specified (dpe_val == 0), it indicates a client from before + * DAOS 2.6.4, so use VERSION 1 for backward compatibility. + */ entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_OBJ_VERSION); if (entry_def && entry_def->dpe_val == 0) - entry_def->dpe_val = pool_hdl->sph_obj_ver; + entry_def->dpe_val = DAOS_POOL_OBJ_VERSION_1; /* for new container set HEALTHY status with current pm ver */ entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_STATUS); From 0d3f0bfdee0fcc2447ea22874fbdf234dbd4d58c Mon Sep 17 00:00:00 2001 From: Ryon Jensen Date: Thu, 23 Oct 2025 14:16:22 -0600 Subject: [PATCH 09/43] DAOS-18121 build: Update dependency versions (#17006) - Bump libfabric version >= 1.22.0-4 - Bump mercury version >= 2.4.0-7 - Bump argobots version >= 1.2-3 - Bump libisal version >= 2.31.1-7 - Bump libisal_crypto version >= 2.24.0-3 - Bump pmemobj1 >= 2.1.0-3 - Bump pmemobj >= 2.1.0-6 Signed-off-by: Ryon Jensen --- .../post_provision_config_common_functions.sh | 2 +- utils/rpms/daos.spec | 25 +++++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/ci/provisioning/post_provision_config_common_functions.sh b/ci/provisioning/post_provision_config_common_functions.sh index a04deba1914..dbf02072bb8 100755 --- a/ci/provisioning/post_provision_config_common_functions.sh +++ b/ci/provisioning/post_provision_config_common_functions.sh @@ -415,7 +415,7 @@ post_provision_config_nodes() { # shellcheck disable=SC2001 if [ ${#inst_rpms[@]} -gt 0 ]; then - if ! retry_dnf 360 install "${inst_rpms[@]/%/${DAOS_VERSION:-}}"; then + if ! retry_dnf 360 install "${inst_rpms[@]/%/${DAOS_VERSION:-}}" --setopt=allow_vendor_change=True; then rc=${PIPESTATUS[0]} dump_repos echo "Failed to install packages" diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 394091a2172..471ba7bc306 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -10,9 +10,9 @@ %else %global daos_build_args client test %endif -%global mercury_version 2.4 -%global libfabric_version 1.15.1-1 -%global argobots_version 1.2 +%global mercury_version 2.4.0-7 +%global libfabric_version 1.22.0-4 +%global argobots_version 1.2-3 %global __python %{__python3} %if (0%{?rhel} >= 8) @@ -81,10 +81,10 @@ BuildRequires: spdk-devel >= 22.01.2 %endif %if (0%{?rhel} >= 8) BuildRequires: isa-l-devel -BuildRequires: libisa-l_crypto-devel +BuildRequires: libisa-l_crypto-devel >= 2.24.0-1 %else -BuildRequires: libisal-devel -BuildRequires: libisal_crypto-devel +BuildRequires: libisal-devel >= 2.31.1-7 +BuildRequires: libisal_crypto-devel >= 2.24.0-3 %endif BuildRequires: openssl-devel BuildRequires: libevent-devel @@ -156,11 +156,11 @@ Requires: ndctl # needed to set PMem configuration goals in BIOS through control-plane %if (0%{?suse_version} >= 1500) Requires: ipmctl >= 03.00.00.0423 -Requires: libpmemobj1 >= 2.1.0-1.suse1500 +Requires: libpmemobj1 >= 2.1.0-3.suse1500 Requires: libfabric1 >= %{libfabric_version} %else Requires: ipmctl >= 03.00.00.0468 -Requires: libpmemobj >= 2.1.0-1%{?dist} +Requires: libpmemobj >= 2.1.0-6%{?dist} %endif Requires: libfabric >= %{libfabric_version} Requires: mercury >= %{mercury_version} @@ -633,6 +633,15 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Tue Oct 21 2025 Ryon Jensen 2.6.4-5 +- Bump libfabric version >= 1.22.0-4 +- Bump mercury version >= 2.4.0-7 +- Bump argobots version >= 1.2-3 +- Bump libisal version >= 2.31.1-7 +- Bump libisal_crypto version >= 2.24.0-3 +- Bump pmemobj1 >= 2.1.0-3 +- Bump pmemobj >= 2.1.0-6 + * Fri Oct 17 2025 Phillip Henderson 2.6.4-4 - Third release candidate for 2.6.4 From 70bcf195883285cc04f131fb082609d49993ffd9 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 23 Oct 2025 22:41:43 -0400 Subject: [PATCH 10/43] DAOS-18129 build: Tag 2.6.4 rc4 (#17024) Tag fourth release candidate for 2.6.4 Signed-off-by: Phil Henderson --- TAG | 2 +- debian/changelog | 6 ++++++ utils/rpms/daos.spec | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/TAG b/TAG index 251785bc13e..175c14e4440 100644 --- a/TAG +++ b/TAG @@ -1 +1 @@ -2.6.4-rc3 \ No newline at end of file +2.6.4-rc4 diff --git a/debian/changelog b/debian/changelog index c8d33052123..cb9eb52f0a2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +daos (2.6.4-6) unstable; urgency=medium + [ Phillip Henderson ] + * Fourth release candidate for 2.6.4 + + -- Phillip Henderson Thu, 23 Oct 2025 16:23:00 -0500 + daos (2.6.4-4) unstable; urgency=medium [ Phillip Henderson ] * Third release candidate for 2.6.4 diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 471ba7bc306..38a99570eef 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -23,7 +23,7 @@ Name: daos Version: 2.6.4 -Release: 4%{?relval}%{?dist} +Release: 6%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -633,6 +633,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Thu Oct 23 2025 Phillip Henderson 2.6.4-6 +- Fourth release candidate for 2.6.4 + * Tue Oct 21 2025 Ryon Jensen 2.6.4-5 - Bump libfabric version >= 1.22.0-4 - Bump mercury version >= 2.4.0-7 From 937eafab5a1fb54ecef0907c33c3ceb2e365cae3 Mon Sep 17 00:00:00 2001 From: Liang Zhen Date: Sat, 1 Nov 2025 02:10:45 +0800 Subject: [PATCH 11/43] DAOS-9355 doc: DAOS 2.6.4 release notes (#17047) Signed-off-by: Liang Zhen Co-authored-by: Mohamad Chaarawi --- docs/release/release_notes.md | 61 +++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/docs/release/release_notes.md b/docs/release/release_notes.md index 7c4d2beb743..8a932941e2b 100644 --- a/docs/release/release_notes.md +++ b/docs/release/release_notes.md @@ -2,6 +2,67 @@ We are pleased to announce the release of DAOS version 2.6. +## DAOS Version 2.6.4 (2025-10-29) + +The DAOS 2.6.4 release includes the daos-2.6.4-6 RPM packages and their prerequisites. +It contains the following updates on top of DAOS 2.6.3: + +* Libfabric has been updated to version 1.22.0-4. +* Mercury has been upgraded to version 2.4.0-7. +* Argobots has been upgraded to version 1.2-3. +* Libisal has been upgraded to version 2.31.1-7. +* pmemobj has been upgraded to version 2.1.0-6. + +### Bug fixes and improvements + +This release includes fixes and improvements, including: + +* Avoid potential races between rebuild and EC aggregation by serializing these two operations; + this prevents inconsistencies between data and parity. +* Store EC aggregation epoch in RDB so the rebuild system retains the value after system restart. +* Add two new DFUSE options: --dump-handles to dump pool, container, and DFS handles when mounting + dfuse on a client. Other clients can use the dumped information to mount dfuse by using the + --read-handles option. This mechanism uses the local2global and global2local to serialize and + share handles from a root process to other processes. +* Correct how the JCH placement algorithm invokes CRC functions and improve data balancing. +* Fixes for EC object consistency verification so it can detect inconsistencies between data + and parity. +* Merge small or fragmented buffers on the client side to reduce IOV count and mitigate resource + exhaustion during network bulk transfers. +* Enable EC\_xP3 and use EC\_xP3 for automatic object-class selection. +* Fix a bug in the rebuild system related to waking up throttled ULTs, improving rebuild + performance. +* Fix pool/container handle behavior after fork. +* Multiple fixes for DTX background services to avoid orphan DTX entries and inconsistent DTX + participant states caused by intermittent network failures. +* Workaround for SPDK AMD IOMMU support. +* Fix a race condition between fetch and aggregation that could cause degraded fetch to return + an incorrect view of data. +* Reset the DTX base UUID after fork to avoid parent and child processes generating identical + DTX IDs. +* Fix a race between DTX refresh and abort that could leave a transaction in an unexpected state. +* Multiple fixes for EC degraded fetch of single values. +* Fix size queries for array values protected by erasure coding. +* Allow modification of a pool's redundancy factor on existing pools. +* Fix a race between DTX aggregation and container close that could cause an assertion failure. +* Multiple fixes in the rebuild system to prevent rebuild hangs on unstable networks. +* Increase the retry delay for collective RPCs to avoid generating large bursts of network traffic. +* Set the DFS chunk size on container creation (if not set by the caller) to a multiple of the full + EC stripe length to avoid misalignment. This ensures that the default DFS chunk size does not + create partial stripe updates if the IO size is full stripe. This auto setting works for + containers created with rd_fac 1 or 2. EC for rd_fac 3 was newly added and missed in this + optimization so users would have to set the chunk-size manually for that. This will be fixed next + release. +* Remove the separate RPM build for raft. +* Add a new environment variable, D\_QUOTA\_BULKS. When enabled, it limits the number of active + Mercury bulk allocations by postponing those that exceed the limit. +* DMG's exclude, drain, and reintegrate operations accept --ranks or --rank-hosts in ranged + format, allowing the same operation to be applied to those ranks across all pools they belong to. +* Ensure server-side verification does not cause a target to be disabled because of checksum + errors that are unrelated to the device. +* Avoid returning non-retryable errors to clients when an SSD is marked FAULTY, allowing + applications to continue accessing data in degraded mode. + ## DAOS Version 2.6.3 (2025-03-26) The DAOS 2.6.3 release includes the daos-2.6.3-4 RPM packages and its prerequisites. From b7dfc66232fb204e1cb792a9901011ee2f0f479b Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Fri, 31 Oct 2025 19:02:38 -0500 Subject: [PATCH 12/43] DAOS-18121 build: Update libfabric dep version (#17066) revert libfabric dep version to 1.20 to account for Slingshot libfabric on aurora. Signed-off-by: Mohamad Chaarawi --- utils/rpms/daos.spec | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 38a99570eef..fc12a9f95b9 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -11,7 +11,7 @@ %global daos_build_args client test %endif %global mercury_version 2.4.0-7 -%global libfabric_version 1.22.0-4 +%global libfabric_version 1.20 %global argobots_version 1.2-3 %global __python %{__python3} @@ -23,7 +23,7 @@ Name: daos Version: 2.6.4 -Release: 6%{?relval}%{?dist} +Release: 7%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -633,6 +633,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Fri Oct 31 2025 Mohamad Chaarawi 2.6.4-7 +- Revert bump of libfabric to 1.20 + * Thu Oct 23 2025 Phillip Henderson 2.6.4-6 - Fourth release candidate for 2.6.4 From 4ab491c3dafddec4ad36ccbfa6c9afd2c5e00540 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 3 Nov 2025 08:36:25 -0600 Subject: [PATCH 13/43] DAOS-18129 build: update the ubuntu changelog and release notes (#17073) Signed-off-by: Mohamad Chaarawi --- debian/changelog | 6 ++++++ docs/release/release_notes.md | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index cb9eb52f0a2..c61acc4ecb7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +daos (2.6.4-7) unstable; urgency=medium + [ Mohamad Chaarawi ] + * Fix libfabric packaging + + -- Mohamad Chaarawi Fri, 31 Oct 2025 20:00:00 -0500 + daos (2.6.4-6) unstable; urgency=medium [ Phillip Henderson ] * Fourth release candidate for 2.6.4 diff --git a/docs/release/release_notes.md b/docs/release/release_notes.md index 8a932941e2b..0f7cded220f 100644 --- a/docs/release/release_notes.md +++ b/docs/release/release_notes.md @@ -4,10 +4,10 @@ We are pleased to announce the release of DAOS version 2.6. ## DAOS Version 2.6.4 (2025-10-29) -The DAOS 2.6.4 release includes the daos-2.6.4-6 RPM packages and their prerequisites. +The DAOS 2.6.4 release includes the daos-2.6.4-7 RPM packages and their prerequisites. It contains the following updates on top of DAOS 2.6.3: -* Libfabric has been updated to version 1.22.0-4. +* Libfabric has been updated to version 1.20.0. * Mercury has been upgraded to version 2.4.0-7. * Argobots has been upgraded to version 1.2-3. * Libisal has been upgraded to version 2.31.1-7. From 06de8a0678a29a0143bbba5415b24282fe26ba49 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Mon, 3 Nov 2025 11:06:14 -0500 Subject: [PATCH 14/43] DAOS-18129 build: Tag 2.6.4 rc5 (#17074) Tag fifth release candidate for 2.6.4 Signed-off-by: Phil Henderson --- TAG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TAG b/TAG index 175c14e4440..59c033b06fe 100644 --- a/TAG +++ b/TAG @@ -1 +1 @@ -2.6.4-rc4 +2.6.4-rc5 From ef5e69f2280c95b3ba9b64c8d99bbc515e23d540 Mon Sep 17 00:00:00 2001 From: Li Wei Date: Mon, 10 Nov 2025 22:35:14 +0900 Subject: [PATCH 15/43] DAOS-17442 engine: Improve start issue handling (#16376) (#16588) We infer from the available log messages that bio_xsctxt_alloc might get stuck after "failed to init bdevs: -1003" for some unknown reason related to SPDK. This patch skips two while statements in bio_xsctxt_free that might be where we got stuck. When not in the CR mode, a pool ignores nonexistent children, leading to states that the current code cannot handle well (yet). This patch restricts this CR behavior to the CR mode. (Ideally, we shall allow the healthy children to start and automatically exclude the nonexistent children in the future.) To make pool start failures easier to spot, this patch adds a RAS event that is raised when we fail to start a pool. Also, this patch fixes "-1" xstream IDs in messages logged during engine setup, fixes a few places where we should write "pool" instead of "pool service", including the name of an internal environment variable, and adds some infrequent log messages to help future debugging of engine start issues. Move the unusually late initialization of dss_tls_key and daos_srv_modkey earlier, so that the TLS checks in server_id_cb will be safe. Both objects are static, and there's no reason to initialize them so late. Signed-off-by: Li Wei --- src/bio/bio_internal.h | 1 + src/bio/bio_xstream.c | 23 ++++++++++++++++------- src/engine/init.c | 37 ++++++++++++++++++++++++++++++++----- src/engine/module.c | 7 +------ src/engine/srv.c | 22 +++++++--------------- src/include/daos_srv/ras.h | 2 ++ src/mgmt/srv_target.c | 3 +++ src/pool/srv.c | 2 +- src/pool/srv_pool.c | 13 ++++++++++--- src/pool/srv_target.c | 7 +++---- 10 files changed, 76 insertions(+), 41 deletions(-) diff --git a/src/bio/bio_internal.h b/src/bio/bio_internal.h index d57ea15201c..04cb6f73613 100644 --- a/src/bio/bio_internal.h +++ b/src/bio/bio_internal.h @@ -385,6 +385,7 @@ struct bio_xs_context { struct bio_xs_blobstore *bxc_xs_blobstores[SMD_DEV_TYPE_MAX]; struct bio_dma_buffer *bxc_dma_buf; unsigned int bxc_self_polling:1; /* for standalone VOS */ + unsigned int bxc_skip_draining : 1; }; /* Per VOS instance I/O context */ diff --git a/src/bio/bio_xstream.c b/src/bio/bio_xstream.c index 0c2e5469404..9177372b498 100644 --- a/src/bio/bio_xstream.c +++ b/src/bio/bio_xstream.c @@ -2,7 +2,6 @@ * (C) Copyright 2018-2024 Intel Corporation. * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * - * * SPDX-License-Identifier: BSD-2-Clause-Patent */ #define D_LOGFAC DD_FAC(bio) @@ -1507,6 +1506,8 @@ bio_xsctxt_free(struct bio_xs_context *ctxt) rc = xs_poll_completion(ctxt, &cp_arg.cca_inflights, bio_spdk_subsys_timeout); DL_CDEBUG(rc == 0, DB_MGMT, DLOG_ERR, rc, "SPDK subsystems finalized"); + if (rc != 0) + ctxt->bxc_skip_draining = 1; nvme_glb.bd_init_thread = NULL; @@ -1518,18 +1519,21 @@ bio_xsctxt_free(struct bio_xs_context *ctxt) ABT_mutex_unlock(nvme_glb.bd_mutex); if (ctxt->bxc_thread != NULL) { - D_DEBUG(DB_MGMT, "Finalizing SPDK thread, tgt_id:%d", - ctxt->bxc_tgt_id); + D_DEBUG(DB_MGMT, "Finalizing SPDK thread, tgt_id:%d, skip_draining:%u", + ctxt->bxc_tgt_id, ctxt->bxc_skip_draining); - /* Don't drain events if spdk_subsystem_fini() timeout */ - while (rc == 0 && !spdk_thread_is_idle(ctxt->bxc_thread)) + /* + * Don't drain events if we are asked to skip this (usually + * due to an earlier error). + */ + while (!ctxt->bxc_skip_draining && !spdk_thread_is_idle(ctxt->bxc_thread)) spdk_thread_poll(ctxt->bxc_thread, 0, 0); D_DEBUG(DB_MGMT, "SPDK thread finalized, tgt_id:%d", ctxt->bxc_tgt_id); spdk_thread_exit(ctxt->bxc_thread); - while (rc == 0 && !spdk_thread_is_exited(ctxt->bxc_thread)) + while (!ctxt->bxc_skip_draining && !spdk_thread_is_exited(ctxt->bxc_thread)) spdk_thread_poll(ctxt->bxc_thread, 0, 0); spdk_thread_destroy(ctxt->bxc_thread); ctxt->bxc_thread = NULL; @@ -1615,7 +1619,12 @@ bio_xsctxt_alloc(struct bio_xs_context **pctxt, int tgt_id, bool self_polling) if (cp_arg.cca_rc != 0) { rc = cp_arg.cca_rc; - D_ERROR("failed to init bdevs, rc:%d\n", rc); + DL_ERROR(rc, "failed to init bdevs"); + /* + * We're afraid that draining the thread might never + * complete (DAOS-17442). + */ + ctxt->bxc_skip_draining = 1; goto out; } diff --git a/src/engine/init.c b/src/engine/init.c index f429a7c48e7..219725a267d 100644 --- a/src/engine/init.c +++ b/src/engine/init.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -160,6 +161,27 @@ hlc_recovery_end(uint64_t bound) } } +static int +init_tls_support(void) +{ + int rc; + + rc = ds_tls_key_create(); + if (rc != 0) + return daos_errno2der(rc); + + dss_register_key(&daos_srv_modkey); + + return 0; +} + +static void +fini_tls_support(void) +{ + dss_unregister_key(&daos_srv_modkey); + ds_tls_key_delete(); +} + /* * Register the dbtree classes used by native server-side modules (e.g., * ds_pool, ds_cont, etc.). Unregistering is currently not supported. @@ -696,10 +718,6 @@ dss_crt_hlc_error_cb(void *arg) static void server_id_cb(uint32_t *tid, uint64_t *uid) { - - if (server_init_state != DSS_INIT_STATE_SET_UP) - return; - if (uid != NULL && dss_abt_init) { ABT_unit_type type = ABT_UNIT_TYPE_EXT; int rc; @@ -751,10 +769,15 @@ server_init(int argc, char *argv[]) gethostname(dss_hostname, DSS_HOSTNAME_MAX_LEN); + /* Must initialize TLS support before using server_id_cb. */ + rc = init_tls_support(); + if (rc != 0) + return rc; + daos_debug_set_id_cb(server_id_cb); rc = daos_debug_init_ex(DAOS_LOG_DEFAULT, DLOG_INFO); if (rc != 0) - return rc; + goto exit_tls_support; /** initialize server topology data - this is needed to set up the number of targets */ rc = dss_topo_init(); @@ -927,6 +950,8 @@ server_init(int argc, char *argv[]) /* dss_topo_fini cleans itself if it fails */ exit_debug_init: daos_debug_fini(); +exit_tls_support: + fini_tls_support(); return rc; } @@ -987,6 +1012,8 @@ server_fini(bool force) D_INFO("dss_top_fini() done\n"); daos_debug_fini(); D_INFO("daos_debug_fini() done\n"); + fini_tls_support(); + D_INFO("fini_tls_support() done\n"); } static void diff --git a/src/engine/module.c b/src/engine/module.c index d43387573e4..847d9d800d3 100644 --- a/src/engine/module.c +++ b/src/engine/module.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -244,12 +245,6 @@ dss_module_init_all(uint64_t *mod_facs) uint64_t fac; int rc = 0; - /* - * first register global tls accessible to all modules, - * it'll be first initialized and last finalized. - */ - dss_register_key(&daos_srv_modkey); - /* lookup the module from the loaded module list */ D_MUTEX_LOCK(&loaded_mod_list_lock); d_list_for_each_entry_safe(lmod, tmp, &loaded_mod_list, lm_lk) { diff --git a/src/engine/srv.c b/src/engine/srv.c index ef25b0ec7f8..49a93668207 100644 --- a/src/engine/srv.c +++ b/src/engine/srv.c @@ -570,8 +570,10 @@ dss_srv_handler(void *arg) } } - if (dss_xstream_exiting(dx)) + if (dss_xstream_exiting(dx)) { + rc = 0; break; + } ABT_thread_yield(); } @@ -608,6 +610,7 @@ dss_srv_handler(void *arg) ABT_cond_signal(xstream_data.xd_ult_init); ABT_mutex_unlock(xstream_data.xd_mutex); } + DL_CDEBUG(rc == 0, DLOG_INFO, DLOG_ERR, rc, "stopping"); } static inline struct dss_xstream * @@ -1066,8 +1069,10 @@ dss_start_xs_id(int tag, int xs_id) } rc = dss_start_one_xstream(obj->cpuset, tag, xs_id); - if (rc) + if (rc != 0) { + DL_ERROR(rc, "failed to start one xstream: tag=%x xs_id=%d", tag, xs_id); return rc; + } return 0; } @@ -1317,7 +1322,6 @@ enum { XD_INIT_MUTEX, XD_INIT_ULT_INIT, XD_INIT_ULT_BARRIER, - XD_INIT_TLS_REG, XD_INIT_TLS_INIT, XD_INIT_SYS_DB, XD_INIT_XSTREAMS, @@ -1355,9 +1359,6 @@ dss_srv_fini(bool force) case XD_INIT_TLS_INIT: vos_standalone_tls_fini(); /* fall through */ - case XD_INIT_TLS_REG: - ds_tls_key_delete(); - /* fall through */ case XD_INIT_ULT_BARRIER: ABT_cond_free(&xstream_data.xd_ult_barrier); /* fall through */ @@ -1452,15 +1453,6 @@ dss_srv_init(void) } xstream_data.xd_init_step = XD_INIT_ULT_BARRIER; - /* register xstream-local storage key */ - rc = ds_tls_key_create(); - if (rc) { - rc = dss_abterr2der(rc); - D_ERROR("Failed to register storage key: "DF_RC"\n", DP_RC(rc)); - D_GOTO(failed, rc); - } - xstream_data.xd_init_step = XD_INIT_TLS_REG; - /* initialize xstream-local storage */ rc = vos_standalone_tls_init(DAOS_SERVER_TAG - DAOS_TGT_TAG); if (rc) { diff --git a/src/include/daos_srv/ras.h b/src/include/daos_srv/ras.h index 5f22a75795f..5df47372eb9 100644 --- a/src/include/daos_srv/ras.h +++ b/src/include/daos_srv/ras.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2020-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -49,6 +50,7 @@ X(RAS_POOL_REPS_UPDATE, "pool_replicas_updated") \ X(RAS_POOL_DF_INCOMPAT, "pool_durable_format_incompatible") \ X(RAS_POOL_DEFER_DESTROY, "pool_destroy_deferred") \ + X(RAS_POOL_START_FAILED, "pool_start_failed") \ X(RAS_CONT_DF_INCOMPAT, "container_durable_format_incompatible") \ X(RAS_RDB_DF_INCOMPAT, "rdb_durable_format_incompatible") \ X(RAS_SWIM_RANK_ALIVE, "swim_rank_alive") \ diff --git a/src/mgmt/srv_target.c b/src/mgmt/srv_target.c index 7975a2115d4..054842e5e76 100644 --- a/src/mgmt/srv_target.c +++ b/src/mgmt/srv_target.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1273,6 +1274,7 @@ tgt_destroy_cleanup(void *arg) /** make sure the rename is persistent */ (void)dir_fsync(zombie); + D_INFO(DF_UUID ": moved %s to %s\n", DP_UUID(tda->tda_id.uuid), tda->tda_path, zombie); /** * once successfully moved to the ZOMBIES directory, the target will @@ -1282,6 +1284,7 @@ tgt_destroy_cleanup(void *arg) if (tda->tda_rc == 0) { (void)subtree_destroy(zombie); (void)rmdir(zombie); + D_INFO(DF_UUID ": removed %s\n", DP_UUID(tda->tda_id.uuid), zombie); } else { D_INFO("Defer cleanup for lingering pool:"DF_UUID"\n", DP_UUID(tda->tda_id.uuid)); diff --git a/src/pool/srv.c b/src/pool/srv.c index df93ecf3a5f..5d933c21684 100644 --- a/src/pool/srv.c +++ b/src/pool/srv.c @@ -118,7 +118,7 @@ setup(void) bool start = true; if (!engine_in_check()) { - d_getenv_bool("DAOS_START_POOL_SVC", &start); + d_getenv_bool("DAOS_START_POOLS", &start); if (start) return ds_pool_start_all(); } diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 57df9b94590..456bf5ec89d 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -2699,9 +2699,17 @@ start_one(uuid_t uuid, void *varg) rc = ds_pool_start(uuid, aft_chk, immutable); if (rc != 0) { + uuid_t uuid_tmp; + DL_ERROR(rc, DF_UUID ": failed to start pool, aft_chk %s, immutable %s", DP_UUID(uuid), aft_chk ? "yes" : "no", immutable ? "yes" : "no"); ds_pool_failed_add(uuid, rc); + uuid_copy(uuid_tmp, uuid); + ds_notify_ras_eventf(RAS_POOL_START_FAILED, RAS_TYPE_INFO, RAS_SEV_ERROR, + NULL /* hwid */, NULL /* rank */, NULL /* inc */, + NULL /* jobid */, &uuid_tmp, NULL /* cont */, NULL /* objid */, + NULL /* ctlop */, NULL /* data */, + "failed to start pool: " DF_RC, DP_RC(rc)); } return 0; @@ -2712,11 +2720,10 @@ pool_start_all(void *arg) { int rc; - /* Scan the storage and start all pool services. */ + /* Scan the storage and start all pools. */ rc = ds_mgmt_tgt_pool_iterate(start_one, NULL /* arg */); if (rc != 0) - D_ERROR("failed to scan all pool services: "DF_RC"\n", - DP_RC(rc)); + DL_ERROR(rc, "failed to scan pools"); } bool diff --git a/src/pool/srv_target.c b/src/pool/srv_target.c index 76a0610a20a..e3c6d615a15 100644 --- a/src/pool/srv_target.c +++ b/src/pool/srv_target.c @@ -460,7 +460,6 @@ pool_child_recreate(struct ds_pool_child *child) } - static int pool_child_start(struct ds_pool_child *child, bool recreate) { @@ -490,14 +489,14 @@ pool_child_start(struct ds_pool_child *child, bool recreate) D_FREE(path); if (rc != 0) { - if (rc != -DER_NONEXIST) { + if (!engine_in_check() || rc != -DER_NONEXIST) { DL_CDEBUG(rc == -DER_NVME_IO, DB_MGMT, DLOG_ERR, rc, DF_UUID": Open VOS pool failed.", DP_UUID(child->spc_uuid)); goto out; } - D_WARN("Lost pool "DF_UUIDF" shard %u on rank %u.\n", - DP_UUID(child->spc_uuid), info->dmi_tgt_id, dss_self_rank()); + D_WARN(DF_UUID ": Lost pool shard %u on rank %u.\n", DP_UUID(child->spc_uuid), + info->dmi_tgt_id, dss_self_rank()); /* * Ignore the failure to allow subsequent logic (such as DAOS check) * to handle the trouble. From 50bac8854d798bc8508382898ac8bb0d4fa5f496 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 10 Nov 2025 09:53:30 -0600 Subject: [PATCH 16/43] DAOS-18171 dfs: set dfs chunk size for RF3 containers (#17060) EC with 3 parity is supported, so we should consider that for the dfs chunk size selection. Signed-off-by: Mohamad Chaarawi --- src/client/dfs/cont.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/dfs/cont.c b/src/client/dfs/cont.c index 5aead6050f5..5f59885fbf5 100644 --- a/src/client/dfs/cont.c +++ b/src/client/dfs/cont.c @@ -27,8 +27,8 @@ suggest_dfs_cs(daos_handle_t poh, daos_prop_t *prop, uint64_t rf, daos_oclass_id daos_size_t base; int rc; - /** No EC above RF 2, use default CS */ - if (rf > 2) { + /** No EC above RF 3, use default CS */ + if (rf > 3) { *cs = DFS_DEFAULT_CHUNK_SIZE; return 0; } From db80417ce5e62f03e0a72e59b41fb84e515a7792 Mon Sep 17 00:00:00 2001 From: Ding Date: Mon, 10 Nov 2025 13:21:13 -0500 Subject: [PATCH 17/43] DAOS-17911 test: fix dfuse/sparse_file.py (#16759) (#17078) Resolve for paramiko ssh issue Signed-off-by: Ding-Hwa Ho --- src/tests/ftest/dfuse/sparse_file.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/dfuse/sparse_file.py b/src/tests/ftest/dfuse/sparse_file.py index 484b787c38c..f1d4f4ad02f 100644 --- a/src/tests/ftest/dfuse/sparse_file.py +++ b/src/tests/ftest/dfuse/sparse_file.py @@ -1,5 +1,6 @@ """ (C) Copyright 2020-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -68,10 +69,11 @@ def test_sparsefile(self): # create and open a connection on remote node to open file on that # remote node + key_path = os.path.expanduser(os.path.join('~', '.ssh', 'id_rsa')) + self.log.info("=does %s exist: %s", key_path, os.path.exists(key_path)) ssh = paramiko.SSHClient() - ssh.load_system_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(self.hostlist_clients[0], username=getuser()) + ssh.connect(hostname=self.hostlist_clients[0], username=getuser(), key_filename=key_path) sftp = ssh.open_sftp() # open remote file From 2c19a8b058a056a1e002135688282939d6f5d312 Mon Sep 17 00:00:00 2001 From: Nasf-Fan Date: Tue, 11 Nov 2025 03:39:48 +0800 Subject: [PATCH 18/43] DAOS-17258 object: more check for CPD RPC bulk transfer - b26 (#16082) To detect potential on-wire data corruption for CPD RPC bulk transfer. CPD RPC bulk logic improvement (use bulk handle instead of bulk handle pointer on-wire) and cleanup. Signed-off-by: Fan Yong --- src/object/obj_rpc.c | 15 ++--------- src/object/obj_rpc.h | 2 +- src/object/obj_tx.c | 63 +++++++++++++++++++++----------------------- src/object/srv_obj.c | 51 ++++++++++++++++++++++++++--------- 4 files changed, 71 insertions(+), 60 deletions(-) diff --git a/src/object/obj_rpc.c b/src/object/obj_rpc.c index 562c5c631bb..e8a0d202bdd 100644 --- a/src/object/obj_rpc.c +++ b/src/object/obj_rpc.c @@ -878,22 +878,11 @@ crt_proc_struct_daos_cpd_bulk(crt_proc_t proc, crt_proc_op_t proc_op, if (unlikely(rc)) return rc; - if (DECODING(proc_op)) { - D_ALLOC_PTR(dcb->dcb_bulk); - if (dcb->dcb_bulk == NULL) - return -DER_NOMEM; - } - - rc = crt_proc_crt_bulk_t(proc, proc_op, dcb->dcb_bulk); - if (unlikely(rc)) - return rc; - - if (FREEING(proc_op)) - D_FREE(dcb->dcb_bulk); + rc = crt_proc_crt_bulk_t(proc, proc_op, &dcb->dcb_bulk); /* The other fields will not be packed on-wire. */ - return 0; + return rc; } static int diff --git a/src/object/obj_rpc.h b/src/object/obj_rpc.h index d4fad534a5f..4c79eb7bef2 100644 --- a/src/object/obj_rpc.h +++ b/src/object/obj_rpc.h @@ -645,7 +645,7 @@ struct daos_cpd_bulk { struct daos_cpd_sub_head dcb_head; uint32_t dcb_size; uint32_t dcb_padding; - crt_bulk_t *dcb_bulk; + crt_bulk_t dcb_bulk; /* * The following are only used to handle the bulk for CPD RPC body temporarily, * do not pack on-wire. diff --git a/src/object/obj_tx.c b/src/object/obj_tx.c index f3eec4d44a9..a52a78be7ff 100644 --- a/src/object/obj_tx.c +++ b/src/object/obj_tx.c @@ -392,6 +392,15 @@ dc_tx_alloc(daos_handle_t coh, daos_epoch_t epoch, uint64_t flags, return rc; } +static inline void +dc_tx_bulk_free(crt_bulk_t *hdl) +{ + if (hdl != NULL && *hdl != CRT_BULK_NULL) { + crt_bulk_free(*hdl); + *hdl = CRT_BULK_NULL; + } +} + static void dc_tx_cleanup_one(struct dc_tx *tx, struct daos_cpd_sub_req *dcsr) { @@ -406,11 +415,8 @@ dc_tx_cleanup_one(struct dc_tx *tx, struct daos_cpd_sub_req *dcsr) csummer = tx->tx_co->dc_csummer; if (dcu->dcu_flags & ORF_CPD_BULK) { - for (i = 0; i < dcsr->dcsr_nr; i++) { - if (dcu->dcu_bulks[i] != CRT_BULK_NULL) - crt_bulk_free(dcu->dcu_bulks[i]); - } - + for (i = 0; i < dcsr->dcsr_nr; i++) + dc_tx_bulk_free(&dcu->dcu_bulks[i]); D_FREE(dcu->dcu_bulks); } @@ -521,12 +527,7 @@ dc_tx_cleanup(struct dc_tx *tx) /* Keep 'tx_set_resend'. */ if (tx->tx_reqs.dcs_type == DCST_BULK_REQ) { - if (tx->tx_reqs_bulk.dcb_bulk != NULL) { - if (tx->tx_reqs_bulk.dcb_bulk[0] != CRT_BULK_NULL) - crt_bulk_free(tx->tx_reqs_bulk.dcb_bulk[0]); - D_FREE(tx->tx_reqs_bulk.dcb_bulk); - } - + dc_tx_bulk_free(&tx->tx_reqs_bulk.dcb_bulk); D_FREE(tx->tx_reqs_bulk.dcb_iov.iov_buf); } @@ -538,11 +539,7 @@ dc_tx_cleanup(struct dc_tx *tx) } if (tx->tx_head.dcs_type == DCST_BULK_HEAD) { - if (tx->tx_head_bulk.dcb_bulk != NULL) { - if (tx->tx_head_bulk.dcb_bulk[0] != CRT_BULK_NULL) - crt_bulk_free(tx->tx_head_bulk.dcb_bulk[0]); - D_FREE(tx->tx_head_bulk.dcb_bulk); - } + dc_tx_bulk_free(&tx->tx_head_bulk.dcb_bulk); /* Free MBS buffer. */ D_FREE(tx->tx_head_bulk.dcb_iov.iov_buf); } else { @@ -555,11 +552,7 @@ dc_tx_cleanup(struct dc_tx *tx) tx->tx_head.dcs_buf = NULL; if (tx->tx_disp.dcs_type == DCST_BULK_ENT) { - if (tx->tx_disp_bulk.dcb_bulk != NULL) { - if (tx->tx_disp_bulk.dcb_bulk[0] != CRT_BULK_NULL) - crt_bulk_free(tx->tx_disp_bulk.dcb_bulk[0]); - D_FREE(tx->tx_disp_bulk.dcb_bulk); - } + dc_tx_bulk_free(&tx->tx_disp_bulk.dcb_bulk); dcde = tx->tx_disp_bulk.dcb_iov.iov_buf; } else { dcde = tx->tx_disp.dcs_buf; @@ -575,11 +568,7 @@ dc_tx_cleanup(struct dc_tx *tx) } if (tx->tx_tgts.dcs_type == DCST_BULK_TGT) { - if (tx->tx_tgts_bulk.dcb_bulk != NULL) { - if (tx->tx_tgts_bulk.dcb_bulk[0] != CRT_BULK_NULL) - crt_bulk_free(tx->tx_tgts_bulk.dcb_bulk[0]); - D_FREE(tx->tx_tgts_bulk.dcb_bulk); - } + dc_tx_bulk_free(&tx->tx_tgts_bulk.dcb_bulk); D_FREE(tx->tx_tgts_bulk.dcb_iov.iov_buf); tx->tx_tgts.dcs_buf = NULL; } else { @@ -1780,15 +1769,23 @@ dc_tx_cpd_body_bulk(struct daos_cpd_sg *dcs, struct daos_cpd_bulk *dcb, dcb->dcb_sgl.sg_nr_out = 1; dcb->dcb_sgl.sg_iovs = &dcb->dcb_iov; - rc = obj_bulk_prep(&dcb->dcb_sgl, 1, true, CRT_BULK_RO, task, &dcb->dcb_bulk); - if (rc == 0) { - dcs->dcs_type = type; - dcs->dcs_nr = nr; - dcs->dcs_buf = dcb; - } else { + rc = crt_bulk_create(daos_task2ctx(task), &dcb->dcb_sgl, CRT_BULK_RO, &dcb->dcb_bulk); + if (rc != 0) + goto out; + + rc = crt_bulk_bind(dcb->dcb_bulk, daos_task2ctx(task)); + if (rc != 0) + goto out; + + dcs->dcs_type = type; + dcs->dcs_buf = dcb; + dcs->dcs_nr = nr; + +out: + if (rc != 0) { + dc_tx_bulk_free(&dcb->dcb_bulk); dcb->dcb_iov.iov_buf = NULL; } - return rc; } diff --git a/src/object/srv_obj.c b/src/object/srv_obj.c index 68512bb653f..db39b83b6ea 100644 --- a/src/object/srv_obj.c +++ b/src/object/srv_obj.c @@ -5263,11 +5263,7 @@ ds_obj_cpd_body_bulk(crt_rpc_t *rpc, struct obj_io_context *ioc, bool leader, struct daos_cpd_bulk **dcbs = NULL; struct daos_cpd_bulk *dcb = NULL; crt_bulk_t *bulks = NULL; - d_sg_list_t **sgls = NULL; - struct daos_cpd_sub_head *dcsh; - struct daos_cpd_disp_ent *dcde; - struct daos_cpd_req_idx *dcri; - void *end; + d_sg_list_t **sgls = NULL; uint32_t total = 0; uint32_t count = 0; int rc = 0; @@ -5350,21 +5346,31 @@ ds_obj_cpd_body_bulk(crt_rpc_t *rpc, struct obj_io_context *ioc, bool leader, D_GOTO(out, rc = -DER_NOMEM); for (i = 0; i < count; i++) { - bulks[i] = *dcbs[i]->dcb_bulk; + bulks[i] = dcbs[i]->dcb_bulk; sgls[i] = &dcbs[i]->dcb_sgl; } - rc = obj_bulk_transfer(rpc, CRT_BULK_GET, ORF_BULK_BIND, bulks, NULL, NULL, DAOS_HDL_INVAL, - sgls, count, count, NULL); + rc = obj_bulk_transfer(rpc, CRT_BULK_GET, true, bulks, NULL, NULL, DAOS_HDL_INVAL, sgls, + count, count, NULL); if (rc != 0) goto out; for (i = 0; i < count; i++) { + D_ASSERTF(dcbs[i]->dcb_size == dcbs[i]->dcb_iov.iov_len, "Bad bulk: %u vs %u\n", + dcbs[i]->dcb_size, (uint32_t)dcbs[i]->dcb_iov.iov_len); + switch (dcbs[i]->dcb_type) { - case DCST_BULK_HEAD: - dcsh = &dcbs[i]->dcb_head; + case DCST_BULK_HEAD: { + struct daos_cpd_sub_head *dcsh = &dcbs[i]->dcb_head; + dcsh->dcsh_mbs = dcbs[i]->dcb_iov.iov_buf; + D_ASSERTF(dcsh->dcsh_mbs->dm_data_size == + dcbs[i]->dcb_iov.iov_len - sizeof(*dcsh->dcsh_mbs), + "Invalid bulk MBS data for CPD RPC on %s: %u vs %u\n", + leader ? "leader" : "follower", dcsh->dcsh_mbs->dm_data_size, + (uint32_t)(dcbs[i]->dcb_iov.iov_len - sizeof(*dcsh->dcsh_mbs))); break; + } case DCST_BULK_REQ: rc = crt_proc_create(dss_get_module_info()->dmi_ctx, dcbs[i]->dcb_iov.iov_buf, dcbs[i]->dcb_iov.iov_len, @@ -5384,17 +5390,36 @@ ds_obj_cpd_body_bulk(crt_rpc_t *rpc, struct obj_io_context *ioc, bool leader, goto out; } break; - case DCST_BULK_ENT: + case DCST_BULK_ENT: { + struct daos_cpd_disp_ent *dcde; + struct daos_cpd_req_idx *dcri; + void *end; + dcde = dcbs[i]->dcb_iov.iov_buf; dcri = dcbs[i]->dcb_iov.iov_buf + sizeof(*dcde) * dcbs[i]->dcb_item_nr; - end = dcbs[i]->dcb_iov.iov_buf + dcbs[i]->dcb_iov.iov_len; + end = dcbs[i]->dcb_iov.iov_buf + dcbs[i]->dcb_iov.iov_len; for (j = 0; j < dcbs[i]->dcb_item_nr; j++) { dcde[j].dcde_reqs = dcri; dcri += dcde[j].dcde_read_cnt + dcde[j].dcde_write_cnt; - D_ASSERT((void *)dcri <= end); + D_ASSERTF((void *)dcri <= end, + "Invalid bulk ENT data for CPD RPC on %s: pos %u, rd %u, " + "wr %u, nr %u, size %u, base %p, cur %p, end %p\n", + leader ? "leader" : "follower", j, dcde[j].dcde_read_cnt, + dcde[j].dcde_write_cnt, dcbs[i]->dcb_item_nr, + dcbs[i]->dcb_size, dcde, dcri, end); } break; + } + case DCST_BULK_TGT: { + struct daos_shard_tgt *tgts = dcbs[i]->dcb_iov.iov_buf; + + D_ASSERTF(dss_self_rank() == tgts[0].st_rank && + dss_get_module_info()->dmi_tgt_id == tgts[0].st_tgt_idx, + "Invalid bulk tgt data on %u: rank %u, tgt_idx %u\n", + dss_self_rank(), tgts[0].st_rank, tgts[0].st_tgt_idx); + break; + } default: break; } From e65a4d7012c53499a367c2fc38bba68842a2fcd2 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Mon, 10 Nov 2025 14:44:27 -0800 Subject: [PATCH 19/43] DAOS-17998 test: enhance test_checker_on_admin_excluded (#16894) (#17106) - Verify ranks can start after clearing exclusions - Stop a random rank instead of hardcoded - Increase to 4 servers / dual engine Signed-off-by: Dalton Bohning --- src/tests/ftest/recovery/ms_membership.py | 79 +++++++++++---------- src/tests/ftest/recovery/ms_membership.yaml | 18 +++-- 2 files changed, 54 insertions(+), 43 deletions(-) diff --git a/src/tests/ftest/recovery/ms_membership.py b/src/tests/ftest/recovery/ms_membership.py index f7c7c8250a5..f490f125989 100644 --- a/src/tests/ftest/recovery/ms_membership.py +++ b/src/tests/ftest/recovery/ms_membership.py @@ -1,5 +1,6 @@ """ (C) Copyright 2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -19,7 +20,7 @@ def test_checker_on_admin_excluded(self): """Test checker can only be run when the system status is AdminExcluded. 1. Call dmg check enable. - 2. Stop rank 1. + 2. Stop 1 random rank 3. Call dmg system query and check that status of at least one rank is not "checkerstarted". We verify that the new checkerstarted state is properly changed. 4. Call dmg check start. It should show error because the stopped rank is not at @@ -38,45 +39,42 @@ def test_checker_on_admin_excluded(self): """ dmg_command = self.get_dmg_command() - # 1. Call dmg check enable. + self.log_step("Call dmg check enable.") dmg_command.check_enable() - # 2. Stop rank 1. - dmg_command.system_stop(ranks="1") + self.log_step("Stop 1 random rank") + rank_to_stop = self.random.choice(list(self.server_managers[0].ranks.keys())) + dmg_command.system_stop(ranks=rank_to_stop) - # 3. Call dmg system query and check that status of at least one rank is not - # "checkerstarted". We verify that the new checkerstarted state is properly - # changed. + self.log_step("Verify the the stopped rank is stopped and all others are checkerstarted") query_out = dmg_command.system_query() - not_checkerstarted_found = False for member in query_out["response"]["members"]: - if member["state"] != "checkerstarted": - not_checkerstarted_found = True - break - if not not_checkerstarted_found: - # All rank's status is "checkerstarted". - self.fail("All rank's status is checkerstarted!") - - # 4. Call dmg check start. It should show error because the stopped rank is not at - # CheckerStarted state. + if member["rank"] == rank_to_stop: + if member["state"] != "stopped": + self.fail(f"Stopped rank has invalid state: {member['state']}") + elif member["state"] != "checkerstarted": + self.fail("Expected non-stopped ranks to be in state checkstarted") + + self.log_step("Verify dmg check start fails") try: dmg_command.check_start() + self.fail("dmg check start did not fail as expected") except CommandFailure as error: self.log.info("dmg check start is expected to fail. Error: %s", error) - # 5. Call dmg check query. It should show error because the stopped rank is not at - # CheckerStarted state. + self.log_step("Verify dmg check query fails") try: dmg_command.check_query() + # To be fixed by DAOS-18001 + # self.fail("dmg check query did not fail as expected") except CommandFailure as error: self.log.info("dmg check query is expected to fail. Error: %s", error) - # 6. Call dmg check disable. It should work. + self.log_step("Verify dmg check disable works") dmg_command.check_disable() - # 7. Restart the stopped rank for cleanup. - self.log.info("Restart stopped rank for cleanup.") - dmg_command.system_start(ranks="1") + self.log_step("Restart stopped rank for cleanup.") + dmg_command.system_start(ranks=rank_to_stop) def test_enable_disable_admin_excluded(self): """Test dmg system exclude and clear-exclude. @@ -84,15 +82,14 @@ def test_enable_disable_admin_excluded(self): Test admin can enable and disable the rank state to AdminExcluded when the rank is down. - 1. Stop rank 1. - 2. Set rank 1 to AdminExcluded by calling dmg system exclude --ranks=1 and verify + 1. Stop all ranks + 2. Set 1 random rank to AdminExcluded by calling dmg system exclude --ranks=1 and verify the state has been changed. 3. Verify that the checker can be run with AdminExcluded state by calling enable, start, query, and disable. Verify that none of the commands returns error. - 4. Disable AdminExcluded of rank 1 by calling dmg system clear-exclude --ranks=1 + 4. Disable AdminExcluded of the excluded rank by calling dmg system clear-exclude and verify the state has been changed. - 5. Servers haven't been started, so update the expected state of rank 0 for - cleanup. + 5. Start the ranks again Jira ID: DAOS-11704 @@ -104,15 +101,16 @@ def test_enable_disable_admin_excluded(self): errors = [] dmg_command = self.get_dmg_command() - # 1. Stop rank 1. - dmg_command.system_stop(ranks="1") + self.log_step("Stop all ranks") + dmg_command.system_stop(force=True) - # 2. Set rank 1 to AdminExcluded and verify the state has been changed. - self.server_managers[-1].system_exclude(ranks=[1]) + self.log_step("Exclude 1 random rank") + rank_to_exclude = self.random.choice(list(self.server_managers[0].ranks.keys())) + self.server_managers[-1].system_exclude(ranks=[rank_to_exclude]) - # 3. Verify that the checker can be run with AdminExcluded state. + self.log_step("Verify that the checker can be run with AdminExcluded state.") try: - dmg_command.check_enable() + dmg_command.check_enable(stop=False) dmg_command.check_start() dmg_command.check_query() # We need to start after calling dmg system clear-exclude, otherwise the start @@ -122,11 +120,14 @@ def test_enable_disable_admin_excluded(self): msg = f"dmg check command failed! {error}" errors.append(msg) - # 4. Disable AdminExcluded of rank 1 and verify the state has been changed. - self.server_managers[-1].system_clear_exclude(ranks=[1]) + self.log_step( + "Disable AdminExcluded of the stopped rank and verify the state has been changed.") + self.server_managers[-1].system_clear_exclude(ranks=[rank_to_exclude]) - # 5. Servers haven't been started, so update the expected state of rank 0 for - # cleanup. - self.server_managers[-1].update_expected_states(0, ['stopped']) + self.log_step("Verify all ranks are able to restart") + dmg_command.system_start() + all_ranks = list(self.server_managers[0].ranks.keys()) + self.server_managers[-1].update_expected_states(all_ranks, ['joined']) + self.server_managers[-1].check_rank_state(all_ranks, ['joined']) report_errors(test=self, errors=errors) diff --git a/src/tests/ftest/recovery/ms_membership.yaml b/src/tests/ftest/recovery/ms_membership.yaml index b698773d1bf..e4157dae333 100644 --- a/src/tests/ftest/recovery/ms_membership.yaml +++ b/src/tests/ftest/recovery/ms_membership.yaml @@ -1,13 +1,23 @@ hosts: - test_servers: 2 + test_servers: 4 timeout: 180 server_config: - name: daos_server - engines_per_host: 1 + engines_per_host: 2 engines: 0: - targets: 1 + pinned_numa_node: 0 + nr_xs_helpers: 1 + fabric_iface: ib0 + fabric_iface_port: 31317 + log_file: daos_server0.log + storage: auto + 1: + pinned_numa_node: 1 + nr_xs_helpers: 1 + fabric_iface: ib1 + fabric_iface_port: 31417 + log_file: daos_server1.log storage: auto system_ram_reserved: 1 From ebdc6cceeda461c62fc5ef7b35109270b70f95f9 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Mon, 10 Nov 2025 15:09:12 -0800 Subject: [PATCH 20/43] DAOS-18102 test: update agent_failure.py to run commands as user (#16980) (#17105) Update get_journalctl to accept user argument other than root. Update stop_processes to accept user argument other than root. Update agent_failure.py to run these as the current user when applicable. Signed-off-by: Dalton Bohning --- src/tests/ftest/deployment/agent_failure.py | 9 +++++---- src/tests/ftest/util/general_utils.py | 15 ++++++++++----- src/tests/ftest/util/run_utils.py | 6 +++--- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/tests/ftest/deployment/agent_failure.py b/src/tests/ftest/deployment/agent_failure.py index eaa97cdde2b..d423d3454bc 100644 --- a/src/tests/ftest/deployment/agent_failure.py +++ b/src/tests/ftest/deployment/agent_failure.py @@ -123,7 +123,7 @@ def test_agent_failure(self): # 5. Verify journalctl shows the log that the agent is stopped. results = get_journalctl( hosts=self.hostlist_clients, since=since, until=until, - journalctl_type="daos_agent") + journalctl_type="daos_agent", run_user=self.test_env.agent_user) self.log.info("journalctl results = %s", results) if "shutting down" not in results[0]["data"]: msg = "Agent shut down message not found in journalctl! Output = {}".format( @@ -210,7 +210,8 @@ def test_agent_failure_isolation(self): since = journalctl_time() self.log.info("Stopping agent on %s", agent_host_kill) pattern = self.agent_managers[0].manager.job.command_regex - detected, running = stop_processes(self.log, hosts=agent_host_kill, pattern=pattern) + detected, running = stop_processes(self.log, hosts=agent_host_kill, pattern=pattern, + user=self.agent_managers[0].manager.job.run_user) if not detected: msg = "No daos_agent process killed on {}!".format(agent_host_kill) errors.append(msg) @@ -241,7 +242,7 @@ def test_agent_failure_isolation(self): # stopped. results = get_journalctl( hosts=NodeSet(agent_host_kill), since=since, until=until, - journalctl_type="daos_agent") + journalctl_type="daos_agent", run_user=self.test_env.agent_user) self.log.info("journalctl results (kill) = %s", results) if "shutting down" not in results[0]["data"]: msg = ("Agent shut down message not found in journalctl on killed client! " @@ -252,7 +253,7 @@ def test_agent_failure_isolation(self): # in the previous step doesn't show that the agent is stopped. results = get_journalctl( hosts=NodeSet(agent_host_keep), since=since, until=until, - journalctl_type="daos_agent") + journalctl_type="daos_agent", run_user=self.test_env.agent_user) self.log.info("journalctl results (keep) = %s", results) if "shutting down" in results[0]["data"]: msg = ("Agent shut down message found in journalctl on keep client! " diff --git a/src/tests/ftest/util/general_utils.py b/src/tests/ftest/util/general_utils.py index 941697f8af0..9a4f82de62d 100644 --- a/src/tests/ftest/util/general_utils.py +++ b/src/tests/ftest/util/general_utils.py @@ -1011,7 +1011,8 @@ def percent_change(val1, val2): return math.nan -def get_journalctl_command(since, until=None, system=False, units=None, identifiers=None): +def get_journalctl_command(since, until=None, system=False, units=None, identifiers=None, + run_user="root"): """Get the journalctl command to capture all unit/identifier activity from since to until. Args: @@ -1023,12 +1024,13 @@ def get_journalctl_command(since, until=None, system=False, units=None, identifi None. identifiers (str/list, optional): show messages for the specified syslog identifier(s). Defaults to None. + run_user (str, optional): user to run as. Defaults to root Returns: str: journalctl command to capture all unit activity """ - command = ["sudo", os.path.join(os.sep, "usr", "bin", "journalctl")] + command = [os.path.join(os.sep, "usr", "bin", "journalctl")] if system: command.append("--system") for key, values in {"unit": units or [], "identifier": identifiers or []}.items(): @@ -1037,10 +1039,10 @@ def get_journalctl_command(since, until=None, system=False, units=None, identifi command.append("--since=\"{}\"".format(since)) if until: command.append("--until=\"{}\"".format(until)) - return " ".join(command) + return command_as_user(" ".join(command), run_user) -def get_journalctl(hosts, since, until, journalctl_type): +def get_journalctl(hosts, since, until, journalctl_type, run_user="root"): """Run the journalctl on the hosts. Args: @@ -1048,6 +1050,7 @@ def get_journalctl(hosts, since, until, journalctl_type): since (str): Start time to search the log. until (str): End time to search the log. journalctl_type (str): String to search in the log. -t param for journalctl. + run_user (str, optional): user to run as. Defaults to root Returns: list: a list of dictionaries containing the following key/value pairs: @@ -1055,7 +1058,9 @@ def get_journalctl(hosts, since, until, journalctl_type): "data": data requested for the group of hosts """ - command = get_journalctl_command(since, until, True, identifiers=journalctl_type) + system = run_user != getuser() + command = get_journalctl_command( + since, until, system, identifiers=journalctl_type, run_user=run_user) err = "Error gathering system log events" return get_host_data(hosts=hosts, command=command, text="journalctl", error=err) diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index a6129bd1d41..3a0510127c2 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -541,7 +541,7 @@ def find_command(source, pattern, depth, other=None): def stop_processes(log, hosts, pattern, verbose=True, timeout=60, exclude=None, force=False, - full_command=False): + full_command=False, user="root"): """Stop the processes on each hosts that match the pattern. Args: @@ -602,9 +602,9 @@ def stop_processes(log, hosts, pattern, verbose=True, timeout=60, exclude=None, log.debug( "Killing%s any processes on %s that match %s and then waiting %s seconds", step[0], result.passed_hosts, pattern_match, step[1]) - kill_command = f"sudo /usr/bin/pkill{step[0]} {pattern}" + kill_command = command_as_user(f"/usr/bin/pkill{step[0]} {pattern}", user) if full_command: - kill_command = f"sudo /usr/bin/pkill{step[0]} --full -x {pattern}" + kill_command = command_as_user(f"/usr/bin/pkill{step[0]} --full -x {pattern}", user) run_remote(log, result.passed_hosts, kill_command, verbose, timeout) time.sleep(step[1]) result = run_remote(log, result.passed_hosts, search_command, verbose, timeout) From 7f9812c88ca431353053ba98daf997cd22c337ce Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Mon, 10 Nov 2025 15:12:03 -0800 Subject: [PATCH 21/43] DAOS-17461 cq: speedup githook submodule check (#16306) (#16442) Re-write the check so it does not re-run the same subprocess commands. Signed-off-by: Dalton Bohning --- .../10-submodule-update-check.py | 56 +++++++++++-------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/utils/githooks/prepare-commit-msg.d/10-submodule-update-check.py b/utils/githooks/prepare-commit-msg.d/10-submodule-update-check.py index 34a30c55c6e..561ae503c1c 100755 --- a/utils/githooks/prepare-commit-msg.d/10-submodule-update-check.py +++ b/utils/githooks/prepare-commit-msg.d/10-submodule-update-check.py @@ -7,8 +7,6 @@ import subprocess # nosec import sys -modified_re = re.compile(r'^(?:M|A)(\s+)(?P.*)') - def rebasing(): """Determines if the current operation is a rebase""" @@ -20,39 +18,53 @@ def rebasing(): return stdout.split('\n', maxsplit=1)[0].startswith("* (no branch, rebasing") -def submodule_check(modname, msg_file): - """Determines if a sub-module has been updated""" - modified = False +def git_submodules(): + """Get a list of submodules""" + lines = subprocess.check_output(['git', 'submodule', 'status']).decode().rstrip().split('\n') + return [ + line[1:].split(' ')[1] + for line in lines + if line + ] + +def git_modified_files(): + """Get a list of modified files""" + modified_re = re.compile(r'^(?:M|A)(\s+)(?P.*)') + files = [] with subprocess.Popen(['git', 'status', '--porcelain'], stdout=subprocess.PIPE) as proc: out, _ = proc.communicate() for line in out.decode().splitlines(): match = modified_re.match(line) if match: - modified = modified | (match.group('name') == modname) + files.append(match.group('name')) + return files - if not rebasing() and modified: - with open(msg_file, 'r', encoding='utf-8') as file: - lines = file.readlines() - message = f'# WARNING *** This patch modifies the {modname} reference. ' \ - 'Are you sure this is intended? *** WARNING' +def main(msg_file): + """main""" + # Ignore if this is a rebase + if rebasing(): + return + + modified_files = git_modified_files() + submodules = git_submodules() - if lines[0] != message: - lines = [message, "\n", "\n"] + lines + messages = [] + for modified_submodule in set(modified_files).intersection(submodules): + messages.append( + f'# WARNING *** This patch modifies the {modified_submodule} reference. ' + 'Are you sure this is intended? *** WARNING\n\n') - with open(msg_file, 'w', encoding='utf-8') as file: - file.writelines(lines) + if messages: + with open(msg_file, 'r', encoding='utf-8') as file: + lines = file.readlines() + lines = messages + lines -def main(msg_file): - """main""" - for line in subprocess.check_output(['git', 'submodule', - 'status']).decode().rstrip().split('\n'): - if line: - submodule_check(line[1:].split(' ')[1], msg_file) + with open(msg_file, 'w', encoding='utf-8') as file: + file.writelines(lines) if __name__ == '__main__': - main(sys.argv[1]) From b54fe5e0f54ab7fa19c5fdc7c53a8d6382848c3e Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Mon, 10 Nov 2025 15:13:28 -0800 Subject: [PATCH 22/43] DAOS-17406 cq: use cancel-in-progress for more workflows (#16244) (#16266) So previous runs are cancelled on new pushes. Signed-off-by: Dalton Bohning --- .github/workflows/bash_unit_testing.yml | 2 +- .github/workflows/bullseye-coverage.yml | 2 +- .github/workflows/ci2.yml | 2 +- .github/workflows/landing-builds.yml | 4 ++++ .github/workflows/linting.yml | 4 ++++ .github/workflows/rpm-build-and-test.yml | 2 +- .github/workflows/unit-testing.yml | 4 ++++ 7 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bash_unit_testing.yml b/.github/workflows/bash_unit_testing.yml index 964f89bf0b7..48b5d4ed149 100644 --- a/.github/workflows/bash_unit_testing.yml +++ b/.github/workflows/bash_unit_testing.yml @@ -4,7 +4,7 @@ on: pull_request: concurrency: - group: bash-unit-test-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true defaults: diff --git a/.github/workflows/bullseye-coverage.yml b/.github/workflows/bullseye-coverage.yml index 10520ecb501..aa553d87a36 100644 --- a/.github/workflows/bullseye-coverage.yml +++ b/.github/workflows/bullseye-coverage.yml @@ -47,7 +47,7 @@ on: concurrency: - group: bullseye-coverage-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true defaults: diff --git a/.github/workflows/ci2.yml b/.github/workflows/ci2.yml index 1ca3676d876..c49e46f3785 100644 --- a/.github/workflows/ci2.yml +++ b/.github/workflows/ci2.yml @@ -4,7 +4,7 @@ on: pull_request: concurrency: - group: ci2-${{ github.head_ref }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true permissions: {} diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index 3957819e14a..22f5eb2a3c8 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -18,6 +18,10 @@ on: - requirements-utest.txt - utils/build.config +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + permissions: {} jobs: diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 90ed021182f..227e1e70103 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -9,6 +9,10 @@ on: - 'release/*' pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + permissions: {} jobs: diff --git a/.github/workflows/rpm-build-and-test.yml b/.github/workflows/rpm-build-and-test.yml index cf26d623563..7ff09072e6f 100644 --- a/.github/workflows/rpm-build-and-test.yml +++ b/.github/workflows/rpm-build-and-test.yml @@ -54,7 +54,7 @@ on: - 'release/*' concurrency: - group: rpm-build-and-test-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true defaults: diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index b09fec29579..088579e2998 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -5,6 +5,10 @@ on: permissions: {} +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: Run_in_docker: if: github.repository == 'daos-stack/daos' From 5920e477d242c22f6e48cb39821dda57abd49699 Mon Sep 17 00:00:00 2001 From: Li Wei Date: Tue, 11 Nov 2025 09:43:45 +0900 Subject: [PATCH 23/43] DAOS-17659 control: Enable setting property value with multiple strings (#16503) (#16703) Fix issue where a property value containing multiple comma-separated strings cannot be set e.g. self_heal:exclude,rebuild. New syntax for key-value positional argument: key:val[,key:val1[;val2...]...] e.g. dmg pool set-prop bob self_heal:"exclude;rebuild" Also includes the fix for DAOS-17666. Signed-off-by: Tom Nabarro Signed-off-by: Li Wei Co-authored-by: Tom Nabarro --- src/control/cmd/dmg/pool.go | 2 +- src/control/cmd/dmg/pool_test.go | 28 ++++++++++++++++++ src/control/lib/control/pool.go | 13 ++++++++- src/control/lib/daos/pool_property.go | 13 +++++---- src/control/lib/daos/pool_property_test.go | 25 ++++++++-------- src/control/lib/ui/prop_flags_test.go | 3 +- src/pool/srv_pool.c | 34 +++++++++++++++++++++- src/tests/suite/daos_rebuild.c | 5 ++-- src/tests/suite/daos_rebuild_ec.c | 3 +- 9 files changed, 100 insertions(+), 26 deletions(-) diff --git a/src/control/cmd/dmg/pool.go b/src/control/cmd/dmg/pool.go index da40fc93abf..d181c88d161 100644 --- a/src/control/cmd/dmg/pool.go +++ b/src/control/cmd/dmg/pool.go @@ -799,7 +799,7 @@ type poolSetPropCmd struct { poolCmd Args struct { - Props PoolSetPropsFlag `positional-arg-name:"" required:"1"` + Props PoolSetPropsFlag `positional-arg-name:"" required:"1"` } `positional-args:"yes"` } diff --git a/src/control/cmd/dmg/pool_test.go b/src/control/cmd/dmg/pool_test.go index b624cf791d6..325d24ee75d 100644 --- a/src/control/cmd/dmg/pool_test.go +++ b/src/control/cmd/dmg/pool_test.go @@ -733,6 +733,20 @@ func TestPoolCommands(t *testing.T) { }, " "), nil, }, + { + "Set pool properties with semi-colon separated self_heal value", + `pool set-prop 031bcaf8-f0f5-42ef-b3c5-ee048676dceb self_heal:exclude;rebuild,space_rb:42`, + strings.Join([]string{ + printRequest(t, &control.PoolSetPropReq{ + ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb", + Properties: []*daos.PoolProperty{ + propWithVal("self_heal", "exclude;rebuild"), + propWithVal("space_rb", "42"), + }, + }), + }, " "), + nil, + }, { "Set pool properties with pool flag", "pool set-prop 031bcaf8-f0f5-42ef-b3c5-ee048676dceb label:foo,space_rb:42", @@ -809,6 +823,20 @@ func TestPoolCommands(t *testing.T) { }, " "), nil, }, + { + "Get pool properties", + "pool get-prop 031bcaf8-f0f5-42ef-b3c5-ee048676dceb label,self_heal", + strings.Join([]string{ + printRequest(t, &control.PoolGetPropReq{ + ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb", + Properties: []*daos.PoolProperty{ + propWithVal("label", ""), + propWithVal("self_heal", ""), + }, + }), + }, " "), + nil, + }, { "Get pool ACL", "pool get-acl 031bcaf8-f0f5-42ef-b3c5-ee048676dceb", diff --git a/src/control/lib/control/pool.go b/src/control/lib/control/pool.go index e9986ed72c7..d989f96c3d1 100644 --- a/src/control/lib/control/pool.go +++ b/src/control/lib/control/pool.go @@ -12,6 +12,7 @@ import ( "encoding/json" "fmt" "math" + "slices" "sort" "strings" "time" @@ -667,14 +668,23 @@ type PoolGetPropReq struct { // PoolGetProp sends a pool get-prop request to the pool service leader. func PoolGetProp(ctx context.Context, rpcClient UnaryInvoker, req *PoolGetPropReq) ([]*daos.PoolProperty, error) { - // Get all by default. if len(req.Properties) == 0 { + // Get all by default. allProps := daos.PoolProperties() req.Properties = make([]*daos.PoolProperty, 0, len(allProps)) for _, key := range allProps.Keys() { hdlr := allProps[key] req.Properties = append(req.Properties, hdlr.GetProperty(key)) } + } else { + for _, prop := range req.Properties { + if prop == nil { + return nil, errors.New("nil property") + } + } + slices.SortFunc(req.Properties, func(a, b *daos.PoolProperty) int { + return strings.Compare(a.Name, b.Name) + }) } pbReq := &mgmtpb.PoolGetPropReq{ @@ -714,6 +724,7 @@ func PoolGetProp(ctx context.Context, rpcClient UnaryInvoker, req *PoolGetPropRe pbMap[prop.GetNumber()] = prop } + // We've already sorted req.Properties above. resp := make([]*daos.PoolProperty, 0, len(req.Properties)) for _, prop := range req.Properties { pbProp, found := pbMap[prop.Number] diff --git a/src/control/lib/daos/pool_property.go b/src/control/lib/daos/pool_property.go index 0d8a8d1e24c..bdb4117e327 100644 --- a/src/control/lib/daos/pool_property.go +++ b/src/control/lib/daos/pool_property.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2023 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -59,9 +60,9 @@ func PoolProperties() PoolPropertyMap { case PoolSelfHealingDelayRebuild: return "delay_rebuild" case PoolSelfHealingAutoExclude | PoolSelfHealingAutoRebuild: - return "exclude,rebuild" + return "exclude;rebuild" case PoolSelfHealingAutoExclude | PoolSelfHealingDelayRebuild: - return "exclude,delay_rebuild" + return "exclude;delay_rebuild" default: return "unknown" } @@ -71,10 +72,10 @@ func PoolProperties() PoolPropertyMap { "exclude": PoolSelfHealingAutoExclude, "rebuild": PoolSelfHealingAutoRebuild, "delay_rebuild": PoolSelfHealingDelayRebuild, - "exclude,rebuild": PoolSelfHealingAutoExclude | PoolSelfHealingAutoRebuild, - "rebuild,exclude": PoolSelfHealingAutoExclude | PoolSelfHealingAutoRebuild, - "delay_rebuild,exclude": PoolSelfHealingAutoExclude | PoolSelfHealingDelayRebuild, - "exclude,delay_rebuild": PoolSelfHealingAutoExclude | PoolSelfHealingDelayRebuild, + "exclude;rebuild": PoolSelfHealingAutoExclude | PoolSelfHealingAutoRebuild, + "rebuild;exclude": PoolSelfHealingAutoExclude | PoolSelfHealingAutoRebuild, + "delay_rebuild;exclude": PoolSelfHealingAutoExclude | PoolSelfHealingDelayRebuild, + "exclude;delay_rebuild": PoolSelfHealingAutoExclude | PoolSelfHealingDelayRebuild, }, }, "space_rb": { diff --git a/src/control/lib/daos/pool_property_test.go b/src/control/lib/daos/pool_property_test.go index e9742a07c05..9e1a4b2e040 100644 --- a/src/control/lib/daos/pool_property_test.go +++ b/src/control/lib/daos/pool_property_test.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2023 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -174,23 +175,23 @@ func TestControl_PoolProperties(t *testing.T) { expStr: "self_heal:delay_rebuild", expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"delay_rebuild"}`), }, - "self_heal-exclude,rebuild": { + "self_heal-exclude;rebuild": { name: "self_heal", - value: "exclude,rebuild", - expStr: "self_heal:exclude,rebuild", - expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude,rebuild"}`), + value: "exclude;rebuild", + expStr: "self_heal:exclude;rebuild", + expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude;rebuild"}`), }, - "self_heal-rebuild,exclude": { + "self_heal-rebuild;exclude": { name: "self_heal", - value: "rebuild,exclude", - expStr: "self_heal:exclude,rebuild", - expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude,rebuild"}`), + value: "rebuild;exclude", + expStr: "self_heal:exclude;rebuild", + expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude;rebuild"}`), }, - "self_heal-exclude,delay_rebuild": { + "self_heal-exclude;delay_rebuild": { name: "self_heal", - value: "exclude,delay_rebuild", - expStr: "self_heal:exclude,delay_rebuild", - expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude,delay_rebuild"}`), + value: "exclude;delay_rebuild", + expStr: "self_heal:exclude;delay_rebuild", + expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude;delay_rebuild"}`), }, "self_heal-invalid": { name: "self_heal", diff --git a/src/control/lib/ui/prop_flags_test.go b/src/control/lib/ui/prop_flags_test.go index 4e9424a6680..c013047d893 100644 --- a/src/control/lib/ui/prop_flags_test.go +++ b/src/control/lib/ui/prop_flags_test.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2023 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -105,7 +106,6 @@ func TestUI_SetPropertiesFlag_UnmarshalFlag(t *testing.T) { } -/* NB: Doesn't work with go < 1.18 func FuzzUI_SetPropertiesFlag_UnmarshalFlag(f *testing.F) { f.Fuzz(func(t *testing.T, fv string) { f := ui.SetPropertiesFlag{} @@ -118,7 +118,6 @@ func FuzzUI_SetPropertiesFlag_UnmarshalFlag(f *testing.F) { } }) } -*/ func TestUI_SetPropertiesFlag_Complete(t *testing.T) { comps := ui.CompletionMap{ diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 456bf5ec89d..be1e80535cb 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -1527,7 +1527,7 @@ handle_event(struct pool_svc *svc, struct pool_svc_event_set *event_set) if (!pool_disable_exclude) { rc = pool_svc_exclude_ranks(svc, event_set); if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to exclude ranks", DP_UUID(svc->ps_uuid)); + DL_INFO(rc, DF_UUID ": exclude ranks", DP_UUID(svc->ps_uuid)); return rc; } } @@ -5527,6 +5527,15 @@ ds_pool_prop_set_handler(crt_rpc_t *rpc) "%d.\n", DP_UUID(in->psi_op.pi_uuid), rc); daos_prop_free(prop); } + /* If self_heal.exclude is set, resume event handling. */ + if (rc == 0 && !dup_op) { + struct daos_prop_entry *entry; + + entry = daos_prop_entry_get(prop_in, DAOS_PROP_PO_SELF_HEAL); + if (entry != NULL && daos_prop_is_set(entry) && + entry->dpe_val & DAOS_SELF_HEAL_AUTO_EXCLUDE) + resume_event_handling(svc); + } out_svc: ds_rsvc_set_hint(&svc->ps_rsvc, &out->pso_op.po_hint); pool_svc_put_leader(svc); @@ -6997,6 +7006,29 @@ pool_svc_update_map_internal(struct pool_svc *svc, unsigned int opc, bool exclud goto out; ABT_rwlock_wrlock(svc->ps_lock); + if (opc == MAP_EXCLUDE && src == MUS_SWIM) { + daos_prop_t *prop; + struct daos_prop_entry *entry; + + /* Check if self_heal.exclude is enabled. */ + rc = pool_prop_read(&tx, svc, DAOS_PO_QUERY_PROP_SELF_HEAL, &prop); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to read self_heal property", + DP_UUID(svc->ps_uuid)); + goto out_lock; + } + entry = daos_prop_entry_get(prop, DAOS_PROP_PO_SELF_HEAL); + D_ASSERT(entry != NULL); + if (!daos_prop_is_set(entry) || !(entry->dpe_val & DAOS_SELF_HEAL_AUTO_EXCLUDE)) { + D_INFO(DF_UUID ": not excluding for SWIM: self_heal.exclude not enabled\n", + DP_UUID(svc->ps_uuid)); + rc = -DER_NO_PERM; + daos_prop_free(prop); + goto out_lock; + } + daos_prop_free(prop); + } + /* Create a temporary pool map based on the last committed version. */ rc = read_map(&tx, &svc->ps_root, &map); if (rc != 0) diff --git a/src/tests/suite/daos_rebuild.c b/src/tests/suite/daos_rebuild.c index 20a751504ac..a30e6f1c643 100644 --- a/src/tests/suite/daos_rebuild.c +++ b/src/tests/suite/daos_rebuild.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1473,7 +1474,7 @@ rebuild_delay_and_reintegrate(void **state) if (!test_runable(arg, 6)) return; - rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "delay_rebuild"); + rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "exclude\\;delay_rebuild"); assert_rc_equal(rc, 0); for (i = 0; i < OBJ_NR; i++) { oids[i] = daos_test_oid_gen(arg->coh, DAOS_OC_R3S_SPEC_RANK, 0, 0, arg->myrank); @@ -1507,7 +1508,7 @@ rebuild_delay_and_extend(void **state) if (!test_runable(arg, 6)) return; - rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "delay_rebuild"); + rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "exclude\\;delay_rebuild"); assert_rc_equal(rc, 0); for (i = 0; i < OBJ_NR; i++) { oids[i] = daos_test_oid_gen(arg->coh, DAOS_OC_R3S_SPEC_RANK, 0, 0, arg->myrank); diff --git a/src/tests/suite/daos_rebuild_ec.c b/src/tests/suite/daos_rebuild_ec.c index 77df81730bd..b640a3df3de 100644 --- a/src/tests/suite/daos_rebuild_ec.c +++ b/src/tests/suite/daos_rebuild_ec.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1366,7 +1367,7 @@ rebuild_ec_combined_ops(void **state) if (!test_runable(arg, 6)) return; - rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "delay_rebuild"); + rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "exclude\\;delay_rebuild"); assert_int_equal(rc, 0); oid = daos_test_oid_gen(arg->coh, OC_EC_4P2GX, 0, 0, arg->myrank); ioreq_init(&req, arg->coh, oid, DAOS_IOD_ARRAY, arg); From 08d59091eefe0948dd933b44f86f39532c014c30 Mon Sep 17 00:00:00 2001 From: Niu Yawei Date: Wed, 12 Nov 2025 09:23:27 +0800 Subject: [PATCH 24/43] DAOS-17628 bio: flush WAL header before unmap (#16478) (#16663) In bio_wal_checkpoint(), we shouldn't unmap the checkpoint-ed region before flushing WAL header (to make the last checkpoint-ed ID persistent), otherwise, if the engine is interrupted in between the unmap and flush, next WAL replay on engine start will replay from the stale checkpoint-ed ID where the WAL tx data is already cleared by unmap. Signed-off-by: Niu Yawei --- src/bio/bio_wal.c | 56 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/src/bio/bio_wal.c b/src/bio/bio_wal.c index 6c99a203966..e82e73c0eef 100644 --- a/src/bio/bio_wal.c +++ b/src/bio/bio_wal.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2018-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1198,44 +1199,63 @@ tx_known_committed(struct wal_super_info *si, uint64_t tx_id) return (wal_id_cmp(si, tx_id, si->si_commit_id) <= 0); } +static inline void +dump_tx_hdr(struct wal_super_info *si, struct wal_trans_head *hdr) +{ + D_ERROR("[WAL header] ckp_id:" DF_U64 "/%u, commit_id:" DF_U64 "/%u\n", si->si_ckp_id, + si->si_ckp_blks, si->si_commit_id, si->si_commit_blks); + D_ERROR("[TX header] magic:%x, gen:%u, id:" DF_U64 ", ents:%u, payload:%u\n", hdr->th_magic, + hdr->th_gen, hdr->th_id, hdr->th_tot_ents, hdr->th_tot_payload); +} + static int verify_tx_hdr(struct wal_super_info *si, struct wal_trans_head *hdr, uint64_t tx_id) { bool committed = tx_known_committed(si, tx_id); + int rc = 0; if (hdr->th_magic != WAL_HDR_MAGIC) { D_CDEBUG(committed, DLOG_ERR, DB_IO, "Mismatched WAL head magic, %x != %x\n", hdr->th_magic, WAL_HDR_MAGIC); - return committed ? -DER_INVAL : 1; + rc = committed ? -DER_INVAL : 1; + goto out; } if (hdr->th_gen != si->si_header.wh_gen) { D_CDEBUG(committed, DLOG_ERR, DB_IO, "Mismatched WAL generation, %u != %u\n", hdr->th_gen, si->si_header.wh_gen); - return committed ? -DER_INVAL : 1; + rc = committed ? -DER_INVAL : 1; + goto out; } if (id2seq(hdr->th_id) < id2seq(tx_id)) { D_CDEBUG(committed, DLOG_ERR, DB_IO, "Stale sequence number detected, %u < %u\n", id2seq(hdr->th_id), id2seq(tx_id)); - return committed ? -DER_INVAL : 1; + rc = committed ? -DER_INVAL : 1; + goto out; } else if (id2seq(hdr->th_id) > id2seq(tx_id)) { D_ERROR("Invalid sequence number detected, %u > %u\n", id2seq(hdr->th_id), id2seq(tx_id)); - return -DER_INVAL; + rc = -DER_INVAL; + goto out; } if (hdr->th_id != tx_id) { D_ERROR("Mismatched transaction ID. "DF_U64" != "DF_U64"\n", hdr->th_id, tx_id); - return -DER_INVAL; + rc = -DER_INVAL; + goto out; } if (hdr->th_tot_ents == 0) { D_ERROR("Invalid entry number\n"); - return -DER_INVAL; + rc = -DER_INVAL; + goto out; } +out: + if (rc < 0) + dump_tx_hdr(si, hdr); - return 0; + return rc; } static int @@ -1795,7 +1815,7 @@ bio_wal_replay(struct bio_meta_context *mc, struct bio_wal_rp_stats *wrs, wrs->wrs_tx_cnt = total_tx; } } else { - D_ERROR("WAL replay failed, "DF_RC"\n", DP_RC(rc)); + DL_ERROR(rc, "WAL replay failed, nr_replayed:%u", nr_replayed); } D_FREE(dbuf); @@ -1841,19 +1861,21 @@ bio_wal_checkpoint(struct bio_meta_context *mc, uint64_t tx_id, uint64_t *purged unmap_start = id2off(wal_next_id(si, si->si_ckp_id, si->si_ckp_blks)); unmap_end = id2off(wal_next_id(si, tx_id, blk_desc.bd_blks)); - /* Unmap the checkpointed regions */ - rc = unmap_wal(mc, unmap_start, unmap_end, purged_blks); - if (rc) /* Flush the WAL header anyway */ - D_ERROR("Unmap checkpointed region failed. "DF_RC"\n", DP_RC(rc)); - si->si_ckp_id = tx_id; si->si_ckp_blks = blk_desc.bd_blks; - wakeup_reserve_waiters(si, false); - /* Flush the WAL header */ rc = bio_wal_flush_header(mc); - if (rc) - D_ERROR("Flush WAL header failed. "DF_RC"\n", DP_RC(rc)); + if (rc) { + DL_ERROR(rc, "Flush WAL header failed."); + goto out; + } + + /* Unmap the checkpointed region */ + rc = unmap_wal(mc, unmap_start, unmap_end, purged_blks); + if (rc) /* Wakeup waiters even if unmap failed */ + DL_ERROR(rc, "Unmap checkpointed region failed."); + + wakeup_reserve_waiters(si, false); out: D_FREE(buf); return rc; From 924eba48bf37b9099c8470db3d1ecc9af210c05e Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 12 Nov 2025 15:30:27 +0100 Subject: [PATCH 25/43] DAOS-18065 ci: empty /sys/class/nvme/ on leap15 when no NVMe device attached (#16956) (#17013) On leap15 the /sys/class/nvme/ folder exists but it is empty when no NVMe device attached. Signed-off-by: Tomasz Gromadzki --- ci/provisioning/post_provision_config_nodes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 47bacfe4ec3..2b271d922ff 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -99,7 +99,7 @@ function mount_nvme_drive { nvme_class="/sys/class/nvme/" function nvme_limit { set +x - if [ ! -d /sys/class/nvme ]; then + if [ ! -d "${nvme_class}" ] || [ -z "$(ls -A "${nvme_class}")" ]; then echo "No NVMe devices found" return fi From c0cb06f0fdcf798f8d57c4c4d6b1a1e51a93a18c Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Wed, 12 Nov 2025 07:53:34 -0800 Subject: [PATCH 26/43] DAOS-17429 cq: use merge-base in clang-format (#16269) (#17102) Use the merge-base in the GHA clang-format instead of the tip of the target Signed-off-by: Dalton Bohning --- .github/workflows/linting.yml | 36 +++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 227e1e70103..4d904905d87 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python environment @@ -48,7 +48,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Run @@ -66,7 +66,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Check DAOS logging macro use. @@ -77,7 +77,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Check DAOS ftest tags. run: \[ ! -x src/tests/ftest/tags.py \] || ./src/tests/ftest/tags.py lint --verbose @@ -86,7 +86,7 @@ jobs: name: Flake8 check steps: - name: Check out source repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python environment @@ -119,7 +119,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Install doxygen @@ -140,7 +140,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 @@ -160,7 +160,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install extra python packages run: python3 -m pip install --requirement utils/cq/requirements.txt - name: Run check @@ -172,16 +172,24 @@ jobs: clang-format: name: Clang Format - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Check out source repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 - name: Pull via git run: git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Get merge-base ref + id: get_merge_base + run: | + ref="$(git merge-base HEAD ${{ github.event.pull_request.base.sha || github.ref }})" + echo "ref=$ref" >> $GITHUB_OUTPUT - name: Run check in docker uses: ./.github/actions/clang-format with: - target: origin/${{ github.event.pull_request.base.ref }} + target: ${{ steps.get_merge_base.outputs.ref }} - name: Export changes uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 if: failure() @@ -194,7 +202,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Check out source repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python environment @@ -211,7 +219,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Check out source repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 From fd3bd1ed3356a0a6c83346b1b1d8919790d3158a Mon Sep 17 00:00:00 2001 From: Nasf-Fan Date: Wed, 12 Nov 2025 23:59:41 +0800 Subject: [PATCH 27/43] DAOS-17977 vos: new parameter acts for sys_db_trav_cb_t - b26 (#16960) db_traverse() calls vos_iterate() to traverse the VOS_ITER_AKEY for related sysdb table. For each found item, it handles it via sys_db_trav_cb_t() that may remove such item from related sysdb table, such as for some CHK cleanup use case. If that happened, We will notify vos_iterate() logic via set "VOS_ITER_CB_DELETE" flag in the "acts" parameter; otherwise some item(s) in related sysdb table may be skipped in subsequent iteration. For CHK use case, that may affect new check instance to reset automatically. Signed-off-by: Fan Yong --- src/bio/smd/smd_device.c | 3 ++- src/bio/smd/smd_pool.c | 3 ++- src/bio/smd/tests/smd_ut.c | 3 ++- src/chk/chk_common.c | 12 ++++++++---- src/chk/chk_internal.h | 10 +++++++--- src/include/daos/sys_db.h | 5 +++-- src/tests/ftest/recovery/cat_recov_core.yaml | 2 +- src/vos/sys_db.c | 4 ++-- 8 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/bio/smd/smd_device.c b/src/bio/smd/smd_device.c index 5b44b38c984..cc14733b255 100644 --- a/src/bio/smd/smd_device.c +++ b/src/bio/smd/smd_device.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2018-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -218,7 +219,7 @@ smd_dev_get_by_tgt(uint32_t tgt_id, enum smd_dev_type st, struct smd_dev_info ** } static int -smd_dev_list_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) +smd_dev_list_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts) { struct smd_trav_data *td = args; struct smd_dev_info *info; diff --git a/src/bio/smd/smd_pool.c b/src/bio/smd/smd_pool.c index 94312e960e1..03bca402ddf 100644 --- a/src/bio/smd/smd_pool.c +++ b/src/bio/smd/smd_pool.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2018-2025 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -336,7 +337,7 @@ smd_rdb_get_blob_sz(uuid_t pool_id, uint64_t *blob_sz) } static int -smd_pool_list_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) +smd_pool_list_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts) { struct smd_trav_data *td = args; struct smd_pool_info *info; diff --git a/src/bio/smd/tests/smd_ut.c b/src/bio/smd/tests/smd_ut.c index 71d2028d194..01172853d43 100644 --- a/src/bio/smd/tests/smd_ut.c +++ b/src/bio/smd/tests/smd_ut.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2018-2025 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -159,7 +160,7 @@ db_traverse(struct sys_db *db, char *table, sys_db_trav_cb_t cb, void *args) d_iov_t key; d_iov_set(&key, chain->uc_key, chain->uc_key_size); - cb(db, table, &key, args); + cb(db, table, &key, args, NULL); } return 0; } diff --git a/src/chk/chk_common.c b/src/chk/chk_common.c index 39821ec6328..e461c2d370c 100644 --- a/src/chk/chk_common.c +++ b/src/chk/chk_common.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2022-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -529,7 +530,7 @@ chk_pool_stop_all(struct chk_instance *ins, uint32_t status, int *ret) } int -chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) +chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts) { struct chk_traverse_pools_args *ctpa = args; char *uuid_str = key->iov_buf; @@ -559,7 +560,7 @@ chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) } int -chk_pools_cleanup_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) +chk_pools_cleanup_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts) { char *uuid_str = key->iov_buf; struct chk_bookmark cbk; @@ -569,8 +570,11 @@ chk_pools_cleanup_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) D_GOTO(out, rc = 0); rc = chk_bk_fetch_pool(&cbk, uuid_str); - if (rc == 0) + if (rc == 0) { rc = chk_bk_delete_pool(uuid_str); + if (rc == 0) + *acts |= VOS_ITER_CB_DELETE; + } out: return rc == -DER_NONEXIST ? 0 : rc; @@ -677,7 +681,7 @@ chk_pools_load_list(struct chk_instance *ins, uint64_t gen, uint32_t flags, } int -chk_pools_load_from_db(struct sys_db *db, char *table, d_iov_t *key, void *args) +chk_pools_load_from_db(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts) { struct chk_traverse_pools_args *ctpa = args; struct chk_instance *ins = ctpa->ctpa_ins; diff --git a/src/chk/chk_internal.h b/src/chk/chk_internal.h index 9ab16b060b3..dc221ebf312 100644 --- a/src/chk/chk_internal.h +++ b/src/chk/chk_internal.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2022-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -688,6 +689,7 @@ extern btr_ops_t chk_pending_ops; extern btr_ops_t chk_rank_ops; extern btr_ops_t chk_cont_ops; +/* clang-format off */ /* chk_common.c */ void chk_ranks_dump(uint32_t rank_nr, d_rank_t *ranks); @@ -703,16 +705,17 @@ void chk_pool_stop_one(struct chk_instance *ins, uuid_t uuid, uint32_t status, u void chk_pool_stop_all(struct chk_instance *ins, uint32_t status, int *ret); -int chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args); +int chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts); -int chk_pools_cleanup_cb(struct sys_db *db, char *table, d_iov_t *key, void *args); +int chk_pools_cleanup_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts); int chk_pool_start_one(struct chk_instance *ins, uuid_t uuid, uint64_t gen); int chk_pools_load_list(struct chk_instance *ins, uint64_t gen, uint32_t flags, int pool_nr, uuid_t pools[], uint32_t *phase); -int chk_pools_load_from_db(struct sys_db *db, char *table, d_iov_t *key, void *args); +int chk_pools_load_from_db(struct sys_db *db, char *table, d_iov_t *key, void *args, + unsigned *acts); int chk_pools_update_bk(struct chk_instance *ins, uint32_t phase); @@ -883,6 +886,7 @@ int chk_traverse_pools(sys_db_trav_cb_t cb, void *args); void chk_vos_init(void); void chk_vos_fini(void); +/* clang-format on */ static inline bool chk_is_ins_reset(struct chk_instance *ins, uint32_t flags) diff --git a/src/include/daos/sys_db.h b/src/include/daos/sys_db.h index 850e9c4203a..91d85e735ab 100644 --- a/src/include/daos/sys_db.h +++ b/src/include/daos/sys_db.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2022 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -12,8 +13,8 @@ #include struct sys_db; -typedef int (*sys_db_trav_cb_t)(struct sys_db *db, char *table, d_iov_t *key, - void *args); +typedef int (*sys_db_trav_cb_t)(struct sys_db *db, char *table, d_iov_t *key, void *args, + unsigned *acts); #define SYS_DB_NAME_SZ 32 diff --git a/src/tests/ftest/recovery/cat_recov_core.yaml b/src/tests/ftest/recovery/cat_recov_core.yaml index 6cba4c6caf3..23200a8b403 100644 --- a/src/tests/ftest/recovery/cat_recov_core.yaml +++ b/src/tests/ftest/recovery/cat_recov_core.yaml @@ -1,6 +1,6 @@ hosts: test_servers: 4 -timeout: 5400 +timeout: 5700 server_config: name: daos_server engines_per_host: 2 diff --git a/src/vos/sys_db.c b/src/vos/sys_db.c index d1f6d4bce98..cd1f10179f2 100644 --- a/src/vos/sys_db.c +++ b/src/vos/sys_db.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2020-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -278,8 +279,7 @@ db_trav_cb(daos_handle_t ih, vos_iter_entry_t *entry, vos_iter_type_t type, { struct db_trav_args *ta = data; - return ta->ta_cb(ta->ta_db, ta->ta_table, &entry->ie_key, - ta->ta_cb_args); + return ta->ta_cb(ta->ta_db, ta->ta_table, &entry->ie_key, ta->ta_cb_args, acts); } static int From 3e541c9aa88133c3e346f30fc5c4dec00e974266 Mon Sep 17 00:00:00 2001 From: Nasf-Fan Date: Thu, 13 Nov 2025 00:12:49 +0800 Subject: [PATCH 28/43] DAOS-17822 chk: filter out repeated pools in the check list - b26 (#16625) The user/admin may specify repeated pool(s) in the check list via 'dmg check' command. The logic needs to handle such case to avoid invalid assertion. The patch also fixes chk dump logic to avoid out-of-boundary DRAM accessing. Signed-off-by: Fan Yong --- src/chk/chk_common.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/chk/chk_common.c b/src/chk/chk_common.c index e461c2d370c..03a8db41487 100644 --- a/src/chk/chk_common.c +++ b/src/chk/chk_common.c @@ -333,20 +333,15 @@ chk_ranks_dump(uint32_t rank_nr, d_rank_t *ranks) D_INFO("Ranks List:\n"); while (rank_nr >= 8) { - D_INFO("%8u %8u %8u %8u %8u %8u %8u %8u\n", - ranks[0], ranks[1], ranks[2], ranks[3], + D_INFO("%8u%8u%8u%8u%8u%8u%8u%8u\n", ranks[0], ranks[1], ranks[2], ranks[3], ranks[4], ranks[5], ranks[6], ranks[7]); rank_nr -= 8; ranks += 8; } if (rank_nr > 0) { - rc = snprintf(ptr, 79, "%8u", ranks[0]); - D_ASSERT(rc > 0); - ptr += rc; - - for (i = 1; i < rank_nr; i++) { - rc = snprintf(ptr, 79 - 8 * i, " %8u", ranks[i]); + for (i = 0; i < rank_nr; i++) { + rc = snprintf(ptr, 79 - 8 * i, "%8u", ranks[i]); D_ASSERT(rc > 0); ptr += rc; } @@ -614,6 +609,8 @@ chk_pools_load_list(struct chk_instance *ins, uint64_t gen, uint32_t flags, struct chk_bookmark cbk; char uuid_str[DAOS_UUID_STR_SIZE]; d_rank_t myrank = dss_self_rank(); + d_iov_t kiov; + d_iov_t riov; int i; int rc = 0; @@ -663,6 +660,16 @@ chk_pools_load_list(struct chk_instance *ins, uint64_t gen, uint32_t flags, if (rc == 0 && cbk.cb_phase == CHK__CHECK_SCAN_PHASE__CSP_DONE && ins->ci_is_leader) continue; + /* There may be repeated pool(s) in the list, filter out. */ + d_iov_set(&kiov, pools[i], sizeof(uuid_t)); + d_iov_set(&riov, NULL, 0); + rc = dbtree_lookup(ins->ci_pool_hdl, &kiov, &riov); + if (unlikely(rc == 0)) + continue; + + if (unlikely(rc != -DER_NONEXIST)) + break; + /* * Here, we only update the pool bookmark in DRAM, the caller will store the update * persistently sometime later. From fede8fd43e745523b2716f65575d73e085290e19 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Wed, 12 Nov 2025 10:00:33 -0800 Subject: [PATCH 29/43] DAOS-18101 test: support DAOS_TEST_SYSTEMD_PATH and LIBRARY_PATH (#16979) (#17104) Support passing DAOS_TEST_SYSTEMD_PATH and DAOS_TEST_SYSTEMD_LIBRARY_PATH to ftest. When set, this tells ftest to put these values into the systemd config for the server and agent. Signed-off-by: Dalton Bohning --- src/tests/ftest/util/environment_utils.py | 39 +++++++++++++++++++---- src/tests/ftest/util/launch_utils.py | 5 +-- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index e36d750500e..bed802347f3 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -1,5 +1,6 @@ """ (C) Copyright 2018-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -109,6 +110,7 @@ class TestEnvironment(): 'bullseye_file': 'COVFILE', 'daos_prefix': 'DAOS_TEST_PREFIX', 'agent_user': 'DAOS_TEST_AGENT_USER', + 'systemd_path': 'DAOS_TEST_SYSTEMD_PATH', 'systemd_library_path': 'DAOS_TEST_SYSTEMD_LIBRARY_PATH', 'control_config': 'DAOS_TEST_CONTROL_CONFIG', 'agent_config': 'DAOS_TEST_AGENT_CONFIG', @@ -120,7 +122,7 @@ def __init__(self): self.set_defaults(None) def set_defaults(self, logger, servers=None, clients=None, provider=None, insecure_mode=None, - agent_user=None, log_dir=None, systemd_lib_path=None): + agent_user=None, log_dir=None, systemd_path=None, systemd_library_path=None): """Set the default test environment variable values with optional inputs. Args: @@ -135,7 +137,8 @@ def set_defaults(self, logger, servers=None, clients=None, provider=None, insecu agent_user (str, optional): user account to use when running the daos_agent. Defaults to None. log_dir (str, optional): test log directory base path. Defaults to None. - systemd_lib_path (str, optional): systemd library path. Defaults to None. + systemd_path (str, optional): systemd path. Defaults to None. + systemd_library_path (str, optional): systemd library path. Defaults to None. Raises: TestEnvironmentException: if there are any issues setting environment variable default @@ -154,8 +157,10 @@ def set_defaults(self, logger, servers=None, clients=None, provider=None, insecu self.insecure_mode = insecure_mode if agent_user is not None: self.agent_user = agent_user - if systemd_lib_path is not None: - self.systemd_library_path = systemd_lib_path + if systemd_path is not None: + self.systemd_path = systemd_path + if systemd_library_path is not None: + self.systemd_library_path = systemd_library_path # Set defaults for any unset values if self.log_dir is None: @@ -520,6 +525,24 @@ def agent_user(self, value): """ self.__set_value('agent_user', value) + @property + def systemd_path(self): + """Get the systemd PATH. + + Returns: + str: the systemd PATH + """ + return os.environ.get(self.__ENV_VAR_MAP['systemd_path']) + + @systemd_path.setter + def systemd_path(self, value): + """Set the systemd PATH. + + Args: + value (str): the systemd PATH + """ + self.__set_value('systemd_path', value) + @property def systemd_library_path(self): """Get the systemd LD_LIBRARY_PATH. @@ -607,7 +630,8 @@ def config_file_directories(self): def set_test_environment(logger, test_env=None, servers=None, clients=None, provider=None, insecure_mode=False, details=None, agent_user=None, log_dir=None, - systemd_lib_path=None): + systemd_path=None, systemd_library_path=None): + # pylint: disable=too-many-arguments """Set up the test environment. Args: @@ -625,7 +649,8 @@ def set_test_environment(logger, test_env=None, servers=None, clients=None, prov agent_user (str, optional): user account to use when running the daos_agent. Defaults to None. log_dir (str, optional): test log directory base path. Defaults to None. - systemd_lib_path (str, optional): systemd library path. Defaults to None. + systemd_path (str, optional): systemd library path. Defaults to None. + systemd_library_path (str, optional): systemd library path. Defaults to None. Raises: TestEnvironmentException: if there is a problem setting up the test environment @@ -638,7 +663,7 @@ def set_test_environment(logger, test_env=None, servers=None, clients=None, prov # Get the default fabric interface, provider, and daos prefix test_env.set_defaults( logger, servers, clients, provider, insecure_mode, agent_user, log_dir, - systemd_lib_path) + systemd_path, systemd_library_path) logger.info("Testing with interface: %s", test_env.interface) logger.info("Testing with provider: %s", test_env.provider) logger.info("Testing with daos_prefix: %s", test_env.daos_prefix) diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index 0f7284c50ef..8001645bbfc 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -1,5 +1,6 @@ """ (C) Copyright 2022-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -129,12 +130,12 @@ def setup_systemctl(logger, servers, clients, test_env): __add_systemctl_override( logger, servers, "daos_server.service", "root", os.path.join(test_env.daos_prefix, "bin", "daos_server"), test_env.server_config, - None, None)) + test_env.systemd_path, test_env.systemd_library_path)) systemctl_configs.update( __add_systemctl_override( logger, clients, "daos_agent.service", test_env.agent_user, os.path.join(test_env.daos_prefix, "bin", "daos_agent"), test_env.agent_config, - None, None)) + test_env.systemd_path, test_env.systemd_library_path)) return systemctl_configs From e296d9ed4c70e0a9248a3c5fded6717c3e34d60b Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Wed, 12 Nov 2025 10:08:23 -0800 Subject: [PATCH 30/43] DAOS-17890 test: condense rebuild/basic.py (#16736) (#16816) Combine all the variants into one complete variant. This is closer to production and is more efficient. Signed-off-by: Dalton Bohning --- src/tests/ftest/rebuild/basic.py | 113 ++++++++++++----------------- src/tests/ftest/rebuild/basic.yaml | 49 +++++-------- 2 files changed, 63 insertions(+), 99 deletions(-) diff --git a/src/tests/ftest/rebuild/basic.py b/src/tests/ftest/rebuild/basic.py index 787e0f922e5..d1a19247067 100644 --- a/src/tests/ftest/rebuild/basic.py +++ b/src/tests/ftest/rebuild/basic.py @@ -1,5 +1,6 @@ """ (C) Copyright 2018-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -15,20 +16,23 @@ class RbldBasic(TestWithServers): :avocado: recursive """ - def run_rebuild_test(self, pool_quantity): - """Run the rebuild test for the specified number of pools. + def test_rebuild_basic(self): + """ + Test Description: + The most basic rebuild test. + + Use Cases: + Multiple pool rebuild, single client, various record/object counts - Args: - pool_quantity (int): number of pools to test + :avocado: tags=all,daily_regression + :avocado: tags=vm + :avocado: tags=rebuild,pool,daos_cmd + :avocado: tags=RbldBasic,test_rebuild_basic """ # Get the test parameters - pools = [] - containers = [] - for _ in range(pool_quantity): - pools.append(self.get_pool(create=False)) - containers.append(self.get_container(pools[-1], create=False)) - rank = self.params.get("rank", "/run/testparams/*") - obj_class = self.params.get("object_class", "/run/testparams/*") + rank = self.random.choice(list(self.server_managers[0].ranks.keys())) + datasets = self.params.get("datasets", "/run/testparams/*") + num_pools = self.params.get("num_pools", "/run/testparams/*") # Collect server configuration information server_count = len(self.hostlist_servers) @@ -39,26 +43,33 @@ def run_rebuild_test(self, pool_quantity): "Running with %s servers, %s engines per server, and %s targets per engine", server_count, engine_count, target_count) - # Create the pools and confirm their status + self.log_step(f"Create {num_pools} pools") + pools = [] status = True - for pool in pools: - pool.create() + for _ in range(num_pools): + pool = self.get_pool() + pools.append(pool) status &= pool.check_pool_info( pi_nnodes=server_count * engine_count, pi_ntargets=server_count * engine_count * target_count, - pi_ndisabled=0 - ) + pi_ndisabled=0) status &= pool.check_rebuild_status(rs_state=1, rs_obj_nr=0, rs_rec_nr=0, rs_errno=0) self.assertTrue(status, "Error confirming pool info before rebuild") - # Create containers in each pool and fill them with data - rs_obj_nr = [] - rs_rec_nr = [] - for container in containers: - container.create() - container.write_objects(rank, obj_class) + self.log_step(f"Create {len(datasets)} containers and fill with data") + containers = [] + for idx, dataset in enumerate(datasets): + # Round-robin create containers across pools + container = self.get_container(pools[idx % len(pools)]) + containers.append(container) + (oclass, data_size, object_qty, record_qty) = dataset + container.update_params( + data_size=data_size, object_qty=object_qty, record_qty=record_qty) + container.write_objects(rank, oclass) # Determine how many objects will need to be rebuilt + rs_obj_nr = [] + rs_rec_nr = [] for container in containers: target_rank_lists = container.get_target_rank_lists(" prior to rebuild") rebuild_qty = container.get_target_rank_count(rank, target_rank_lists) @@ -72,24 +83,21 @@ def run_rebuild_test(self, pool_quantity): rs_rec_nr[-1], container.object_qty.value * container.record_qty.value, container, rank) - # Manually exclude the specified rank - for index, pool in enumerate(pools): - if index == 0: - self.server_managers[0].stop_ranks([rank], self.d_log, True) - else: - # Use the direct dmg pool exclude command to avoid updating the pool version again - pool.exclude([rank]) + self.log_step(f"Exclude random rank {rank}") + self.server_managers[0].stop_ranks([rank], self.d_log, True) + for pool in pools: + pool.exclude([rank]) - # Wait for recovery to start for first pool. - pools[0].wait_for_rebuild_to_start() + self.log_step("Wait for rebuild to start") + pools[0].wait_for_rebuild_to_start(interval=3) - # Wait for recovery to complete + self.log_step("Wait for rebuild to end") for pool in pools: - pool.wait_for_rebuild_to_end() + pool.wait_for_rebuild_to_end(interval=3) - # Check the pool information after the rebuild + self.log_step("Verify pool info after rebuild") status = True - for index, pool in enumerate(pools): + for pool in pools: status &= pool.check_pool_info( pi_nnodes=server_count * engine_count, pi_ntargets=server_count * engine_count * target_count, @@ -99,41 +107,10 @@ def run_rebuild_test(self, pool_quantity): rs_state=2, rs_errno=0) self.assertTrue(status, "Error confirming pool info after rebuild") - # Verify the data after rebuild + self.log_step("Verify container data after rebuild") for container in containers: container.set_prop(prop="status", value="healthy") if container.object_qty.value != 0: self.assertTrue(container.read_objects(), "Data verification error after rebuild") - self.log.info("Test Passed") - - def test_simple_rebuild(self): - """JIRA ID: DAOS-XXXX Rebuild-001. - - Test Description: - The most basic rebuild test. - - Use Cases: - single pool rebuild, single client, various record/object counts - - :avocado: tags=all,daily_regression - :avocado: tags=vm - :avocado: tags=rebuild,pool,daos_cmd - :avocado: tags=RbldBasic,test_simple_rebuild - """ - self.run_rebuild_test(1) - - def test_multipool_rebuild(self): - """JIRA ID: DAOS-XXXX (Rebuild-002). - - Test Description: - Expand on the basic test by rebuilding 2 pools at once. - Use Cases: - multi-pool rebuild, single client, various object and record counts - - :avocado: tags=all,daily_regression - :avocado: tags=vm - :avocado: tags=rebuild,pool - :avocado: tags=RbldBasic,test_multipool_rebuild - """ - self.run_rebuild_test(self.params.get("pool_quantity", "/run/testparams/*")) + self.log_step("Test Passed") diff --git a/src/tests/ftest/rebuild/basic.yaml b/src/tests/ftest/rebuild/basic.yaml index 31d203f3036..80fbb15d9ce 100644 --- a/src/tests/ftest/rebuild/basic.yaml +++ b/src/tests/ftest/rebuild/basic.yaml @@ -1,8 +1,9 @@ -# change host names to your reserved nodes, the -# required quantity is indicated by the placeholders hosts: test_servers: 6 + test_clients: 1 + timeout: 600 + server_config: name: daos_server crt_timeout: 10 @@ -19,42 +20,28 @@ server_config: class: ram scm_mount: /mnt/daos system_ram_reserved: 1 + pool: size: 1G pool_query_timeout: 30 properties: rd_fac:2 + container: akey_size: 5 dkey_size: 5 debug: true properties: rd_fac:2 - data: !mux - small_data: - data_size: 32 - large_data: - data_size: 250 - objects: !mux - zero_objects: - # Don't run duplicate variants when there are zero objects - !filter-only : "/run/container/data/small_data" # yamllint disable-line rule:colons - !filter-only : "/run/container/records/one_record" # yamllint disable-line rule:colons - object_qty: 0 - one_object: - object_qty: 1 - twenty_objects: - object_qty: 20 - records: !mux - one_record: - record_qty: 1 - ten_records: - record_qty: 10 + testparams: - ranks: !mux - rank1: - rank: 1 - rank2: - rank: 2 - rank3: - rank: 3 - pool_quantity: 2 - object_class: OC_RP_3G1 + num_pools: 2 + datasets: + # oclass, data_size, object_qty, record_qty + - [OC_RP_3G1, 32, 0, 1] + - [OC_RP_3G1, 32, 1, 1] + - [OC_RP_3G1, 32, 1, 10] + - [OC_RP_3G1, 32, 20, 1] + - [OC_RP_3G1, 32, 20, 10] + - [OC_RP_3G1, 250, 1, 1] + - [OC_RP_3G1, 250, 1, 10] + - [OC_RP_3G1, 250, 20, 1] + - [OC_RP_3G1, 250, 20, 10] From bd7c85b329e5927455611d8781a06d2dc995f9d7 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Wed, 12 Nov 2025 11:44:58 -0700 Subject: [PATCH 31/43] DAOS-16565 control: Log component check at Trace level (#16329) (#16806) The component compatibility check is noisy, and reduces the usefulness of debug-level logging for customers. It's still useful for devs as a trace-level message. Signed-off-by: Kris Jacque --- src/control/server/interceptors.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/control/server/interceptors.go b/src/control/server/interceptors.go index b76041a0602..db116fb95fa 100644 --- a/src/control/server/interceptors.go +++ b/src/control/server/interceptors.go @@ -1,5 +1,6 @@ // // (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -192,7 +193,7 @@ func checkVersion(ctx context.Context, log logging.Logger, self *build.Versioned return FaultIncompatibleComponents(self, other) } - log.Debugf("%s is compatible with %s", other, self) + log.Tracef("%s is compatible with %s", other, self) return nil } From 8a20f44f33b6a6b763141699bb6e9d892f22846c Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Wed, 12 Nov 2025 11:56:58 -0700 Subject: [PATCH 32/43] DAOS-17643 control: Disallow most transitions from Excluded (#16475) (#17093) Excluded ranks should be able to move only to the Joined or AdminExcluded states. Regardless of the process state, without re-joining successfully, the rank is still effectively excluded. We saw a real life case where Excluded ranks were transitioning to the Errored state when they SIGKILLed themselves after discovering their exclusion. Signed-off-by: Kris Jacque --- src/control/system/member.go | 9 + src/control/system/member_internal_test.go | 198 +++++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 src/control/system/member_internal_test.go diff --git a/src/control/system/member.go b/src/control/system/member.go index 9ff5b6f720d..7edd34e1498 100644 --- a/src/control/system/member.go +++ b/src/control/system/member.go @@ -1,5 +1,6 @@ // // (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -161,6 +162,14 @@ func (ms MemberState) isTransitionIllegal(to MemberState) bool { MemberStateReady: true, MemberStateJoined: true, }, + MemberStateExcluded: { + MemberStateErrored: true, + MemberStateStopping: true, + MemberStateStopped: true, + MemberStateStarting: true, + MemberStateReady: true, + MemberStateUnresponsive: true, + }, }[ms][to] } diff --git a/src/control/system/member_internal_test.go b/src/control/system/member_internal_test.go new file mode 100644 index 00000000000..cf3aafad07c --- /dev/null +++ b/src/control/system/member_internal_test.go @@ -0,0 +1,198 @@ +// +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package system + +import ( + "fmt" + "testing" + + "github.com/daos-stack/daos/src/control/common/test" +) + +func TestSystem_MemberState_isTransitionIllegal(t *testing.T) { + nextState := func(state MemberState) MemberState { + if state == MemberStateUnknown { + return state + 1 + } + return state << 1 + } + + for name, tc := range map[string]struct { + start MemberState + expLegalStates []MemberState + }{ + "unknown": { + start: MemberStateUnknown, + expLegalStates: []MemberState{}, + }, + "admin excluded": { + start: MemberStateAdminExcluded, + expLegalStates: []MemberState{}, + }, + "await format": { + start: MemberStateAwaitFormat, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateJoined, + MemberStateReady, + MemberStateStarting, + MemberStateStopped, + MemberStateStopping, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "starting": { + start: MemberStateStarting, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateJoined, + MemberStateReady, + MemberStateStopped, + MemberStateStopping, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "ready": { + start: MemberStateReady, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateJoined, + MemberStateStarting, + MemberStateStopped, + MemberStateStopping, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "joined": { + start: MemberStateJoined, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateExcluded, + MemberStateStarting, + MemberStateStopped, + MemberStateStopping, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "errored": { + start: MemberStateErrored, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateExcluded, + MemberStateStarting, + MemberStateStopped, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "unresponsive": { + start: MemberStateUnresponsive, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateExcluded, + MemberStateStarting, + MemberStateStopped, + MemberStateUnknown, + }, + }, + "checker": { + start: MemberStateCheckerStarted, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateErrored, + MemberStateExcluded, + MemberStateStarting, + MemberStateStopping, + MemberStateStopped, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "excluded": { + start: MemberStateExcluded, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateJoined, + MemberStateCheckerStarted, + MemberStateUnknown, + }, + }, + "stopping": { + start: MemberStateStopping, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateExcluded, + MemberStateJoined, + MemberStateReady, + MemberStateStarting, + MemberStateStopped, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "stopped": { + start: MemberStateStopped, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateExcluded, + MemberStateJoined, + MemberStateReady, + MemberStateStarting, + MemberStateStopping, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + } { + expLegal := func(state MemberState) bool { + for _, l := range tc.expLegalStates { + if l == state { + return true + } + } + return false + } + + for end := MemberStateUnknown; end < MemberStateMax; end = nextState(end) { + caseName := fmt.Sprintf("%s/%s -> %s", name, tc.start, end) + + t.Run(caseName, func(t *testing.T) { + result := tc.start.isTransitionIllegal(end) + + test.AssertEqual(t, !expLegal(end), result, "") + }) + } + } +} From eda8d6bc74490f242f5e2b80b93560539f755b13 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 12 Nov 2025 16:42:44 -0500 Subject: [PATCH 33/43] SRE-3150 test: Add device verification and test summary to launch (#16583) (#16991) Verify the server storage devices exist before and after running the tests. Generate a test summary indicating the order, status, and host usage for each test file and its variants. Signed-off-by: Phil Henderson --- src/tests/ftest/launch.py | 5 +- src/tests/ftest/pool/create_all_hw.yaml | 179 ++++---------------- src/tests/ftest/server/storage_tiers.py | 8 +- src/tests/ftest/server/storage_tiers.yaml | 14 +- src/tests/ftest/util/collection_utils.py | 120 ++++++++++++- src/tests/ftest/util/command_utils_base.py | 11 +- src/tests/ftest/util/launch_utils.py | 125 ++++++++------ src/tests/ftest/util/server_utils_params.py | 8 +- 8 files changed, 265 insertions(+), 205 deletions(-) diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index f3daa8464d7..a279f7027d4 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """ (C) Copyright 2018-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -14,6 +15,7 @@ from collections import OrderedDict from tempfile import TemporaryDirectory +import distro from ClusterShell.NodeSet import NodeSet from process_core_files import get_core_file_pattern # pylint: disable=import-error,no-name-in-module @@ -176,6 +178,7 @@ def _configure(self, overwrite_config=False): # Add details about the run self.details["avocado version"] = str(self.avocado) + self.details["distro"] = distro.linux_distribution(full_distribution_name=True) self.details["launch host"] = str(self.local_host) def _create_log_dir(self): @@ -272,7 +275,7 @@ def _run(self, args): all_hosts = args.test_servers | args.test_clients | self.local_host self.details["installed packages"] = find_packages( logger, all_hosts, - "'^(daos|libfabric|mercury|ior|openmpi|mpifileutils|mlnx-ofed-basic)-'") + "'^(daos|libfabric|mercury|ior|openmpi|mpich|mpifileutils|mlnx-ofed-basic)-'") # Setup the test environment test_env = TestEnvironment() diff --git a/src/tests/ftest/pool/create_all_hw.yaml b/src/tests/ftest/pool/create_all_hw.yaml index a8070c93b5d..b47802955d6 100644 --- a/src/tests/ftest/pool/create_all_hw.yaml +++ b/src/tests/ftest/pool/create_all_hw.yaml @@ -1,7 +1,6 @@ setup: start_agents_once: False start_servers_once: False - server_config_namespace: /run/server_configurations/*/server_config/* hosts: test_servers: 3 @@ -39,148 +38,44 @@ test_two_pools_hw: scm: 2147483648 # 2GiB nvme: 0 -# Mux the number of targets per engine (1-4) with both engines using the same number of targets -server_configurations: !mux - server_config_1_target: - server_config: - name: daos_server - engines_per_host: 2 - engines: +server_config: + name: daos_server + engines_per_host: 2 + engines: + 0: + pinned_numa_node: 0 + fabric_iface: ib0 + fabric_iface_port: 31317 + log_file: daos_server0.log + storage: 0: - targets: 1 - pinned_numa_node: 0 - nr_xs_helpers: 1 - fabric_iface: ib0 - fabric_iface_port: 31317 - log_file: daos_server0.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem0"] - scm_mount: /mnt/daos0 - 1: - class: nvme - bdev_list: ["aaaa:aa:aa.a"] + class: dcpm + scm_list: ["/dev/pmem0"] + scm_mount: /mnt/daos0 1: - targets: 1 - pinned_numa_node: 1 - nr_xs_helpers: 1 - fabric_iface: ib1 - fabric_iface_port: 31417 - log_file: daos_server1.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem1"] - scm_mount: /mnt/daos1 - 1: - class: nvme - bdev_list: ["bbbb:bb:bb.b"] - server_config_2_target: - server_config: - name: daos_server - engines_per_host: 2 - engines: + class: nvme + bdev_list: ["aaaa:aa:aa.a"] + 1: + pinned_numa_node: 1 + fabric_iface: ib1 + fabric_iface_port: 31417 + log_file: daos_server1.log + storage: 0: - targets: 2 - pinned_numa_node: 0 - nr_xs_helpers: 1 - fabric_iface: ib0 - fabric_iface_port: 31317 - log_file: daos_server0.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem0"] - scm_mount: /mnt/daos0 - 1: - class: nvme - bdev_list: ["aaaa:aa:aa.a"] + class: dcpm + scm_list: ["/dev/pmem1"] + scm_mount: /mnt/daos1 1: - targets: 2 - pinned_numa_node: 1 - nr_xs_helpers: 1 - fabric_iface: ib1 - fabric_iface_port: 31417 - log_file: daos_server1.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem1"] - scm_mount: /mnt/daos1 - 1: - class: nvme - bdev_list: ["bbbb:bb:bb.b"] - server_config_3_target: - server_config: - name: daos_server - engines_per_host: 2 - engines: - 0: - targets: 3 - pinned_numa_node: 0 - nr_xs_helpers: 1 - fabric_iface: ib0 - fabric_iface_port: 31317 - log_file: daos_server0.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem0"] - scm_mount: /mnt/daos0 - 1: - class: nvme - bdev_list: ["aaaa:aa:aa.a"] - 1: - targets: 3 - pinned_numa_node: 1 - nr_xs_helpers: 1 - fabric_iface: ib1 - fabric_iface_port: 31417 - log_file: daos_server1.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem1"] - scm_mount: /mnt/daos1 - 1: - class: nvme - bdev_list: ["bbbb:bb:bb.b"] - server_config_5_target: - server_config: - name: daos_server - engines_per_host: 2 - engines: - 0: - targets: 5 - pinned_numa_node: 0 - nr_xs_helpers: 1 - fabric_iface: ib0 - fabric_iface_port: 31317 - log_file: daos_server0.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem0"] - scm_mount: /mnt/daos0 - 1: - class: nvme - bdev_list: ["aaaa:aa:aa.a"] - 1: - targets: 5 - pinned_numa_node: 1 - nr_xs_helpers: 1 - fabric_iface: ib1 - fabric_iface_port: 31417 - log_file: daos_server1.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem1"] - scm_mount: /mnt/daos1 - 1: - class: nvme - bdev_list: ["bbbb:bb:bb.b"] - -pool: - control_method: dmg + class: nvme + bdev_list: ["bbbb:bb:bb.b"] + engines_common: + targets_mux: !mux + one_target: + targets: 1 + two_targets: + targets: 2 + three_targets: + targets: 3 + five_targets: + targets: 5 + nr_xs_helpers: 1 diff --git a/src/tests/ftest/server/storage_tiers.py b/src/tests/ftest/server/storage_tiers.py index e627b8d62c2..0dd71d79b65 100644 --- a/src/tests/ftest/server/storage_tiers.py +++ b/src/tests/ftest/server/storage_tiers.py @@ -1,5 +1,6 @@ """ (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -25,9 +26,10 @@ def test_tiers(self): Test Description: Verify storage tiers are correctly obtained from the test yaml file. + Does not test or rely on actual hardware being present. :avocado: tags=all,pr - :avocado: tags=hw,medium + :avocado: tags=vm :avocado: tags=server,storage,storage_tiers :avocado: tags=StorageTiers,test_tiers """ @@ -58,9 +60,9 @@ def test_tiers(self): "class": storage_class, "bdev_list": self.params.get("bdev_list", namespace), } - storage_roles = self.params.get("roles", namespace, None) + storage_roles = self.params.get("bdev_roles", namespace, None) if storage_roles: - data["roles"] = storage_roles + data["bdev_roles"] = storage_roles storage.append(data) expected.append(storage) self.log.info("expected:\n%s", yaml.dump(expected, default_flow_style=False)) diff --git a/src/tests/ftest/server/storage_tiers.yaml b/src/tests/ftest/server/storage_tiers.yaml index 225bbae47e1..b9bc500e30f 100644 --- a/src/tests/ftest/server/storage_tiers.yaml +++ b/src/tests/ftest/server/storage_tiers.yaml @@ -5,7 +5,9 @@ hosts: setup: start_servers: false start_agents: false - server_config_namespace: /run/server_configurations/*/server_config/* + server_config_namespace: /run/configurations/*/server_config/* + +check_server_storage: false timeout: 1800 @@ -33,23 +35,23 @@ storage_1: &storage_ram storage_2: &storage_nvme_all class: nvme - roles: [wal, index, data] + bdev_roles: [wal, index, data] storage_3: &storage_nvme_wal_index class: nvme - roles: [wal, index] + bdev_roles: [wal, index] storage_4: &storage_nvme_wal class: nvme - roles: [wal] + bdev_roles: [wal] storage_5: &storage_nvme_index class: nvme - roles: [index] + bdev_roles: [index] storage_6: &storage_nvme_data class: nvme - roles: [data] + bdev_roles: [data] configurations: !mux 1_tier_dcpm: diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index a900769f4d6..7c0d3ccf08d 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -1,10 +1,12 @@ """ (C) Copyright 2022-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ # pylint: disable=too-many-lines import glob +import json import os import re import sys @@ -17,6 +19,7 @@ from util.environment_utils import TestEnvironment from util.host_utils import get_local_host from util.run_utils import find_command, run_local, run_remote, stop_processes +from util.storage_utils import find_pci_address from util.systemctl_utils import stop_service from util.user_utils import get_chown_command from util.yaml_utils import get_test_category @@ -154,6 +157,73 @@ def cleanup_processes(logger, test, result): return not any_found +def check_server_storage(logger, test, test_result, stage): + """Verify that the expected server storage devices exist. + + Args: + logger (Logger): logger for the messages produced by this method + test (TestInfo): the test information + test_result (TestResult): the test result used to update the status of the test + stage (str): current launch test execution stage + + Returns: + bool: True if all expected devices found on all server hosts; False otherwise + """ + status = True + commands = { + "scm_list": + r"ndctl list -c -v | grep pmem | sed -e 's/.*:\"\(.*\)\"/\/dev\/\1/' | grep -Eo '{}'", + "bdev_list": + r"lspci -D | grep -E '{}'" + } + logger.debug("-" * 80) + logger.debug(f"Verifying server storage during the {stage.lower()} stage for \'{test}\'") + if "check_server_storage" in test.yaml_info and test.yaml_info["check_server_storage"] is False: + logger.debug(" - Test requested skip via 'check_server_storage' yaml entry") + return status + for key, command in commands.items(): + if key not in test.yaml_info or test.yaml_info[key] is None: + # No need to check storage w/o a scm/bdev entry + logger.debug(f" - No {key} storage defined for this test") + continue + + # Convert single engine storage entries into lists + test_yaml_info = test.yaml_info[key] + if not isinstance(test_yaml_info, (list, tuple)): + test_yaml_info = [test_yaml_info] + + if "daos" in test_yaml_info: + # No need to check ram class storage (using /mnt/daos* scm) + logger.debug(f" - {key} storage using ram class") + continue + + result = run_remote( + logger, test.host_info.servers.hosts, command.format('|'.join(test_yaml_info))) + if not result.passed: + message = f"Failure detected attempting to verify {key} storage for '{test}'" + test_result.fail_test(logger, stage, message) + status = False + continue + + item_set = set(test_yaml_info) + logger.debug(f" - Expecting the following {key} storage: {item_set}") + for data in result.output: + if key == "bdev_list": + result_set = set(find_pci_address("\n".join(data.stdout))) + else: + result_set = set(data.stdout) + match = bool(item_set & result_set == item_set) + if not match: + logger.error( + f"Mismatch detected for {key}: expected:{item_set} != detected:{result_set}") + message = f"Mismatch detected verifying {key} storage for '{test}'" + test_result.fail_test(logger, stage, message) + status = False + else: + logger.debug(f"Storage verified: expected:{item_set} == detected:{result_set}") + return status + + def archive_files(logger, summary, hosts, source, pattern, destination, depth, threshold, timeout, test_result, test=None): # pylint: disable=too-many-arguments @@ -565,6 +635,46 @@ def create_steps_log(logger, job_results_dir, test_result): return 0 +def record_variant_details(logger, job_results_dir, test_result, details): + """Record variant test results to the details.json file. + + Args: + logger (Logger): logger for the messages produced by this method + job_results_dir (str): path to the avocado job results + test_result (TestResult): the test result used to update the status of the test + details (dict): dictionary to update with test results + + Returns: + int: status code: 16384 = problem parsing the results.json; 0 = success + """ + logger.debug("=" * 80) + logger.info("Recording variant details/results") + if "test_variants" not in details: + details["test_variants"] = [] + + test_logs_lnk = os.path.join(job_results_dir, "latest") + test_logs_dir = os.path.realpath(test_logs_lnk) + results_json = os.path.join(test_logs_dir, 'results.json') + try: + with open(results_json, "r", encoding="utf-8") as results: + data = json.loads(results.read()) + for index, test in enumerate(data["tests"]): + if len(details["test_variants"]) == index: + # Add an entry for this test variant the first time its run in the loop + details["test_variants"].append({ + "variant": test["id"].split(";")[0], + "status/loop": [], + "time/loop": []}) + # Append status and run times for each loop + details["test_variants"][index]["status/loop"].append(test["status"]) + details["test_variants"][index]["time/loop"].append(f"{round(test['time'], 2)}s") + except Exception: # pylint: disable=broad-except + message = f"Error parsing {results_json}" + test_result.fail_test(logger, "Process", message, sys.exc_info()) + return 16384 + return 0 + + def rename_avocado_test_dir(logger, test, job_results_dir, test_result, jenkins_xml, total_repeats): """Append the test name to its avocado job-results directory name. @@ -827,7 +937,7 @@ def replace_xml(logger, xml_file, pattern, replacement, xml_data, test_result): def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, archive, rename, - jenkins_xml, core_files, threshold, total_repeats): + jenkins_xml, core_files, threshold, total_repeats, details): # pylint: disable=too-many-arguments """Process the test results. @@ -851,6 +961,7 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a core_files (dict): location and pattern defining where core files may be written threshold (str): optional upper size limit for test log files total_repeats (int): total number of times the test will be repeated + details (dict): dictionary to update with test results Returns: int: status code: 0 = success, >0 = failure @@ -869,6 +980,10 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a if not cleanup_processes(logger, test, test_result): return_code |= 4096 + # Check storage devices for servers + if not check_server_storage(logger, test, test_result, "Process"): + return_code |= 512 + # Mark the test execution as failed if a results.xml file is not found test_logs_dir = os.path.realpath(os.path.join(job_results_dir, "latest")) results_xml = os.path.join(test_logs_dir, "results.xml") @@ -952,6 +1067,9 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a # Generate a steps.log file return_code |= create_steps_log(logger, job_results_dir, test_result) + # Add test variant results to the details + return_code |= record_variant_details(logger, job_results_dir, test_result, details) + # Optionally rename the test results directory for this test if rename: return_code |= rename_avocado_test_dir( diff --git a/src/tests/ftest/util/command_utils_base.py b/src/tests/ftest/util/command_utils_base.py index 837e61b339f..0c19e4ea322 100644 --- a/src/tests/ftest/util/command_utils_base.py +++ b/src/tests/ftest/util/command_utils_base.py @@ -1,5 +1,6 @@ """ (C) Copyright 2020-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -159,6 +160,9 @@ def get_yaml_value(self, name, test, path): name (str): name of the value in the yaml file test (Test): avocado Test object to use to read the yaml file path (str): yaml path where the name is to be found + + Returns: + bool: True if a new value was assigned; False if the default value was assigned """ if self._yaml_key is not None: # Use the yaml key name instead of the variable name @@ -167,6 +171,7 @@ def get_yaml_value(self, name, test, path): self.value = test.config.get(name, path, self._default) else: self.value = test.params.get(name, path, self._default) + return self.value != self._default def update(self, value, name=None, append=False): """Update the value of the parameter. @@ -333,10 +338,14 @@ def get_yaml_value(self, name, test, path): name (str): name of the value in the yaml file test (Test): avocado Test object to use to read the yaml file path (str): yaml path where the name is to be found + + Returns: + bool: True if a new value was assigned; False if the default value was assigned """ - super().get_yaml_value(name, test, path) + new_value = super().get_yaml_value(name, test, path) self._add_directory() self.log.debug(" Added the directory: %s => %s", name, self.value) + return new_value def update(self, value, name=None, append=False): """Update the value of the parameter. diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index 8001645bbfc..02f90083f03 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -15,7 +15,7 @@ from ClusterShell.NodeSet import NodeSet from slurm_setup import SlurmSetup, SlurmSetupException # pylint: disable=import-error,no-name-in-module -from util.collection_utils import TEST_RESULTS_DIRS, collect_test_result +from util.collection_utils import TEST_RESULTS_DIRS, check_server_storage, collect_test_result from util.data_utils import dict_extract_values, list_flatten, list_unique from util.environment_utils import TestEnvironment from util.host_utils import HostException, HostInfo, get_local_host, get_node_set @@ -183,7 +183,8 @@ def summarize_run(logger, mode, status): 1024: "ERROR: Failed to rename logs and results after one or more tests!", 2048: "ERROR: Core stack trace files detected!", 4096: "ERROR: Unexpected processes or mounts found running!", - 8192: "ERROR: Failed to create steps.log!" + 8192: "ERROR: Failed to create steps.log!", + 16384: "ERROR: Failed to parse test variant status!" } for bit_code, error_message in bit_error_map.items(): if status & bit_code == bit_code: @@ -246,6 +247,9 @@ class TestInfo(): "client_partition", "client_reservation", "client_users", + "scm_list", + "bdev_list", + "check_server_storage", ] def __init__(self, test_file, order, yaml_extension=None): @@ -291,10 +295,13 @@ def set_yaml_info(self, logger, include_local_host=False): """ self.yaml_info = {"include_local_host": include_local_host} yaml_data = get_yaml_data(self.yaml_file) + for extra_yaml in self.extra_yaml: + yaml_data.update(get_yaml_data(extra_yaml)) info = {} for key in self.YAML_INFO_KEYS: # Get the unique values with lists flattened - values = list_unique(list_flatten(dict_extract_values(yaml_data, [key], (str, list)))) + values = list_unique(list_flatten( + dict_extract_values(yaml_data, [key], (str, list, bool)))) if values: # Use single value if list only contains 1 element info[key] = values if len(values) > 1 else values[0] @@ -379,6 +386,7 @@ def __init__(self, avocado, launch_result, total_tests, total_repeats, tag_filte def prepare(self, logger, test_log_file, test, repeat, user_create, slurm_setup, control_host, partition_hosts, clear_mounts): + # pylint: disable=too-many-return-statements """Prepare the test for execution. Args: @@ -405,28 +413,30 @@ def prepare(self, logger, test_log_file, test, repeat, user_create, slurm_setup, self.test_result.start() # Setup the test host information, including creating any required slurm partitions - status = self._setup_host_information( - logger, test, slurm_setup, control_host, partition_hosts) - if status: - return status + if not self._setup_host_information( + logger, test, slurm_setup, control_host, partition_hosts): + return 128 # Setup (remove/create/list) the common test directory on each test host - status = self._setup_test_directory(logger, test) - if status: - return status + if not self._setup_test_directory(logger, test): + return 128 # Setup additional test users - status = self._user_setup(logger, test, user_create) - if status: - return status + if not self._user_setup(logger, test, user_create): + return 128 # Remove existing mount points on each test host - status = self._clear_mount_points(logger, test, clear_mounts) - if status: - return status + if not self._clear_mount_points(logger, test, clear_mounts): + return 128 + + # Check storage devices for servers + if not check_server_storage(logger, test, self.test_result, "Prepare"): + return 128 # Generate certificate files for the test - return self._generate_certs(logger) + if not self._generate_certs(logger): + return 128 + return 0 def execute(self, logger, test, repeat, number, sparse, fail_fast): """Run the specified test. @@ -475,8 +485,8 @@ def execute(self, logger, test, repeat, number, sparse, fail_fast): logger.info("Total test time: %ss", end_time - start_time) return return_code - def process(self, logger, job_results_dir, test, repeat, stop_daos, archive, rename, - jenkins_xml, core_files, threshold): + def process(self, logger, job_results_dir, test, loop, stop_daos, archive, rename, + jenkins_xml, core_files, threshold, details): # pylint: disable=too-many-arguments """Process the test results. @@ -490,13 +500,14 @@ def process(self, logger, job_results_dir, test, repeat, stop_daos, archive, ren Args: logger (Logger): logger for the messages produced by this method test (TestInfo): the test information - repeat (int): the test repetition number + loop (int): the test repetition number (1s based) stop_daos (bool): whether or not to stop daos servers/clients after the test archive (bool): whether or not to collect remote files generated by the test rename (bool): whether or not to rename the default avocado job-results directory names jenkins_xml (bool): whether or not to update the results.xml to use Jenkins-style names core_files (dict): location and pattern defining where core files may be written threshold (str): optional upper size limit for test log files + details (dict): dictionary to update with test results Returns: int: status code: 0 = success, >0 = failure @@ -507,10 +518,10 @@ def process(self, logger, job_results_dir, test, repeat, stop_daos, archive, ren logger.debug("=" * 80) logger.info( "Processing the %s test after the run on repeat %s/%s", - test, repeat, self.total_repeats) + test, loop, self.total_repeats) status = collect_test_result( logger, test, self.test_result, job_results_dir, stop_daos, archive, rename, - jenkins_xml, core_files, threshold, self.total_repeats) + jenkins_xml, core_files, threshold, self.total_repeats, details) # Mark the execution of the test as passed if nothing went wrong if self.test_result.status is None: @@ -532,7 +543,7 @@ def _setup_host_information(self, logger, test, slurm_setup, control_host, parti partition_hosts (NodeSet): slurm partition hosts Returns: - int: status code: 0 = success, 128 = failure + bool: True if successful; False otherwise """ logger.debug("-" * 80) logger.debug("Setting up host information for %s", test) @@ -545,14 +556,14 @@ def _setup_host_information(self, logger, test, slurm_setup, control_host, parti if not exists and not slurm_setup: message = f"Error missing {partition} partition" self.test_result.fail_test(logger, "Prepare", message, None) - return 128 + return False if slurm_setup and exists: logger.info( "Removing existing %s partition to ensure correct configuration", partition) if not delete_partition(logger, control_host, partition).passed: message = f"Error removing existing {partition} partition" self.test_result.fail_test(logger, "Prepare", message, None) - return 128 + return False if slurm_setup: hosts = partition_hosts.difference(test.yaml_info["test_servers"]) logger.debug( @@ -561,12 +572,12 @@ def _setup_host_information(self, logger, test, slurm_setup, control_host, parti if not hosts: message = "Error no partition hosts exist after removing the test servers" self.test_result.fail_test(logger, "Prepare", message, None) - return 128 + return False logger.info("Creating the '%s' partition with the '%s' hosts", partition, hosts) if not create_partition(logger, control_host, partition, hosts).passed: message = f"Error adding the {partition} partition" self.test_result.fail_test(logger, "Prepare", message, None) - return 128 + return False # Define the hosts for this test try: @@ -574,7 +585,7 @@ def _setup_host_information(self, logger, test, slurm_setup, control_host, parti except LaunchException: message = "Error setting up host information" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 + return False # Log the test information msg_format = "%3s %-40s %-60s %-20s %-20s" @@ -585,7 +596,7 @@ def _setup_host_information(self, logger, test, slurm_setup, control_host, parti logger.debug( msg_format, test.name.order, test.test_file, test.yaml_file, test.host_info.servers.hosts, test.host_info.clients.hosts) - return 0 + return True def _setup_test_directory(self, logger, test): """Set up the common test directory on all hosts. @@ -595,7 +606,7 @@ def _setup_test_directory(self, logger, test): test (TestInfo): the test information Returns: - int: status code: 0 = success, 128 = failure + bool: True if successful; False otherwise """ logger.debug("-" * 80) test_env = TestEnvironment() @@ -617,8 +628,8 @@ def _setup_test_directory(self, logger, test): if not run_remote(logger, hosts, command).passed: message = "Error setting up the common test directory on all hosts" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 - return 0 + return False + return True def _user_setup(self, logger, test, create=False): """Set up test users on client nodes. @@ -629,7 +640,7 @@ def _user_setup(self, logger, test, create=False): create (bool, optional): whether to create extra test users defined by the test Returns: - int: status code: 0 = success, 128 = failure + bool: True if successful; False otherwise """ users = test.get_yaml_client_users() clients = test.host_info.clients.hosts @@ -650,16 +661,16 @@ def _user_setup(self, logger, test, create=False): group_gid[group] = self._query_create_group(logger, clients, group, create) except LaunchException as error: self.test_result.fail_test(logger, "Prepare", str(error), sys.exc_info()) - return 128 + return False gid = group_gid.get(group, None) try: self._query_create_user(logger, clients, user, gid, create) except LaunchException as error: self.test_result.fail_test(logger, "Prepare", str(error), sys.exc_info()) - return 128 + return False - return 0 + return True @staticmethod def _query_create_group(logger, hosts, group, create=False): @@ -743,10 +754,10 @@ def _clear_mount_points(self, logger, test, clear_mounts): clear_mounts (list): mount points to remove before the test Returns: - int: status code: 0 = success, 128 = failure + bool: True if successful; False otherwise """ if not clear_mounts: - return 0 + return True logger.debug("-" * 80) hosts = test.host_info.all_hosts @@ -766,20 +777,20 @@ def _clear_mount_points(self, logger, test, clear_mounts): if not self._remove_super_blocks(logger, mount_hosts, mount_point): message = "Error removing superblocks for existing mount points" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 + return False if not self._remove_shared_memory_segments(logger, hosts): message = "Error removing shared memory segments for existing mount points" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 + return False for mount_point, mount_hosts in mount_point_hosts.items(): if not self._remove_mount_point(logger, mount_hosts, mount_point): message = "Error removing existing mount points" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 + return False - return 0 + return True def _remove_super_blocks(self, logger, hosts, mount_point): """Remove the super blocks from the specified mount point. @@ -848,10 +859,11 @@ def _remove_mount_point(self, logger, hosts, mount_point): def _generate_certs(self, logger): """Generate the certificates for the test. - Returns: + Args: logger (Logger): logger for the messages produced by this method - int: status code: 0 = success, 128 = failure + Returns: + bool: True if successful; False otherwise """ logger.debug("-" * 80) logger.debug("Generating certificates") @@ -863,12 +875,12 @@ def _generate_certs(self, logger): if not run_local(logger, f"/usr/bin/rm -rf {certs_dir}").passed: message = "Error removing old certificates" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 + return False if not run_local(logger, f"{command} {test_env.log_dir}").passed: message = "Error generating certificates" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 - return 0 + return False + return True def _collect_crash_files(self, logger): """Move any avocado crash files into job-results/latest/crashes. @@ -1261,12 +1273,14 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop if not code_coverage.setup(logger, result.tests[0]): return_code |= 128 + self._details["tests"] = [] + # Run each test for as many repetitions as requested for loop in range(1, repeat + 1): logger.info("-" * 80) logger.info("Starting test repetition %s/%s", loop, repeat) - for index, test in enumerate(self.tests): + for sequence, test in enumerate(self.tests): # Define a log for the execution of this test for this repetition test_log_file = test.get_log_file(logdir, loop, repeat) logger.info("-" * 80) @@ -1274,6 +1288,17 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop test_file_handler = get_file_handler(test_log_file, LOG_FILE_FORMAT, logging.DEBUG) logger.addHandler(test_file_handler) + if len(self._details["tests"]) == sequence: + # Add an entry for this test the first time its run in the loop + self._details["tests"].append({ + "sequence": sequence + 1, + "test_file": str(test), + "results": test_log_file, + "clients": str(test.yaml_info["test_clients"]), + "servers": str(test.yaml_info["test_servers"]), + "bdev_list": test.yaml_info["bdev_list"]}) + self._details["tests"][sequence]["repetitions"] = loop + # Prepare the hosts to run the tests step_status = runner.prepare( logger, test_log_file, test, loop, user_create, slurm_setup, self._control, @@ -1284,12 +1309,12 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop continue # Run the test with avocado - return_code |= runner.execute(logger, test, loop, index + 1, sparse, fail_fast) + return_code |= runner.execute(logger, test, loop, sequence + 1, sparse, fail_fast) # Archive the test results return_code |= runner.process( logger, job_results_dir, test, loop, stop_daos, archive, rename, - jenkins_log, core_files, threshold) + jenkins_log, core_files, threshold, self._details["tests"][sequence]) # Display disk usage after the test is complete display_disk_space(logger, logdir) diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index 6ed19436d97..996f380b004 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -464,6 +464,8 @@ def __init__(self, base_namespace, index, provider=None, max_storage_tiers=MAX_S Defaults to MAX_STORAGE_TIERS. """ namespace = [os.sep] + base_namespace.split(os.sep)[1:-1] + ["engines", str(index), "*"] + common_ns = [os.sep] + base_namespace.split(os.sep)[1:-1] + ["engines_common", "*"] + self.common_namespace = os.path.join(*common_ns) self._base_namespace = base_namespace self._index = index self._provider = provider or os.environ.get("D_PROVIDER", "ofi+tcp") @@ -519,7 +521,11 @@ def get_params(self, test): Args: test (Test): avocado Test object """ - super().get_params(test) + # Get the values for the engine yaml parameters + for name in self.get_param_names(): + if not getattr(self, name).get_yaml_value(name, test, self.namespace): + # If a new value was not assigned, check the engine's common namespace for a value + getattr(self, name).get_yaml_value(name, test, self.common_namespace) # Override the log file file name with the test log file name if hasattr(test, "server_log") and test.server_log is not None: From a42f21df4157d11e5690b87c8943726d923d1984 Mon Sep 17 00:00:00 2001 From: Nasf-Fan Date: Fri, 14 Nov 2025 23:58:23 +0800 Subject: [PATCH 34/43] DAOS-17845 container: keep CONT_TGT_SNAPSHOT_NOTIFY RPC errno - b26 (#16673) Return CONT_TGT_SNAPSHOT_NOTIFY RPC errno to the snap_oit_create() sponsor instead of converting it as -DER_IO. Then related client side logic can know what the failure is. Signed-off-by: Fan Yong --- src/container/srv_epoch.c | 5 ++--- src/container/srv_target.c | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/container/srv_epoch.c b/src/container/srv_epoch.c index a504dd32145..5d5e01e4469 100644 --- a/src/container/srv_epoch.c +++ b/src/container/srv_epoch.c @@ -220,9 +220,8 @@ snap_oit_create(struct rdb_tx *tx, struct cont *cont, uuid_t coh_uuid, out = crt_reply_get(rpc); rc = out->tso_rc; if (rc != 0) { - D_ERROR(DF_CONT": snapshot notify failed on %d targets\n", - DP_CONT(cont->c_svc->cs_pool_uuid, cont->c_uuid), rc); - rc = -DER_IO; + D_ERROR(DF_CONT " snapshot notify failed: " DF_RC "\n", + DP_CONT(cont->c_svc->cs_pool_uuid, cont->c_uuid), DP_RC(rc)); goto out_rpc; } *epoch = in->tsi_epoch; diff --git a/src/container/srv_target.c b/src/container/srv_target.c index e30738e1696..deaaba884a3 100644 --- a/src/container/srv_target.c +++ b/src/container/srv_target.c @@ -2201,7 +2201,8 @@ ds_cont_tgt_snapshot_notify_aggregator(crt_rpc_t *source, crt_rpc_t *result, out_source = crt_reply_get(source); out_result = crt_reply_get(result); - out_result->tso_rc += out_source->tso_rc; + if (out_result->tso_rc >= 0 && out_source->tso_rc < 0) + out_result->tso_rc = out_source->tso_rc; return 0; } From d8c1016ecac3dab26dd531dfdc00bb11a32f4b0d Mon Sep 17 00:00:00 2001 From: Alexander Oganezov Date: Fri, 14 Nov 2025 08:47:15 -0800 Subject: [PATCH 35/43] DAOS-17871 cart: Improve swim notification test (#16752) (#17119) Change how the test behaves. At start it will now wait for all 3 ranks to be present and waits 30 seconds after that before rank=2 is shut down. 30 seconds is left to allow for swim notification status to be propagated across all 3 servers in this group. Added --skip_wait option to clients to skip crtu_wait_for_ranks() call as only first client invocation needs to ensure all 3 servers are up. Signed-off-by: Alexander A Oganezov --- .../ftest/cart/rpc/swim_notification.yaml | 18 +-- src/tests/ftest/cart/test_group_np_cli.c | 93 +++++++------- src/tests/ftest/cart/test_group_np_common.h | 120 ++++++++---------- 3 files changed, 112 insertions(+), 119 deletions(-) diff --git a/src/tests/ftest/cart/rpc/swim_notification.yaml b/src/tests/ftest/cart/rpc/swim_notification.yaml index 89e3af14238..83cbac5799d 100644 --- a/src/tests/ftest/cart/rpc/swim_notification.yaml +++ b/src/tests/ftest/cart/rpc/swim_notification.yaml @@ -4,7 +4,7 @@ ENV: default: #!filter-only : /run/tests/proto_np - - D_LOG_MASK: "WARN" + - D_LOG_MASK: "INFO" - D_INTERFACE: "eth0" - CRT_TEST_CONT: "1" - test_servers_CRT_CTX_NUM: "16" @@ -29,11 +29,11 @@ tests: !mux test_clients_env: "" test_clients_ppn: "1" test_clients_arg: - - "--name client_group --attach_to tg_srv_grp_swim_test --init_only --holdtime 20" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank2=alive' --skip_shutdown --skip_check_in" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 2 --shut_only --holdtime 30" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank2=dead' --skip_shutdown --skip_check_in" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank1=alive' --skip_shutdown --skip_check_in" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank0=alive' --skip_shutdown --skip_check_in" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --disable_swim --skip_shutdown --skip_check_in" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --shut_only" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1,2 --init_only --holdtime 30" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank2=alive' --skip_shutdown --skip_check_in --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 2 --shut_only --holdtime 30 --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank2=dead' --skip_shutdown --skip_check_in --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank1=alive' --skip_shutdown --skip_check_in --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank0=alive' --skip_shutdown --skip_check_in --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --disable_swim --skip_shutdown --skip_check_in --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --shut_only --skip_wait" diff --git a/src/tests/ftest/cart/test_group_np_cli.c b/src/tests/ftest/cart/test_group_np_cli.c index f796fc1a152..0ca71e895c9 100644 --- a/src/tests/ftest/cart/test_group_np_cli.c +++ b/src/tests/ftest/cart/test_group_np_cli.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -23,21 +24,20 @@ static void send_rpc_swim_check(crt_endpoint_t server_ep, crt_rpc_t *rpc_req) { - struct test_swim_status_in *rpc_req_input; + struct test_swim_status_in *input; int rc = crt_req_create(test_g.t_crt_ctx[0], &server_ep, TEST_OPC_SWIM_STATUS, &rpc_req); D_ASSERTF(rc == 0 && rpc_req != NULL, "crt_req_create() failed. " - "rc: %d, rpc_req: %p\n", rc, rpc_req); + "rc: %d, rpc_req: %p\n", + rc, rpc_req); - rpc_req_input = crt_req_get(rpc_req); - D_ASSERTF(rpc_req_input != NULL, "crt_req_get() failed." - " rpc_req_input: %p\n", rpc_req_input); + input = crt_req_get(rpc_req); /* Set rank and expected swim status based on CLI options */ - rpc_req_input->rank = test_g.t_verify_swim_status.rank; - rpc_req_input->exp_status = test_g.t_verify_swim_status.swim_status; + input->rank = test_g.t_verify_swim_status.rank; + input->exp_status = test_g.t_verify_swim_status.swim_status; /* RPC is expected to finish in 10 seconds */ rc = crt_req_set_timeout(rpc_req, 10); @@ -52,20 +52,19 @@ send_rpc_swim_check(crt_endpoint_t server_ep, crt_rpc_t *rpc_req) static void send_rpc_disable_swim(crt_endpoint_t server_ep, crt_rpc_t *rpc_req) { - struct test_disable_swim_in *rpc_req_input; + struct test_disable_swim_in *input; int rc = crt_req_create(test_g.t_crt_ctx[0], &server_ep, TEST_OPC_DISABLE_SWIM, &rpc_req); D_ASSERTF(rc == 0 && rpc_req != NULL, "crt_req_create() failed. " - "rc: %d, rpc_req: %p\n", rc, rpc_req); + "rc: %d, rpc_req: %p\n", + rc, rpc_req); - rpc_req_input = crt_req_get(rpc_req); - D_ASSERTF(rpc_req_input != NULL, "crt_req_get() failed." - " rpc_req_input: %p\n", rpc_req_input); + input = crt_req_get(rpc_req); /* Set rank and expected swim status based on CLI options */ - rpc_req_input->rank = server_ep.ep_rank; + input->rank = server_ep.ep_rank; /* RPC is expected to finish in 10 seconds */ rc = crt_req_set_timeout(rpc_req, 10); @@ -94,43 +93,43 @@ test_run(void) if (test_g.t_skip_init) { DBG_PRINT("Skipping init stage.\n"); + goto skip_init; + } - } else { - if (test_g.t_save_cfg) { - rc = crt_group_config_path_set(test_g.t_cfg_path); - D_ASSERTF(rc == 0, - "crt_group_config_path_set failed %d\n", rc); - } + if (test_g.t_save_cfg) { + DBG_PRINT("Setting group config path: %p\n", test_g.t_cfg_path); + rc = crt_group_config_path_set(test_g.t_cfg_path); + D_ASSERTF(rc == 0, "crt_group_config_path_set failed %d\n", rc); + } - rc = crtu_cli_start_basic(test_g.t_local_group_name, - test_g.t_remote_group_name, - &grp, &rank_list, &test_g.t_crt_ctx[0], - &test_g.t_tid[0], test_g.t_srv_ctx_num, - test_g.t_use_cfg, NULL, test_g.t_use_daos_agent_env); - D_ASSERTF(rc == 0, "crtu_cli_start_basic() failed\n"); - - rc = sem_init(&test_g.t_token_to_proceed, 0, 0); - D_ASSERTF(rc == 0, "sem_init() failed.\n"); - - /* register RPCs */ - rc = crt_proto_register(&my_proto_fmt_test_group1); - D_ASSERTF(rc == 0, "crt_proto_register() failed. rc: %d\n", - rc); - - /* Process the --rank option, e.g., --rank 1,2-4 */ - if (test_g.cg_num_ranks > 0) { - _cg_ranks = (uint32_t *)test_g.cg_ranks; - _cg_num_ranks = test_g.cg_num_ranks; - - /* free up rank list from crtu_cli_start_basic */ - if (rank_list != NULL) { - /* avoid checkpatch warning */ - d_rank_list_free(rank_list); - } - rank_list = uint32_array_to_rank_list(_cg_ranks, _cg_num_ranks); - D_ASSERTF(rank_list != NULL, "failed to convert array to rank list\n"); + rc = crtu_cli_start_basic(test_g.t_local_group_name, test_g.t_remote_group_name, &grp, + &rank_list, &test_g.t_crt_ctx[0], &test_g.t_tid[0], + test_g.t_srv_ctx_num, test_g.t_use_cfg, NULL, + test_g.t_use_daos_agent_env); + D_ASSERTF(rc == 0, "crtu_cli_start_basic() failed\n"); + + rc = sem_init(&test_g.t_token_to_proceed, 0, 0); + D_ASSERTF(rc == 0, "sem_init() failed.\n"); + + /* register RPCs */ + rc = crt_proto_register(&my_proto_fmt_test_group1); + D_ASSERTF(rc == 0, "crt_proto_register() failed. rc: %d\n", rc); + + /* Process the --rank option, e.g., --rank 1,2-4 */ + if (test_g.cg_num_ranks > 0) { + _cg_ranks = (uint32_t *)test_g.cg_ranks; + _cg_num_ranks = test_g.cg_num_ranks; + + /* free up rank list from crtu_cli_start_basic */ + if (rank_list != NULL) { + /* avoid checkpatch warning */ + d_rank_list_free(rank_list); } + rank_list = uint32_array_to_rank_list(_cg_ranks, _cg_num_ranks); + D_ASSERTF(rank_list != NULL, "failed to convert array to rank list\n"); + } + if (!test_g.t_skip_wait) { rc = crtu_wait_for_ranks(test_g.t_crt_ctx[0], grp, rank_list, @@ -141,6 +140,8 @@ test_run(void) D_ASSERTF(rc == 0, "wait_for_ranks() failed; rc=%d\n", rc); } +skip_init: + if (test_g.t_init_only) { DBG_PRINT("Init only. Returning now.\n"); D_GOTO(clean_up, rc = 0); diff --git a/src/tests/ftest/cart/test_group_np_common.h b/src/tests/ftest/cart/test_group_np_common.h index 8da4c047c7f..86786ba5ab2 100644 --- a/src/tests/ftest/cart/test_group_np_common.h +++ b/src/tests/ftest/cart/test_group_np_common.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -33,6 +34,7 @@ struct test_t { int t_shut_only; int t_init_only; int t_skip_init; + int t_skip_wait; int t_skip_shutdown; int t_skip_check_in; bool t_save_cfg; @@ -189,6 +191,7 @@ test_swim_status_handler(crt_rpc_t *rpc_req) int rank; int rc_dead; int rc_alive; + bool result_ok = false; /* CaRT internally already allocated the input/output buffer */ e_req = crt_req_get(rpc_req); @@ -198,51 +201,43 @@ test_swim_status_handler(crt_rpc_t *rpc_req) /* compile and run regex's */ regcomp(®ex_dead, dead_regex, REG_EXTENDED); - rc_dead = regexec(®ex_dead, - swim_seq_by_rank[rank], - 0, NULL, 0); + rc_dead = regexec(®ex_dead, swim_seq_by_rank[rank], 0, NULL, 0); + regcomp(®ex_alive, alive_regex, REG_EXTENDED); - rc_alive = regexec(®ex_alive, - swim_seq_by_rank[rank], - 0, NULL, 0); + rc_alive = regexec(®ex_alive, swim_seq_by_rank[rank], 0, NULL, 0); regfree(®ex_alive); regfree(®ex_dead); - DBG_PRINT("tier1 test_server recv'd swim_status, opc: %#x.\n", - rpc_req->cr_opc); - DBG_PRINT("tier1 swim_status input - rank: %d, exp_status: %d.\n", - rank, e_req->exp_status); - - if (e_req->exp_status == CRT_EVT_ALIVE) - D_ASSERTF(rc_alive == 0, - "Swim status alive sequence (%s) " - "does not match '%s' for rank %d.\n", - swim_seq_by_rank[rank], alive_regex, rank); - else if (e_req->exp_status == CRT_EVT_DEAD) - D_ASSERTF(rc_dead == 0, - "Swim status dead sequence (%s) " - "does not match '%s' for rank %d..\n", - swim_seq_by_rank[rank], dead_regex, rank); - - DBG_PRINT("Rank [%d] SWIM state sequence (%s) for " - "status [%d] is as expected.\n", - rank, swim_seq_by_rank[rank], - e_req->exp_status); + if (e_req->exp_status == CRT_EVT_ALIVE) { + if (rc_alive != 0) { + D_ERROR("Swim status alive sequence (%s) " + "does not match '%s' for rank %d.\n", + swim_seq_by_rank[rank], alive_regex, rank); + result_ok = false; + } else { + result_ok = true; + } + } else if (e_req->exp_status == CRT_EVT_DEAD) { + if (rc_dead != 0) { + D_ERROR("Swim status dead sequence (%s) " + "does not match '%s' for rank %d..\n", + swim_seq_by_rank[rank], dead_regex, rank); + result_ok = false; + } else { + result_ok = true; + } + } e_reply = crt_reply_get(rpc_req); + D_ASSERTF(e_reply != NULL, "crt_reply_get() failed"); + e_reply->bool_val = result_ok; - /* If we got past the previous assert, then we've succeeded */ - e_reply->bool_val = true; - D_ASSERTF(e_reply != NULL, "crt_reply_get() failed. e_reply: %p\n", - e_reply); + DBG_PRINT("rank:%d exp_status: %d matches_expected: %d\n", rank, e_req->exp_status, + result_ok); rc = crt_reply_send(rpc_req); D_ASSERTF(rc == 0, "crt_reply_send() failed. rc: %d\n", rc); - - DBG_PRINT("tier1 test_srver sent swim_status reply," - "e_reply->bool_val: %d.\n", - e_reply->bool_val); } static void @@ -256,15 +251,13 @@ test_ping_delay_handler(crt_rpc_t *rpc_req) p_req = crt_req_get(rpc_req); D_ASSERTF(p_req != NULL, "crt_req_get() failed. p_req: %p\n", p_req); - DBG_PRINT("tier1 test_server recv'd ping delay, opc: %#x.\n", - rpc_req->cr_opc); - DBG_PRINT("tier1 delayed ping input - age: %d, name: %s, days: %d, " - "delay: %u.\n", p_req->age, p_req->name, p_req->days, - p_req->delay); + DBG_PRINT("test_server recv'd ping delay, opc: %#x.\n", rpc_req->cr_opc); + DBG_PRINT("delayed ping input - age: %d, name: %s, days: %d, " + "delay: %u.\n", + p_req->age, p_req->name, p_req->days, p_req->delay); p_reply = crt_reply_get(rpc_req); - D_ASSERTF(p_reply != NULL, "crt_reply_get() failed. p_reply: %p\n", - p_reply); + D_ASSERTF(p_reply != NULL, "crt_reply_get() failed\n"); p_reply->ret = 0; p_reply->room_no = test_g.t_roomno++; @@ -745,29 +738,28 @@ test_parse_args(int argc, char **argv) int option_index = 0; int rc = 0; int ss; - struct option long_options[] = { - {"name", required_argument, 0, 'n'}, - {"attach_to", required_argument, 0, 'a'}, - {"holdtime", required_argument, 0, 'h'}, - {"hold", no_argument, &test_g.t_hold, 1}, - {"srv_ctx_num", required_argument, 0, 'c'}, - {"shut_only", no_argument, &test_g.t_shut_only, 1}, - {"init_only", no_argument, &test_g.t_init_only, 1}, - {"skip_init", no_argument, &test_g.t_skip_init, 1}, - {"skip_shutdown", no_argument, &test_g.t_skip_shutdown, 1}, - {"skip_check_in", no_argument, &test_g.t_skip_check_in, 1}, - {"rank", required_argument, 0, 'r'}, - {"cfg_path", required_argument, 0, 's'}, - {"use_cfg", required_argument, 0, 'u'}, - {"register_swim_callback", required_argument, 0, 'w'}, - {"verify_swim_status", required_argument, 0, 'v'}, - {"disable_swim", no_argument, &test_g.t_disable_swim, 1}, - {"get_swim_status", no_argument, 0, 'g'}, - {"shutdown_delay", required_argument, 0, 'd'}, - {"write_completion_file", no_argument, - &test_g.t_write_completion_file, 1}, - {0, 0, 0, 0} - }; + struct option long_options[] = { + {"name", required_argument, 0, 'n'}, + {"attach_to", required_argument, 0, 'a'}, + {"holdtime", required_argument, 0, 'h'}, + {"hold", no_argument, &test_g.t_hold, 1}, + {"srv_ctx_num", required_argument, 0, 'c'}, + {"shut_only", no_argument, &test_g.t_shut_only, 1}, + {"init_only", no_argument, &test_g.t_init_only, 1}, + {"skip_init", no_argument, &test_g.t_skip_init, 1}, + {"skip_wait", no_argument, &test_g.t_skip_wait, 1}, + {"skip_shutdown", no_argument, &test_g.t_skip_shutdown, 1}, + {"skip_check_in", no_argument, &test_g.t_skip_check_in, 1}, + {"rank", required_argument, 0, 'r'}, + {"cfg_path", required_argument, 0, 's'}, + {"use_cfg", required_argument, 0, 'u'}, + {"register_swim_callback", required_argument, 0, 'w'}, + {"verify_swim_status", required_argument, 0, 'v'}, + {"disable_swim", no_argument, &test_g.t_disable_swim, 1}, + {"get_swim_status", no_argument, 0, 'g'}, + {"shutdown_delay", required_argument, 0, 'd'}, + {"write_completion_file", no_argument, &test_g.t_write_completion_file, 1}, + {0, 0, 0, 0}}; test_g.cg_num_ranks = 0; test_g.t_use_cfg = true; From af379a2083b3c05f67d25ef974057111b2b5edb4 Mon Sep 17 00:00:00 2001 From: Alexander Oganezov Date: Fri, 14 Nov 2025 08:51:34 -0800 Subject: [PATCH 36/43] DAOS-17470 cart: add out of quota RPC to timeout (#16308) (#16582) Add out of quota RPC to timeout list once resubmit them. Signed-off-by: Di Wang Signed-off-by: Alexander A Oganezov --- src/cart/crt_context.c | 113 ++++++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 53 deletions(-) diff --git a/src/cart/crt_context.c b/src/cart/crt_context.c index 6ddd3707138..e38d5348d34 100644 --- a/src/cart/crt_context.c +++ b/src/cart/crt_context.c @@ -1443,6 +1443,52 @@ crt_context_req_untrack_internal(struct crt_rpc_priv *rpc_priv) RPC_DECREF(rpc_priv); } +static void +add_rpc_to_list(struct crt_rpc_priv *rpc_priv, d_list_t *submit_list) +{ + struct crt_context *crt_ctx = rpc_priv->crp_pub.cr_ctx; + struct crt_ep_inflight *epi = rpc_priv->crp_epi; + + D_ASSERT(epi != NULL); + + RPC_ADDREF(rpc_priv); + + crt_rpc_lock(rpc_priv); + D_MUTEX_LOCK(&epi->epi_mutex); + if (rpc_priv->crp_state == RPC_STATE_QUEUED) { + bool submit_rpc = true; + int rc; + + rpc_priv->crp_state = RPC_STATE_INITED; + /* RPC got cancelled or timed out before it got here */ + if (rpc_priv->crp_timeout_ts == 0) { + submit_rpc = false; + } else { + crt_set_timeout(rpc_priv); + + D_MUTEX_LOCK(&crt_ctx->cc_mutex); + rc = crt_req_timeout_track(rpc_priv); + D_MUTEX_UNLOCK(&crt_ctx->cc_mutex); + if (rc != 0) + RPC_ERROR(rpc_priv, "crt_req_timeout_track failed, rc: %d.\n", rc); + } + + d_list_move_tail(&rpc_priv->crp_epi_link, &epi->epi_req_q); + /* add to submit list if not cancelled or timed out already */ + if (submit_rpc) { + /* prevent rpc from being released before it is dispatched below */ + RPC_ADDREF(rpc_priv); + + D_ASSERTF(d_list_empty(&rpc_priv->crp_tmp_link_submit), + "already on submit list\n"); + d_list_add_tail(&rpc_priv->crp_tmp_link_submit, submit_list); + } + } + D_MUTEX_UNLOCK(&epi->epi_mutex); + crt_rpc_unlock(rpc_priv); + RPC_DECREF(rpc_priv); +} + static void dispatch_rpc(struct crt_rpc_priv *rpc) { int rc; @@ -1476,8 +1522,7 @@ crt_context_req_untrack(struct crt_rpc_priv *rpc_priv) struct crt_context *crt_ctx = rpc_priv->crp_pub.cr_ctx; struct crt_ep_inflight *epi; d_list_t submit_list; - struct crt_rpc_priv *tmp_rpc; - int rc; + struct crt_rpc_priv *tmp_rpc; D_ASSERT(crt_ctx != NULL); @@ -1493,8 +1538,9 @@ crt_context_req_untrack(struct crt_rpc_priv *rpc_priv) struct crt_rpc_priv, crp_waitq_link); D_MUTEX_UNLOCK(&crt_ctx->cc_mutex); + D_INIT_LIST_HEAD(&submit_list); if (tmp_rpc != NULL) - dispatch_rpc(tmp_rpc); + add_rpc_to_list(tmp_rpc, &submit_list); else put_quota_resource(rpc_priv->crp_pub.cr_ctx, CRT_QUOTA_RPCS); @@ -1502,65 +1548,26 @@ crt_context_req_untrack(struct crt_rpc_priv *rpc_priv) /* done if ep credit flow control is disabled */ if (crt_gdata.cg_credit_ep_ctx == 0) - return; - - D_INIT_LIST_HEAD(&submit_list); - - D_MUTEX_LOCK(&epi->epi_mutex); + goto out; /* process waitq */ + D_MUTEX_LOCK(&epi->epi_mutex); while (credits_available(epi) > 0 && !d_list_empty(&epi->epi_req_waitq)) { - D_ASSERT(epi->epi_req_wait_num > 0); - tmp_rpc = d_list_entry(epi->epi_req_waitq.next, struct crt_rpc_priv, crp_epi_link); - RPC_ADDREF(tmp_rpc); - D_MUTEX_UNLOCK(&epi->epi_mutex); - - crt_rpc_lock(tmp_rpc); - D_MUTEX_LOCK(&epi->epi_mutex); - if (tmp_rpc->crp_state == RPC_STATE_QUEUED && credits_available(epi) > 0) { - bool submit_rpc = true; - - tmp_rpc->crp_state = RPC_STATE_INITED; - /* RPC got cancelled or timed out before it got here */ - if (tmp_rpc->crp_timeout_ts == 0) { - submit_rpc = false; - } else { - crt_set_timeout(tmp_rpc); - - D_MUTEX_LOCK(&crt_ctx->cc_mutex); - rc = crt_req_timeout_track(tmp_rpc); - D_MUTEX_UNLOCK(&crt_ctx->cc_mutex); - if (rc != 0) - RPC_ERROR(tmp_rpc, - "crt_req_timeout_track failed, rc: %d.\n", rc); - } - - /* remove from waitq and add to in-flight queue */ - d_list_move_tail(&tmp_rpc->crp_epi_link, &epi->epi_req_q); - epi->epi_req_wait_num--; - D_ASSERT(epi->epi_req_wait_num >= 0); - epi->epi_req_num++; - D_ASSERT(epi->epi_req_num >= epi->epi_reply_num); - - /* add to submit list if not cancelled or timed out already */ - if (submit_rpc) { - /* prevent rpc from being released before it is dispatched below */ - RPC_ADDREF(tmp_rpc); - - D_ASSERTF(d_list_empty(&tmp_rpc->crp_tmp_link_submit), - "already on submit list\n"); - d_list_add_tail(&tmp_rpc->crp_tmp_link_submit, &submit_list); - } - } + tmp_rpc = d_list_pop_entry(&epi->epi_req_waitq, struct crt_rpc_priv, crp_epi_link); + epi->epi_req_wait_num--; + D_ASSERTF(epi->epi_req_wait_num >= 0, "wait %jd\n", epi->epi_req_wait_num); + /* remove from waitq and add to in-flight queue */ + epi->epi_req_num++; + D_ASSERTF(epi->epi_req_num >= epi->epi_reply_num, "req %jd reply %jd\n", + epi->epi_req_num, epi->epi_reply_num); D_MUTEX_UNLOCK(&epi->epi_mutex); - crt_rpc_unlock(tmp_rpc); - RPC_DECREF(tmp_rpc); - + add_rpc_to_list(tmp_rpc, &submit_list); D_MUTEX_LOCK(&epi->epi_mutex); } D_MUTEX_UNLOCK(&epi->epi_mutex); +out: /* re-submit the rpc req */ while ( (tmp_rpc = d_list_pop_entry(&submit_list, struct crt_rpc_priv, crp_tmp_link_submit))) { From 6abffd7d9ef10d521ba58ba797769f793d58bdba Mon Sep 17 00:00:00 2001 From: Cedric Koch-Hofer <94527853+knard38@users.noreply.github.com> Date: Fri, 14 Nov 2025 17:53:15 +0100 Subject: [PATCH 37/43] DAOS-17286 gurt: fix rank set string convertion (#16204) (#16591) Remove string maximal length limitation of the d_rank_range_list_str() conversion function. Signed-off-by: Cedric Koch-Hofer --- src/common/tests_dmg_helpers.c | 6 +- src/gurt/misc.c | 148 ++++++++++++++-------- src/gurt/tests/test_gurt.c | 222 ++++++++++++++++++++++++++++++++- src/include/gurt/common.h | 7 +- src/mgmt/srv_drpc.c | 58 ++++----- src/mgmt/srv_pool.c | 11 +- 6 files changed, 358 insertions(+), 94 deletions(-) diff --git a/src/common/tests_dmg_helpers.c b/src/common/tests_dmg_helpers.c index 928d9c844cf..4aed95df659 100644 --- a/src/common/tests_dmg_helpers.c +++ b/src/common/tests_dmg_helpers.c @@ -1121,9 +1121,9 @@ dmg_pool_extend(const char *dmg_config_file, const uuid_t uuid, rank_list.rl_ranks = ranks; rank_list.rl_nr = rank_nr; - rank_str = d_rank_list_to_str(&rank_list); - if (rank_str == NULL) - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_to_str(&rank_list, &rank_str); + if (rc != 0) + D_GOTO(out, rc); uuid_unparse_lower(uuid, uuid_str); args = cmd_push_arg(args, &argcount, "%s ", uuid_str); diff --git a/src/gurt/misc.c b/src/gurt/misc.c index 29cc6d3bc24..094b3cc1a56 100644 --- a/src/gurt/misc.c +++ b/src/gurt/misc.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -744,24 +745,39 @@ d_rank_list_dump(d_rank_list_t *rank_list, d_string_t name, int name_len) * Create a ranged string representation of a rank list. * * \param[in] rank_list the rank list to represent + * \param[out] ranks_str Returned ranged string (caller must free) * - * \return a ranged string (caller must free) + * \return 0 on success, a negative value on error */ -char * -d_rank_list_to_str(d_rank_list_t *rank_list) +int +d_rank_list_to_str(d_rank_list_t *ranks, char **ranks_str) { - char *str; - bool truncated = false; - d_rank_range_list_t *range_list; + d_rank_range_list_t *range_list = NULL; + char *range_list_str; + int rc; - range_list = d_rank_range_list_create_from_ranks(rank_list); + D_ASSERT(ranks_str != NULL); + + if (ranks == NULL) { + range_list_str = NULL; + D_GOTO(out, rc = -DER_SUCCESS); + } + + range_list = d_rank_range_list_create_from_ranks(ranks); if (range_list == NULL) - return NULL; - str = d_rank_range_list_str(range_list, &truncated); + D_GOTO(error, rc = -DER_NOMEM); + + rc = d_rank_range_list_str(range_list, &range_list_str); + if (rc != 0) + D_GOTO(error, rc); +out: + *ranks_str = range_list_str; + +error: d_rank_range_list_free(range_list); - return str; + return rc; } d_rank_list_t * @@ -844,7 +860,6 @@ d_rank_range_list_realloc(d_rank_range_list_t *range_list, uint32_t size) return range_list; } -/* TODO (DAOS-10253) Add unit tests for this function */ d_rank_range_list_t * d_rank_range_list_create_from_ranks(d_rank_list_t *rank_list) { @@ -890,56 +905,91 @@ d_rank_range_list_create_from_ranks(d_rank_list_t *rank_list) return range_list; } -/* TODO (DAOS-10253) Add unit tests for this function */ -char * -d_rank_range_list_str(d_rank_range_list_t *list, bool *truncated) +static inline int +ranks2str(char *buf, size_t buf_size, uint32_t rank_lo, uint32_t rank_hi, size_t *buf_len) { - const size_t MAXBYTES = 512; - char *line; - char *linepos; - int ret = 0; - size_t remaining = MAXBYTES - 2u; - int i; - int err = 0; + int len; - *truncated = false; - D_ALLOC(line, MAXBYTES); - if (line == NULL) - return NULL; + if (rank_lo == rank_hi) + len = snprintf(buf, buf_size, "%u", rank_lo); + else + len = snprintf(buf, buf_size, "%u-%u", rank_lo, rank_hi); - *line = '['; - linepos = line + 1; - for (i = 0; i < list->rrl_nr; i++) { - uint32_t lo = list->rrl_ranges[i].lo; - uint32_t hi = list->rrl_ranges[i].hi; - bool lastrange = (i == (list->rrl_nr - 1)); + if (len < 0) + return -DER_INVAL; + if (len >= buf_size) + return -DER_TRUNC; - if (lo == hi) - ret = snprintf(linepos, remaining, "%u%s", lo, lastrange ? "" : ","); - else - ret = snprintf(linepos, remaining, "%u-%u%s", lo, hi, lastrange ? "" : ","); + *buf_len = (size_t)len; + return -DER_SUCCESS; +} - if (ret < 0) { - err = errno; - D_ERROR("rank set could not be serialized: %s (%d)\n", strerror(err), err); - break; +int +d_rank_range_list_str(d_rank_range_list_t *list, char **ranks_str) +{ + const size_t MINBYTES = 256u; + char *line; + size_t line_size; + char *buf; + size_t buf_size; + int i; + int rc; + + D_ASSERT(list != NULL); + + D_ALLOC_ARRAY(line, MINBYTES); + if (line == NULL) + D_GOTO(error, rc = -DER_NOMEM); + line_size = MINBYTES; + buf = line; + buf_size = line_size; + + buf[0] = '['; + buf++; + buf_size--; + for (i = 0; i < list->rrl_nr; i++) { + size_t len = 0; + uint32_t rank_lo = list->rrl_ranges[i].lo; + uint32_t rank_hi = list->rrl_ranges[i].hi; + bool is_last_iter = (i == (list->rrl_nr - 1)); + + rc = ranks2str(buf, buf_size, rank_lo, rank_hi, &len); + if (rc == -DER_INVAL) + D_GOTO(error, rc); + if (rc == -DER_TRUNC || + (rc == -DER_SUCCESS && buf_size < len + sizeof("]") + (is_last_iter ? 0 : 1))) { + int buf_idx = buf - line; + + D_REALLOC_ARRAY(buf, line, line_size, line_size + MINBYTES); + if (buf == NULL) + D_GOTO(error, rc = -DER_NOMEM); + line = buf; + line_size += MINBYTES; + buf = line + buf_idx; + buf_size += MINBYTES; + + rc = ranks2str(buf, buf_size, rank_lo, rank_hi, &len); } + D_ASSERT(rc == -DER_SUCCESS && + buf_size >= len + sizeof("]") + (is_last_iter ? 0 : 1)); - if (ret >= remaining) { - err = EOVERFLOW; - D_WARN("rank set has been partially serialized\n"); - break; + if (!is_last_iter) { + buf[len] = ','; + len++; } - remaining -= ret; - linepos += ret; + buf += len; + buf_size -= len; } - memcpy(linepos, "]", 2u); + memcpy(buf, "]", sizeof("]")); - if (err != 0) - *truncated = true; + *ranks_str = line; + D_GOTO(out, rc = -DER_SUCCESS); - return line; +error: + D_FREE(line); +out: + return rc; } void diff --git a/src/gurt/tests/test_gurt.c b/src/gurt/tests/test_gurt.c index db4f3015ff0..92ca0aff9e1 100644 --- a/src/gurt/tests/test_gurt.c +++ b/src/gurt/tests/test_gurt.c @@ -2511,6 +2511,223 @@ test_d_setenv(void **state) assert_false(d_isenv_def("foo")); } +static void +test_d_rank_list_to_str(void **state) +{ + d_rank_list_t *ranks; + char *ranks_str = NULL; + int i; + char *line; + char *buf; + int rc; + + // Test with null list + rc = d_rank_list_to_str(NULL, &ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_null(ranks_str); + + // Test with empty list + ranks = d_rank_list_alloc(0); + assert_non_null(ranks); + + rc = d_rank_list_to_str(ranks, &ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_string_equal(ranks_str, "[]"); + + D_FREE(ranks_str); + d_rank_list_free(ranks); + + // Test with one rank + ranks = d_rank_list_alloc(1); + assert_non_null(ranks); + ranks->rl_ranks[0] = 2; + + rc = d_rank_list_to_str(ranks, &ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_string_equal(ranks_str, "[2]"); + + D_FREE(ranks_str); + d_rank_list_free(ranks); + + // Test with 4 ranks and two ranges + ranks = d_rank_list_alloc(4); + assert_non_null(ranks); + ranks->rl_ranks[0] = 2; + ranks->rl_ranks[1] = 1; + ranks->rl_ranks[2] = 5; + ranks->rl_ranks[3] = 3; + + rc = d_rank_list_to_str(ranks, &ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_string_equal(ranks_str, "[1-3,5]"); + + D_FREE(ranks_str); + d_rank_list_free(ranks); + + // Test with long list + ranks = d_rank_list_alloc(1024); + assert_non_null(ranks); + D_ALLOC(line, 1024 * 5); + assert_non_null(line); + buf = line; + buf[0] = '['; + buf++; + for (i = 0; i < 1024; ++i) { + int len; + + ranks->rl_ranks[i] = 2 * i + 1; + len = sprintf(buf, "%d,", ranks->rl_ranks[i]); + assert_true(len > 0); + buf += len; + } + memcpy(buf - 1, "]", sizeof("]")); + + rc = d_rank_list_to_str(ranks, &ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_non_null(ranks_str); + assert_string_equal(ranks_str, line); + + D_FREE(line); + D_FREE(ranks_str); + d_rank_list_free(ranks); +} + +static void +test_d_rank_range_list_create_from_ranks(void **state) +{ + d_rank_list_t *ranks; + d_rank_range_list_t *range_list; + + // Test with null list + range_list = d_rank_range_list_create_from_ranks(NULL); + assert_non_null(range_list); + assert_int_equal(range_list->rrl_nr, 0); + + d_rank_range_list_free(range_list); + + // Test with empty list + ranks = d_rank_list_alloc(0); + assert_non_null(ranks); + + range_list = d_rank_range_list_create_from_ranks(ranks); + assert_non_null(range_list); + assert_int_equal(range_list->rrl_nr, 0); + + d_rank_range_list_free(range_list); + d_rank_list_free(ranks); + + // Test with one rank + ranks = d_rank_list_alloc(1); + assert_non_null(ranks); + ranks->rl_ranks[0] = 2; + + range_list = d_rank_range_list_create_from_ranks(ranks); + assert_non_null(range_list); + assert_int_equal(range_list->rrl_nr, 1); + assert_int_equal(range_list->rrl_ranges[0].lo, 2); + assert_int_equal(range_list->rrl_ranges[0].hi, 2); + + d_rank_range_list_free(range_list); + d_rank_list_free(ranks); + + // Test with 4 ranks and two ranges + ranks = d_rank_list_alloc(4); + assert_non_null(ranks); + ranks->rl_ranks[0] = 2; + ranks->rl_ranks[1] = 1; + ranks->rl_ranks[2] = 5; + ranks->rl_ranks[3] = 3; + + range_list = d_rank_range_list_create_from_ranks(ranks); + assert_non_null(range_list); + assert_int_equal(range_list->rrl_nr, 2); + assert_int_equal(range_list->rrl_ranges[0].lo, 1); + assert_int_equal(range_list->rrl_ranges[0].hi, 3); + assert_int_equal(range_list->rrl_ranges[1].lo, 5); + assert_int_equal(range_list->rrl_ranges[1].hi, 5); + + d_rank_range_list_free(range_list); + d_rank_list_free(ranks); +} + +static void +test_d_rank_range_list_str(void **state) +{ + d_rank_range_list_t *range_list; + char *ranks_str = NULL; + int i; + char *line; + char *buf; + int rc; + + // Test with empty list + range_list = d_rank_range_list_alloc(0); + assert_non_null(range_list); + + rc = d_rank_range_list_str(range_list, &ranks_str); + assert_int_equal(rc, 0); + assert_string_equal(ranks_str, "[]"); + + D_FREE(ranks_str); + d_rank_range_list_free(range_list); + + // Test with one rank + range_list = d_rank_range_list_alloc(1); + assert_non_null(range_list); + range_list->rrl_ranges[0].lo = 2; + range_list->rrl_ranges[0].hi = 2; + + rc = d_rank_range_list_str(range_list, &ranks_str); + assert_int_equal(rc, 0); + assert_string_equal(ranks_str, "[2]"); + + D_FREE(ranks_str); + d_rank_range_list_free(range_list); + + // Test with 4 ranks and two ranges + range_list = d_rank_range_list_alloc(2); + assert_non_null(range_list); + range_list->rrl_ranges[0].lo = 1; + range_list->rrl_ranges[0].hi = 3; + range_list->rrl_ranges[1].lo = 5; + range_list->rrl_ranges[1].hi = 5; + + rc = d_rank_range_list_str(range_list, &ranks_str); + assert_int_equal(rc, 0); + assert_string_equal(ranks_str, "[1-3,5]"); + + D_FREE(ranks_str); + d_rank_range_list_free(range_list); + + // Test with lot of ranks + range_list = d_rank_range_list_alloc(1024); + assert_non_null(range_list); + D_ALLOC(line, 1024 * 5); + assert_non_null(line); + buf = line; + buf[0] = '['; + buf++; + for (i = 0; i < 1024; ++i) { + int len; + + range_list->rrl_ranges[i].lo = i; + range_list->rrl_ranges[i].hi = i; + len = sprintf(buf, "%d,", i); + assert_true(len > 0); + buf += len; + } + memcpy(buf - 1, "]", sizeof("]")); + + rc = d_rank_range_list_str(range_list, &ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_non_null(ranks_str); + assert_string_equal(ranks_str, line); + + D_FREE(line); + D_FREE(ranks_str); + d_rank_range_list_free(range_list); +} + int main(int argc, char **argv) { @@ -2547,7 +2764,10 @@ main(int argc, char **argv) teardown_getenv_mocks), cmocka_unit_test_setup_teardown(test_d_getenv_uint64_t, setup_getenv_mocks, teardown_getenv_mocks), - cmocka_unit_test(test_d_setenv)}; + cmocka_unit_test(test_d_setenv), + cmocka_unit_test(test_d_rank_list_to_str), + cmocka_unit_test(test_d_rank_range_list_create_from_ranks), + cmocka_unit_test(test_d_rank_range_list_str)}; d_register_alt_assert(mock_assert); diff --git a/src/include/gurt/common.h b/src/include/gurt/common.h index 8dec9ea9871..241dcd9b74a 100644 --- a/src/include/gurt/common.h +++ b/src/include/gurt/common.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -481,12 +482,14 @@ int d_rank_list_append(d_rank_list_t *rank_list, d_rank_t rank); int d_rank_list_dump(d_rank_list_t *rank_list, d_string_t name, int name_len); d_rank_list_t *uint32_array_to_rank_list(uint32_t *ints, size_t len); int rank_list_to_uint32_array(d_rank_list_t *rl, uint32_t **ints, size_t *len); -char *d_rank_list_to_str(d_rank_list_t *rank_list); +int + d_rank_list_to_str(d_rank_list_t *rank_list, char **rank_str); d_rank_range_list_t *d_rank_range_list_alloc(uint32_t size); d_rank_range_list_t *d_rank_range_list_realloc(d_rank_range_list_t *range_list, uint32_t size); d_rank_range_list_t *d_rank_range_list_create_from_ranks(d_rank_list_t *rank_list); -char *d_rank_range_list_str(d_rank_range_list_t *list, bool *truncated); +int + d_rank_range_list_str(d_rank_range_list_t *list, char **ranks_str); void d_rank_range_list_free(d_rank_range_list_t *range_list); #ifdef FAULT_INJECTION diff --git a/src/mgmt/srv_drpc.c b/src/mgmt/srv_drpc.c index 6361cb8de18..a389c7b0f3e 100644 --- a/src/mgmt/srv_drpc.c +++ b/src/mgmt/srv_drpc.c @@ -1165,10 +1165,10 @@ add_props_to_resp(daos_prop_t *prop, Mgmt__PoolGetPropResp *resp) D_ERROR("svc rank list unset\n"); D_GOTO(out, rc = -DER_INVAL); } - resp_props[j]->strval = d_rank_list_to_str( - (d_rank_list_t *)entry->dpe_val_ptr); - if (resp_props[j]->strval == NULL) - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_to_str((d_rank_list_t *)entry->dpe_val_ptr, + &resp_props[j]->strval); + if (rc != 0) + D_GOTO(out, rc); resp_props[j]->value_case = MGMT__POOL_PROPERTY__VALUE_STRVAL; break; @@ -1744,14 +1744,10 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) uuid_t uuid; daos_pool_info_t pool_info = {0}; d_rank_list_t *svc_ranks; - d_rank_list_t *ranks = NULL; - d_rank_range_list_t *range_list = NULL; - d_rank_range_list_t *range_list1 = NULL; + d_rank_list_t *ranks = NULL; d_rank_list_t *dead_ranks = NULL; char *range_list_str = NULL; char *dead_ranks_str = NULL; - bool truncated; - bool truncated1; size_t len; uint8_t *body; @@ -1790,25 +1786,27 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) } /* Calculate and stringify rank ranges to return to control plane for display */ - range_list = d_rank_range_list_create_from_ranks(ranks); - if (range_list == NULL) - D_GOTO(out_ranks, rc = -DER_NOMEM); - range_list_str = d_rank_range_list_str(range_list, &truncated); - if (range_list_str == NULL) - D_GOTO(out_ranges, rc = -DER_NOMEM); - range_list1 = d_rank_range_list_create_from_ranks(dead_ranks); - if (range_list1 == NULL) - D_GOTO(out_suspect, rc = -DER_NOMEM); - dead_ranks_str = d_rank_range_list_str(range_list1, &truncated1); - if (dead_ranks_str == NULL) { - DL_ERROR(rc, DF_UUID ": Failed to serialize the list of suspect ranks", + rc = d_rank_list_to_str(ranks, &range_list_str); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": Failed to serialize the list of enabled ranks", + DP_UUID(uuid)); + D_GOTO(out_ranks, rc); + } else if (range_list_str == NULL) { + D_STRNDUP(range_list_str, "[]", strlen("[]")); + } + + rc = d_rank_list_to_str(dead_ranks, &dead_ranks_str); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": Failed to serialize the list of disabled ranks", DP_UUID(uuid)); - D_GOTO(out_suspect, rc = -DER_NOMEM); + D_GOTO(out_dead_ranks, rc); + } else if (dead_ranks_str == NULL) { + D_STRNDUP(dead_ranks_str, "[]", strlen("[]")); } - D_DEBUG(DB_MGMT, DF_UUID ": %s ranks: %s%s, dead_ranks: %s%s\n", DP_UUID(uuid), + + D_DEBUG(DB_MGMT, DF_UUID ": %s ranks: %s, dead_ranks: %s\n", DP_UUID(uuid), pool_info.pi_bits & DPI_ENGINES_ENABLED ? "ENABLED" : "DISABLED", range_list_str, - truncated ? " ...(TRUNCATED)" : "", dead_ranks_str, - truncated1 ? "...(TRUNCATED)" : ""); + dead_ranks_str); /* Populate the response */ resp.query_mask = pool_info.pi_bits; @@ -1823,12 +1821,11 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) resp.version = pool_info.pi_map_ver; resp.enabled_ranks = (req->query_mask & DPI_ENGINES_ENABLED) ? range_list_str : ""; resp.disabled_ranks = (req->query_mask & DPI_ENGINES_DISABLED) ? range_list_str : ""; - if (dead_ranks_str != NULL) - resp.dead_ranks = dead_ranks_str; + resp.dead_ranks = dead_ranks_str; D_ALLOC_ARRAY(resp.tier_stats, DAOS_MEDIA_MAX); if (resp.tier_stats == NULL) - D_GOTO(out_suspect, rc = -DER_NOMEM); + D_GOTO(out_dead_ranks, rc = -DER_NOMEM); storage_usage_stats_from_pool_space(&scm, &pool_info.pi_space, DAOS_MEDIA_SCM); @@ -1843,11 +1840,8 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) pool_rebuild_status_from_info(&rebuild, &pool_info.pi_rebuild_st); resp.rebuild = &rebuild; -out_suspect: - d_rank_range_list_free(range_list1); +out_dead_ranks: d_rank_list_free(dead_ranks); -out_ranges: - d_rank_range_list_free(range_list); out_ranks: d_rank_list_free(ranks); out_svc_ranks: diff --git a/src/mgmt/srv_pool.c b/src/mgmt/srv_pool.c index 5f35f727281..e7b974a7c40 100644 --- a/src/mgmt/srv_pool.c +++ b/src/mgmt/srv_pool.c @@ -197,16 +197,13 @@ ds_mgmt_create_pool(uuid_t pool_uuid, const char *group, char *tgt_dev, d_rank_l if (!d_rank_list_identical(pg_targets, targets)) { char *pg_str, *tgt_str; - pg_str = d_rank_list_to_str(pg_ranks); - if (pg_str == NULL) { - rc = -DER_NOMEM; + rc = d_rank_list_to_str(pg_ranks, &pg_str); + if (rc != 0) D_GOTO(out, rc); - } - tgt_str = d_rank_list_to_str(targets); - if (tgt_str == NULL) { + rc = d_rank_list_to_str(targets, &tgt_str); + if (rc != 0) { D_FREE(pg_str); - rc = -DER_NOMEM; D_GOTO(out, rc); } From f784c5b8cdff102d862e74c5817d3a1d5fcbec62 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Fri, 14 Nov 2025 16:55:11 +0000 Subject: [PATCH 38/43] DAOS-17508 vos: allow DAOS_INTENT_CHECK for EV iterators (#16737) (#16784) DAOS_INTENT_CHECK is intended for offline utilities like DDB and DLCK to inspect the contents of the pool while it is idle. EV iterators must support it as well. Signed-off-by: Jan Michalski Co-authored-by: Fan Yong --- src/include/daos_srv/evtree.h | 23 +++++++++++++---------- src/vos/evt_iter.c | 2 ++ src/vos/vos_obj.c | 2 ++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/include/daos_srv/evtree.h b/src/include/daos_srv/evtree.h index 8ae4f12638e..366b8d8b1e5 100644 --- a/src/include/daos_srv/evtree.h +++ b/src/include/daos_srv/evtree.h @@ -644,19 +644,19 @@ int evt_debug(daos_handle_t toh, int debug_level); enum { /** Return extents visible in the search rectangle */ - EVT_ITER_VISIBLE = (1 << 0), + EVT_ITER_VISIBLE = (1 << 0), /** Add fully or partially covered extents to EVT_ITER_VISIBLE */ - EVT_ITER_COVERED = (1 << 1) | EVT_ITER_VISIBLE, + EVT_ITER_COVERED = (1 << 1) | EVT_ITER_VISIBLE, /** Skip visible holes (Only valid with EVT_ITER_VISIBLE) */ - EVT_ITER_SKIP_HOLES = (1 << 2), + EVT_ITER_SKIP_HOLES = (1 << 2), /** * Use the embedded iterator of the open handle. * It can reduce memory consumption, but state of iterator can be * overwritten by other tree operation. */ - EVT_ITER_EMBEDDED = (1 << 3), + EVT_ITER_EMBEDDED = (1 << 3), /** Reverse iterator (ordered iterator only) */ - EVT_ITER_REVERSE = (1 << 4), + EVT_ITER_REVERSE = (1 << 4), /* If EVT_ITER_VISIBLE is set, evt_iter_probe will calculate and cache * visible extents and iterate through the cached extents. Each * rectangle will be marked as visible or covered. The partial bit will @@ -668,15 +668,18 @@ enum { * search rectangle, including punched extents, are returned. */ /** The iterator is for purge operation */ - EVT_ITER_FOR_PURGE = (1 << 5), + EVT_ITER_FOR_PURGE = (1 << 5), /** The iterator is for data migration scan */ - EVT_ITER_FOR_MIGRATION = (1 << 6), + EVT_ITER_FOR_MIGRATION = (1 << 6), /** The iterator is for data discard */ - EVT_ITER_FOR_DISCARD = (1 << 7), + EVT_ITER_FOR_DISCARD = (1 << 7), /** Skip visible data (Only valid with EVT_ITER_VISIBLE) */ - EVT_ITER_SKIP_DATA = (1 << 8), + EVT_ITER_SKIP_DATA = (1 << 8), /** Only process removals */ - EVT_ITER_REMOVALS = (1 << 9), + EVT_ITER_REMOVALS = (1 << 9), + /** Checking whether the target is aborted or not. */ + EVT_ITER_FOR_CHECK = (1 << 10), + }; D_CASSERT((int)EVT_VISIBLE == (int)EVT_ITER_VISIBLE); diff --git a/src/vos/evt_iter.c b/src/vos/evt_iter.c index fe5ad6b4f48..c251fb3b45c 100644 --- a/src/vos/evt_iter.c +++ b/src/vos/evt_iter.c @@ -214,6 +214,8 @@ evt_iter_intent(struct evt_iterator *iter) return DAOS_INTENT_DISCARD; if (iter->it_options & EVT_ITER_FOR_MIGRATION) return DAOS_INTENT_MIGRATION; + if (iter->it_options & EVT_ITER_FOR_CHECK) + return DAOS_INTENT_CHECK; return DAOS_INTENT_DEFAULT; } diff --git a/src/vos/vos_obj.c b/src/vos/vos_obj.c index 364d3d9145c..fff7e059eed 100644 --- a/src/vos/vos_obj.c +++ b/src/vos/vos_obj.c @@ -1526,6 +1526,8 @@ recx_get_flags(struct vos_obj_iter *oiter, bool embed) options |= EVT_ITER_FOR_DISCARD; if (oiter->it_flags & VOS_IT_FOR_MIGRATION) options |= EVT_ITER_FOR_MIGRATION; + if (oiter->it_flags & VOS_IT_FOR_CHECK) + options |= EVT_ITER_FOR_CHECK; return options; } From 601bc29b9c397332e20dcb52c819d042c63dc7e6 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Fri, 14 Nov 2025 10:20:55 -0700 Subject: [PATCH 39/43] DAOS-17854 control: Only add voter if it doesn't exist (#16721) (#17117) Don't modify the voter list unless the voter isn't already listed. Signed-off-by: Kris Jacque Co-authored-by: Tom Nabarro --- src/control/system/raft/database.go | 57 +++++++++++++++++++++++------ src/control/system/raft/mocks.go | 26 +++++++++---- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/src/control/system/raft/database.go b/src/control/system/raft/database.go index 5aedf7cd460..926d35610a6 100644 --- a/src/control/system/raft/database.go +++ b/src/control/system/raft/database.go @@ -54,6 +54,7 @@ type ( Barrier(time.Duration) raft.Future Shutdown() raft.Future State() raft.RaftState + GetConfiguration() raft.ConfigurationFuture } // syncRaft provides a wrapper for synchronized access to the @@ -734,6 +735,21 @@ func (db *Database) RemoveMember(m *system.Member) error { return db.submitMemberUpdate(raftOpRemoveMember, &memberUpdate{Member: m}) } +func raftSvcHasVoter(svc raftService, voterAddr raft.ServerAddress) (bool, error) { + cfgFuture := svc.GetConfiguration() + if cfgFuture.Error() != nil { + return false, errors.Wrapf(cfgFuture.Error(), "get raft configuration") + } + + cfg := cfgFuture.Configuration() + for _, rs := range cfg.Servers { + if rs.Address == voterAddr { + return true, nil + } + } + return false, nil +} + func (db *Database) manageVoter(vc *system.Member, op raftOp) error { // Ignore self as a voter candidate. if common.CmpTCPAddr(db.replicaAddr, vc.Addr) { @@ -749,12 +765,36 @@ func (db *Database) manageVoter(vc *system.Member, op raftOp) error { rsa := raft.ServerAddress(vc.Addr.String()) switch op { - case raftOpAddMember: - case raftOpUpdateMember, raftOpRemoveMember: - // If we're updating an existing member, we need to kick it out of the - // raft cluster and then re-add it so that it doesn't hijack the campaign. - db.log.Debugf("removing %s as a current raft voter", vc) + case raftOpAddMember, raftOpUpdateMember: + return db.raft.withReadLock(func(svc raftService) error { + if hasVoter, err := raftSvcHasVoter(svc, rsa); err != nil { + return errors.Wrapf(err, "check if %s is already a raft voter", rsa) + } else if !hasVoter { + db.log.Debugf("adding %s as a new raft voter", vc) + return svc.AddVoter(rsi, rsa, 0, 0).Error() + } + + // It's already a voter + return nil + }) + case raftOpRemoveMember: + // We only need to remove a voter if all of its members are gone. if err := db.raft.withReadLock(func(svc raftService) error { + members, err := db.FindMembersByAddr(vc.Addr) + if err != nil { + return errors.Wrapf(err, "look up members at address %s", rsa) + } + for _, m := range members { + if m.UUID == vc.UUID { + // This is the member we're planning to remove + continue + } + + // There are members other than this one + return nil + } + + db.log.Debugf("removing %s as a current raft voter", vc) return svc.RemoveServer(rsi, 0, 0).Error() }); err != nil { return errors.Wrapf(err, "failed to remove %q as a raft replica", vc.Addr) @@ -763,13 +803,6 @@ func (db *Database) manageVoter(vc *system.Member, op raftOp) error { return errors.Errorf("unhandled manageVoter op: %s", op) } - db.log.Debugf("adding %s as a new raft voter", vc) - if err := db.raft.withReadLock(func(svc raftService) error { - return svc.AddVoter(rsi, rsa, 0, 0).Error() - }); err != nil { - return errors.Wrapf(err, "failed to add %q as raft replica", vc.Addr) - } - return nil } diff --git a/src/control/system/raft/mocks.go b/src/control/system/raft/mocks.go index 9f6b9a846e8..6104a3532a0 100644 --- a/src/control/system/raft/mocks.go +++ b/src/control/system/raft/mocks.go @@ -26,13 +26,15 @@ type ( err error index uint64 response interface{} + config raft.Configuration } mockRaftServiceConfig struct { - LeaderCh <-chan bool - ServerAddress raft.ServerAddress - State raft.RaftState - LeadershipTransferErr error - BarrierReturn raft.Future + LeaderCh <-chan bool + ServerAddress raft.ServerAddress + State raft.RaftState + LeadershipTransferErr error + BarrierReturn raft.Future + GetConfigurationReturn raft.ConfigurationFuture } mockRaftService struct { cfg mockRaftServiceConfig @@ -42,9 +44,10 @@ type ( ) // mockRaftFuture implements raft.Future, raft.IndexFuture, and raft.ApplyFuture -func (mrf *mockRaftFuture) Error() error { return mrf.err } -func (mrf *mockRaftFuture) Index() uint64 { return mrf.index } -func (mrf *mockRaftFuture) Response() interface{} { return mrf.response } +func (mrf *mockRaftFuture) Error() error { return mrf.err } +func (mrf *mockRaftFuture) Index() uint64 { return mrf.index } +func (mrf *mockRaftFuture) Response() interface{} { return mrf.response } +func (mrf *mockRaftFuture) Configuration() raft.Configuration { return mrf.config } func (mrs *mockRaftService) Apply(cmd []byte, timeout time.Duration) raft.ApplyFuture { mrs.fsm.Apply(&raft.Log{Data: cmd}) @@ -95,6 +98,13 @@ func (mrs *mockRaftService) Barrier(time.Duration) raft.Future { return mrs.cfg.BarrierReturn } +func (mrs *mockRaftService) GetConfiguration() raft.ConfigurationFuture { + if mrs.cfg.GetConfigurationReturn == nil { + return &mockRaftFuture{} + } + return mrs.cfg.GetConfigurationReturn +} + // resetCalls resets call counts for the mock. func (mrs *mockRaftService) resetCalls() { mrs.addVoterCalledForAddrs = nil From 3835e9318f195c6e061abb09bf3b17c633c21490 Mon Sep 17 00:00:00 2001 From: Joseph Moore <26410038+jgmoore-or@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:21:49 -0700 Subject: [PATCH 40/43] DAOS-16478 test: Change pool autotest to use DAOS_EQ_NOWAIT. (#15936) (#17089) Signed-off-by: Joseph Moore --- src/utils/daos_autotest.c | 75 +++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/src/utils/daos_autotest.c b/src/utils/daos_autotest.c index a8b1b1a39b2..5d9f591b7fa 100644 --- a/src/utils/daos_autotest.c +++ b/src/utils/daos_autotest.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2020-2022 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -440,7 +441,9 @@ kv_put(daos_handle_t oh, daos_size_t size) char *val; daos_event_t *evp; int rc, usage_ratio1, usage_ratio2; - int eq_rc; + int eq_rc; + int num_events; + int completions = 0; clock_t last_query = start, current; deadline_count = 1; @@ -485,12 +488,14 @@ kv_put(daos_handle_t oh, daos_size_t size) * Max request in flight reached, wait for one i/o to * complete to reuse the slot */ - rc = daos_eq_poll(eq, 1, DAOS_EQ_WAIT, 1, &evp); - if (rc < 0) - break; - if (rc == 0) { - rc = -DER_IO; - break; + while (1) { + rc = daos_eq_poll(eq, 1, DAOS_EQ_NOWAIT, 1, &evp); + if (rc > 0) + break; + if (rc < 0) { + rc = -DER_IO; + break; + } } /** Check if completed operation failed */ @@ -540,19 +545,20 @@ kv_put(daos_handle_t oh, daos_size_t size) break; deadline_count++; - } /** Wait for completion of all in-flight requests */ - do { - eq_rc = daos_eq_poll(eq, 1, DAOS_EQ_WAIT, 1, &evp); - if (rc == 0 && eq_rc == 1) { - rc = evp->ev_error; + num_events = daos_eq_query(eq, DAOS_EQR_ALL, 0, NULL); + while (1) { + eq_rc = daos_eq_poll(eq, 1, DAOS_EQ_NOWAIT, 1, &evp); + if (eq_rc > 0) + completions += eq_rc; + if (eq_rc < 0) { + rc = eq_rc; + break; } - } while (eq_rc == 1); - - if (rc == 0 && eq_rc < 0) { - rc = eq_rc; + if (completions >= num_events) + break; } D_FREE(val); @@ -581,6 +587,8 @@ kv_get(daos_handle_t oh, daos_size_t size) uint64_t res = 0; int rc; int eq_rc; + int completions = 0; + int num_events; total_nr = deadline_count; setup_progress(); @@ -620,12 +628,14 @@ kv_get(daos_handle_t oh, daos_size_t size) * Max request in flight reached, wait for one i/o to * complete to reuse the slot */ - rc = daos_eq_poll(eq, 1, DAOS_EQ_WAIT, 1, &evp); - if (rc < 0) - break; - if (rc == 0) { - rc = -DER_IO; - break; + while (1) { + rc = daos_eq_poll(eq, 1, DAOS_EQ_NOWAIT, 1, &evp); + if (rc > 0) + break; + if (rc < 0) { + rc = -DER_IO; + break; + } } /** Check if completed operation failed */ @@ -665,11 +675,12 @@ kv_get(daos_handle_t oh, daos_size_t size) } /** Wait for completion of all in-flight requests */ - do { - eq_rc = daos_eq_poll(eq, 1, DAOS_EQ_WAIT, 1, &evp); - if (rc == 0 && eq_rc == 1) { - rc = evp->ev_error; - if (rc == 0) { + num_events = daos_eq_query(eq, DAOS_EQR_ALL, 0, NULL); + while (1) { + eq_rc = daos_eq_poll(eq, 1, DAOS_EQ_NOWAIT, 1, &evp); + if (eq_rc > 0) { + completions += eq_rc; + if (!evp->ev_error) { int slot = evp - ev_array; if (val_sz[slot] != size) { @@ -679,10 +690,12 @@ kv_get(daos_handle_t oh, daos_size_t size) } } } - } while (eq_rc == 1); - - if (rc == 0 && eq_rc < 0) { - rc = eq_rc; + if (eq_rc < 0) { + rc = eq_rc; + break; + } + if (completions >= num_events) + break; } D_FREE(val); From ebb845df1e2bba620fdfc8a809bd485a698357f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:24:49 -0800 Subject: [PATCH 41/43] DAOS-18211 cq: bump GHA versions (#17096) (#17097) Updates `EnricoMi/publish-unit-test-result-action` from 2.20.0 to 2.21.0 Updates `actions/upload-artifact` from 4 to 5 Updates `codespell-project/actions-codespell` from 2.1 to 2.2 Updates `github/codeql-action` from 4.30.7 to 4.31.2 Signed-off-by: dependabot[bot] Signed-off-by: Dalton Bohning --- .github/workflows/bullseye-coverage.yml | 12 ++++++------ .github/workflows/ci2.yml | 2 +- .github/workflows/landing-builds.yml | 2 +- .github/workflows/linting.yml | 6 +++--- .github/workflows/ossf-scorecard.yml | 4 ++-- .github/workflows/rpm-build-and-test.yml | 12 ++++++------ .github/workflows/trivy.yml | 4 ++-- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/bullseye-coverage.yml b/.github/workflows/bullseye-coverage.yml index aa553d87a36..b6b5c20f8fb 100644 --- a/.github/workflows/bullseye-coverage.yml +++ b/.github/workflows/bullseye-coverage.yml @@ -366,7 +366,7 @@ jobs: if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: check_name: ${{ env.STAGE_NAME }} Test Results github_token: ${{ secrets.GITHUB_TOKEN }} @@ -374,14 +374,14 @@ jobs: - name: Publish artifacts if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} artifacts path: ${{ env.STAGE_NAME }}/** - name: Upload test results if: (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} test-results path: ${{ env.STAGE_NAME }}/**/results.xml @@ -634,7 +634,7 @@ jobs: if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: check_name: ${{ env.STAGE_NAME }} Test Results github_token: ${{ secrets.GITHUB_TOKEN }} @@ -642,14 +642,14 @@ jobs: - name: Publish artifacts if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} artifacts path: ${{ env.STAGE_NAME }}/** - name: Upload test results if: (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} test-results path: ${{ env.STAGE_NAME }}/**/results.xml diff --git a/.github/workflows/ci2.yml b/.github/workflows/ci2.yml index c49e46f3785..3832de6f163 100644 --- a/.github/workflows/ci2.yml +++ b/.github/workflows/ci2.yml @@ -68,7 +68,7 @@ jobs: - name: Publish NLT test results if: always() # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} files: nlt-junit.xml diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index 22f5eb2a3c8..556418160ea 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -143,7 +143,7 @@ jobs: - name: Publish NLT test results if: always() # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} files: nlt-junit.xml diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 4d904905d87..8ed8011a2bf 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -129,7 +129,7 @@ jobs: - name: Run check run: doxygen Doxyfile - name: 'Upload Artifact' - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: API Documentation path: docs/doxygen/html/ @@ -164,7 +164,7 @@ jobs: - name: Install extra python packages run: python3 -m pip install --requirement utils/cq/requirements.txt - name: Run check - uses: codespell-project/actions-codespell@406322ec52dd7b488e48c1c4b82e2a8b3a1bf630 # master + uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # master with: skip: ./src/control/vendor,./src/control/go.sum,./.git ignore_words_file: ci/codespell.ignores @@ -191,7 +191,7 @@ jobs: with: target: ${{ steps.get_merge_base.outputs.ref }} - name: Export changes - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 if: failure() with: name: format-patch-for-pr-${{ github.event.pull_request.number }} diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index 8546979fed1..0c36cecbb8e 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -62,7 +62,7 @@ jobs: # uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: SARIF file path: results.sarif @@ -71,6 +71,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7 + uses: github/codeql-action/upload-sarif@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: sarif_file: results.sarif diff --git a/.github/workflows/rpm-build-and-test.yml b/.github/workflows/rpm-build-and-test.yml index 7ff09072e6f..2648af1e31b 100644 --- a/.github/workflows/rpm-build-and-test.yml +++ b/.github/workflows/rpm-build-and-test.yml @@ -374,7 +374,7 @@ jobs: if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: check_name: ${{ env.STAGE_NAME }} Test Results github_token: ${{ secrets.GITHUB_TOKEN }} @@ -382,14 +382,14 @@ jobs: - name: Publish artifacts if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} artifacts path: ${{ env.STAGE_NAME }}/** - name: Upload test results if: (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} test-results path: ${{ env.STAGE_NAME }}/**/results.xml @@ -642,7 +642,7 @@ jobs: if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: check_name: ${{ env.STAGE_NAME }} Test Results github_token: ${{ secrets.GITHUB_TOKEN }} @@ -650,14 +650,14 @@ jobs: - name: Publish artifacts if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} artifacts path: ${{ env.STAGE_NAME }}/** - name: Upload test results if: (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} test-results path: ${{ env.STAGE_NAME }}/**/results.xml diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 1f4b2f52a28..4a708cdcbf8 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -49,7 +49,7 @@ jobs: cp utils/trivy/.trivyignore report/trivyignore.txt - name: Upload the report to the GitHub artifact store - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: path: report/* name: trivy-report-daos @@ -68,7 +68,7 @@ jobs: trivy-config: 'utils/trivy/trivy.yaml' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7 + uses: github/codeql-action/upload-sarif@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: sarif_file: 'trivy-results.sarif' From 4373168d978bc5a6909943a424bfa4c6c40b115f Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Fri, 14 Nov 2025 15:57:16 -0700 Subject: [PATCH 42/43] DAOS-17865 control: Prevent duplicate pool props (#16883) (#17116) Due to a bug in parsing props, it was possible to crash the daos_engine by entering the same pool property twice during pool create by using the --properties flag twice. A prop could also be duplicated within the --properties flag without causing an error. This patch checks for duplication at three different points: - Within all props set within a --properties flag. - Within different property strings, if --properties was used multiple times. - At the engine level. Signed-off-by: Kris Jacque --- src/client/dfs/cont.c | 49 ++++++-- src/common/prop.c | 36 ++++-- src/common/tests/prop_tests.c | 152 +++++++++++++++++++++++-- src/control/cmd/dmg/pool.go | 4 +- src/control/cmd/dmg/pool_properties.go | 41 ++++++- src/control/cmd/dmg/pool_test.go | 24 ++++ src/control/lib/ui/prop_flags.go | 4 + src/control/lib/ui/prop_flags_test.go | 5 + src/include/daos_prop.h | 3 + src/mgmt/srv_pool.c | 8 +- src/pool/srv_pool.c | 5 + src/utils/daos_hdlr.c | 5 +- 12 files changed, 304 insertions(+), 32 deletions(-) diff --git a/src/client/dfs/cont.c b/src/client/dfs/cont.c index 5f59885fbf5..8edc630ec1b 100644 --- a/src/client/dfs/cont.c +++ b/src/client/dfs/cont.c @@ -126,9 +126,13 @@ dfs_cont_create(daos_handle_t poh, uuid_t *cuuid, dfs_attr_t *attr, daos_handle_ struct daos_prop_co_roots roots; int rc, rc2; struct daos_prop_entry *dpe; + struct daos_prop_entry *roots_entry = NULL; + struct daos_prop_entry *layout_entry = NULL; struct timespec now; uint32_t cid_tf; uint32_t pa_domain; + uint32_t prop_nr = 0; + uint32_t props_to_add = 0; int cont_tf; if (cuuid == NULL) @@ -138,10 +142,23 @@ dfs_cont_create(daos_handle_t poh, uuid_t *cuuid, dfs_attr_t *attr, daos_handle_ return EINVAL; } - if (attr != NULL && attr->da_props != NULL) - prop = daos_prop_alloc(attr->da_props->dpp_nr + 2); - else - prop = daos_prop_alloc(2); + if (attr != NULL && attr->da_props != NULL) { + prop_nr = attr->da_props->dpp_nr; + + roots_entry = daos_prop_entry_get(attr->da_props, DAOS_PROP_CO_ROOTS); + if (roots_entry == NULL) + props_to_add++; + + layout_entry = daos_prop_entry_get(attr->da_props, DAOS_PROP_CO_LAYOUT_TYPE); + if (layout_entry == NULL) + props_to_add++; + } else { + /* roots and layout are still needed */ + props_to_add = 2; + } + prop_nr += props_to_add; + + prop = daos_prop_alloc(prop_nr); if (prop == NULL) return ENOMEM; @@ -267,13 +284,29 @@ dfs_cont_create(daos_handle_t poh, uuid_t *cuuid, dfs_attr_t *attr, daos_handle_ /* store SB & root OIDs as container property */ roots.cr_oids[2] = roots.cr_oids[3] = DAOS_OBJ_NIL; - prop->dpp_entries[prop->dpp_nr - 2].dpe_type = DAOS_PROP_CO_ROOTS; - rc = daos_prop_entry_set_ptr(&prop->dpp_entries[prop->dpp_nr - 2], &roots, sizeof(roots)); + + if (roots_entry == NULL) { + /* need to add roots prop to list */ + roots_entry = &prop->dpp_entries[prop->dpp_nr - props_to_add]; + roots_entry->dpe_type = DAOS_PROP_CO_ROOTS; + props_to_add--; + } + /* if an existing roots prop was passed in, it will be overwritten */ + rc = daos_prop_entry_set_ptr(roots_entry, &roots, sizeof(roots)); if (rc) D_GOTO(err_prop, rc = daos_der2errno(rc)); - prop->dpp_entries[prop->dpp_nr - 1].dpe_type = DAOS_PROP_CO_LAYOUT_TYPE; - prop->dpp_entries[prop->dpp_nr - 1].dpe_val = DAOS_PROP_CO_LAYOUT_POSIX; + if (layout_entry == NULL) { + /* need to add layout prop to list */ + layout_entry = &prop->dpp_entries[prop->dpp_nr - props_to_add]; + layout_entry->dpe_type = DAOS_PROP_CO_LAYOUT_TYPE; + props_to_add--; + } + + /* layout passed in will be overwritten */ + layout_entry->dpe_val = DAOS_PROP_CO_LAYOUT_POSIX; + + D_ASSERT(props_to_add == 0); rc = daos_cont_create(poh, cuuid, prop, NULL); if (rc) { diff --git a/src/common/prop.c b/src/common/prop.c index 5b927120c1b..a29d33948de 100644 --- a/src/common/prop.c +++ b/src/common/prop.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -135,7 +136,7 @@ daos_prop_merge2(daos_prop_t *old_prop, daos_prop_t *new_prop, daos_prop_t **out int rc; uint32_t result_nr; uint32_t i, result_i; - struct daos_prop_entry *entry; + struct daos_prop_entry *entry; if (old_prop == NULL || new_prop == NULL) { D_ERROR("NULL input\n"); @@ -143,8 +144,8 @@ daos_prop_merge2(daos_prop_t *old_prop, daos_prop_t *new_prop, daos_prop_t **out } /* - * We might override some values in the old prop. Need to account for that in the final prop - * count. + * We might override some values in the old prop. Need to account for that in the + * final prop count. */ result_nr = old_prop->dpp_nr; for (i = 0; i < new_prop->dpp_nr; i++) { @@ -162,8 +163,8 @@ daos_prop_merge2(daos_prop_t *old_prop, daos_prop_t *new_prop, daos_prop_t **out result_i = 0; for (i = 0; i < old_prop->dpp_nr; i++, result_i++) { - rc = daos_prop_entry_copy(&old_prop->dpp_entries[i], - &result->dpp_entries[result_i]); + rc = + daos_prop_entry_copy(&old_prop->dpp_entries[i], &result->dpp_entries[result_i]); if (rc != 0) goto err; } @@ -244,10 +245,14 @@ daos_prop_owner_group_valid(d_string_t owner) bool daos_prop_valid(daos_prop_t *prop, bool pool, bool input) { - uint32_t type; - uint64_t val; - struct daos_acl *acl_ptr; - int i; + uint32_t type; + uint64_t val; + struct daos_acl *acl_ptr; + int i; + const uint32_t seen_nr = pool ? DAOS_PROP_PO_NUM : DAOS_PROP_CO_NUM; + bool seen[seen_nr]; + + memset(seen, 0, sizeof(seen)); if (prop == NULL) { D_ERROR("NULL properties\n"); @@ -270,6 +275,8 @@ daos_prop_valid(daos_prop_t *prop, bool pool, bool input) for (i = 0; i < prop->dpp_nr; i++) { struct daos_co_status co_status; int rc; + int type_min; + int type_idx; type = prop->dpp_entries[i].dpe_type; if (pool) { @@ -278,6 +285,7 @@ daos_prop_valid(daos_prop_t *prop, bool pool, bool input) D_ERROR("invalid type %d for pool.\n", type); return false; } + type_min = DAOS_PROP_PO_MIN; } else { if (type <= DAOS_PROP_CO_MIN || type >= DAOS_PROP_CO_MAX) { @@ -285,7 +293,17 @@ daos_prop_valid(daos_prop_t *prop, bool pool, bool input) type); return false; } + type_min = DAOS_PROP_CO_MIN; } + + /* detect duplicate types */ + type_idx = type - type_min - 1; + if (seen[type_idx]) { + D_ERROR("type %d (idx=%d) appears more than once\n", type, type_idx); + return false; + } + seen[type_idx] = true; + /* for output parameter need not check entry value */ if (!input) continue; diff --git a/src/common/tests/prop_tests.c b/src/common/tests/prop_tests.c index 2f33f568da3..0c2c51f5c72 100644 --- a/src/common/tests/prop_tests.c +++ b/src/common/tests/prop_tests.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2020-2022 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -374,18 +375,153 @@ test_daos_prop_from_str(void **state) daos_prop_free(prop); } +static void +test_daos_prop_valid_null(void **state) +{ + assert_false(daos_prop_valid(NULL, false, false)); +} + +static void +test_daos_prop_valid_too_many(void **state) +{ + daos_prop_t *prop = daos_prop_alloc(0); + uint32_t nr = prop->dpp_nr; + + prop->dpp_nr = DAOS_PROP_ENTRIES_MAX_NR + 1; + + assert_false(daos_prop_valid(prop, true, false)); + + prop->dpp_nr = nr; + daos_prop_free(prop); +} + +static void +test_daos_prop_valid_inconsistent_nr_entries(void **state) +{ + const int nr = 2; + daos_prop_t *prop = daos_prop_alloc(nr); + struct daos_prop_entry *entries = prop->dpp_entries; + + D_PRINT("zero entries, but non-null array\n"); + prop->dpp_nr = 0; + assert_false(daos_prop_valid(prop, false, false)); + + D_PRINT("non-zero entries, but null array\n"); + prop->dpp_nr = nr; + prop->dpp_entries = NULL; + assert_false(daos_prop_valid(prop, false, false)); + + /* cleanup */ + prop->dpp_entries = entries; + daos_prop_free(prop); +} + +static daos_prop_t * +get_prop_with_duplicates(int total_nr, bool pool) +{ + daos_prop_t *prop = daos_prop_alloc(total_nr); + int i; + uint32_t prop_min; + + assert_true(total_nr >= 2); + + if (pool) + prop_min = DAOS_PROP_PO_MIN; + else + prop_min = DAOS_PROP_CO_MIN; + + for (i = 0; i < total_nr; i++) + prop->dpp_entries[i].dpe_type = prop_min + i + 1; + + /* last one is a duplicate type */ + prop->dpp_entries[total_nr - 1].dpe_type = prop->dpp_entries[total_nr - 2].dpe_type; + + return prop; +} + +static void +check_daos_prop_valid_duplicate_types(bool pool) +{ + daos_prop_t *prop; + + prop = get_prop_with_duplicates(5, pool); + + assert_false(daos_prop_valid(prop, pool, false)); + + daos_prop_free(prop); +} + +static void +test_daos_prop_valid_duplicate_types(void **state) +{ + D_PRINT("check pool prop with duplicate types\n"); + check_daos_prop_valid_duplicate_types(true); + + D_PRINT("check container prop with duplicate types\n"); + check_daos_prop_valid_duplicate_types(false); +} + +static void +test_daos_prop_valid_pool_success_no_val_check(void **state) +{ + const int nr = DAOS_PROP_PO_MAX - DAOS_PROP_PO_MIN - 1; + daos_prop_t *prop = daos_prop_alloc(nr); + int i; + + for (i = 0; i < nr; i++) + prop->dpp_entries[i].dpe_type = DAOS_PROP_PO_MIN + i + 1; + + assert_true(daos_prop_valid(prop, true, false)); + + daos_prop_free(prop); +} + +static void +test_daos_prop_valid_cont_success_no_val_check(void **state) +{ + const int nr = DAOS_PROP_CO_MAX - DAOS_PROP_CO_MIN - 1; + daos_prop_t *prop = daos_prop_alloc(nr); + int i; + + for (i = 0; i < nr; i++) + prop->dpp_entries[i].dpe_type = DAOS_PROP_CO_MIN + i + 1; + + assert_true(daos_prop_valid(prop, false, false)); + + daos_prop_free(prop); +} + +static int +suite_setup(void **state) +{ + return d_log_init(); +} + +static int +suite_teardown(void **state) +{ + d_log_fini(); + return 0; +} + int main(void) { const struct CMUnitTest tests[] = { - cmocka_unit_test(test_daos_prop_merge_null), - cmocka_unit_test(test_daos_prop_merge_empty), - cmocka_unit_test(test_daos_prop_merge_add_only), - cmocka_unit_test(test_daos_prop_merge_total_update), - cmocka_unit_test(test_daos_prop_merge_subset_update), - cmocka_unit_test(test_daos_prop_merge_add_and_update), - cmocka_unit_test(test_daos_prop_from_str), + cmocka_unit_test(test_daos_prop_merge_null), + cmocka_unit_test(test_daos_prop_merge_empty), + cmocka_unit_test(test_daos_prop_merge_add_only), + cmocka_unit_test(test_daos_prop_merge_total_update), + cmocka_unit_test(test_daos_prop_merge_subset_update), + cmocka_unit_test(test_daos_prop_merge_add_and_update), + cmocka_unit_test(test_daos_prop_from_str), + cmocka_unit_test(test_daos_prop_valid_null), + cmocka_unit_test(test_daos_prop_valid_too_many), + cmocka_unit_test(test_daos_prop_valid_inconsistent_nr_entries), + cmocka_unit_test(test_daos_prop_valid_duplicate_types), + cmocka_unit_test(test_daos_prop_valid_pool_success_no_val_check), + cmocka_unit_test(test_daos_prop_valid_cont_success_no_val_check), }; - return cmocka_run_group_tests_name("common_prop", tests, NULL, NULL); + return cmocka_run_group_tests_name("common_prop", tests, suite_setup, suite_teardown); } diff --git a/src/control/cmd/dmg/pool.go b/src/control/cmd/dmg/pool.go index d181c88d161..8b8b2dbe664 100644 --- a/src/control/cmd/dmg/pool.go +++ b/src/control/cmd/dmg/pool.go @@ -332,7 +332,7 @@ func (cmd *poolCreateCmd) Execute(args []string) error { User: cmd.UserName.String(), UserGroup: cmd.GroupName.String(), NumSvcReps: cmd.NumSvcReps, - Properties: cmd.Properties.ToSet, + Properties: cmd.Properties.ToSet.Slice(), Ranks: cmd.RankList.Ranks(), } @@ -819,7 +819,7 @@ func (cmd *poolSetPropCmd) Execute(_ []string) error { req := &control.PoolSetPropReq{ ID: cmd.PoolID().String(), - Properties: cmd.Args.Props.ToSet, + Properties: cmd.Args.Props.ToSet.Slice(), } err := control.PoolSetProp(cmd.MustLogCtx(), cmd.ctlInvoker, req) diff --git a/src/control/cmd/dmg/pool_properties.go b/src/control/cmd/dmg/pool_properties.go index 10921ef8cae..108c943c029 100644 --- a/src/control/cmd/dmg/pool_properties.go +++ b/src/control/cmd/dmg/pool_properties.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2022 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -7,16 +8,46 @@ package main import ( + "fmt" + "sort" + "github.com/jessevdk/go-flags" "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ui" ) +// PoolPropSet is a non-duplicated set of pool properties. +type PoolPropSet map[string]*daos.PoolProperty + +// Slice returns a sorted slice of the pool properties in the set. +func (pps PoolPropSet) Slice() []*daos.PoolProperty { + slice := make([]*daos.PoolProperty, 0, len(pps)) + for _, prop := range pps { + slice = append(slice, prop) + } + sort.Slice(slice, func(i, j int) bool { + return slice[i].Name < slice[j].Name + }) + + return slice +} + +// Add adds a property to the set, or overrides the existing value. +func (pps PoolPropSet) Add(prop *daos.PoolProperty) { + pps[prop.Name] = prop +} + +// HasProp checks if the named prop has already been added to the set. +func (pps PoolPropSet) HasProp(name string) bool { + _, found := pps[name] + return found +} + type PoolSetPropsFlag struct { ui.SetPropertiesFlag - ToSet []*daos.PoolProperty + ToSet PoolPropSet } func (f *PoolSetPropsFlag) UnmarshalFlag(fv string) error { @@ -28,17 +59,23 @@ func (f *PoolSetPropsFlag) UnmarshalFlag(fv string) error { return err } + if f.ToSet == nil { + f.ToSet = make(PoolPropSet) + } for _, key := range propHdlrs.Keys() { val, parsed := f.ParsedProps[key] if !parsed { continue } + if f.ToSet.HasProp(key) { + return fmt.Errorf("%q: same key was specified more than once", key) + } hdlr := propHdlrs[key] p := hdlr.GetProperty(key) if err := p.SetValue(val); err != nil { return err } - f.ToSet = append(f.ToSet, p) + f.ToSet.Add(p) } return nil diff --git a/src/control/cmd/dmg/pool_test.go b/src/control/cmd/dmg/pool_test.go index 325d24ee75d..b877ba61e6c 100644 --- a/src/control/cmd/dmg/pool_test.go +++ b/src/control/cmd/dmg/pool_test.go @@ -516,9 +516,27 @@ func TestPoolCommands(t *testing.T) { strings.Join([]string{ printRequest(t, &control.PoolCreateReq{ Properties: []*daos.PoolProperty{ + propWithVal("label", "label"), propWithVal("scrub", "timed"), propWithVal("scrub_freq", "1"), + }, + User: eUsr.Username + "@", + UserGroup: eGrp.Name + "@", + Ranks: []ranklist.Rank{}, + TierBytes: []uint64{uint64(testSize), 0}, + }), + }, " "), + nil, + }, + { + "Create pool with properties specified separately", + fmt.Sprintf("pool create label --scm-size %s --properties scrub:timed --properties scrub_freq:1", testSizeStr), + strings.Join([]string{ + printRequest(t, &control.PoolCreateReq{ + Properties: []*daos.PoolProperty{ propWithVal("label", "label"), + propWithVal("scrub", "timed"), + propWithVal("scrub_freq", "1"), }, User: eUsr.Username + "@", UserGroup: eGrp.Name + "@", @@ -528,6 +546,12 @@ func TestPoolCommands(t *testing.T) { }, " "), nil, }, + { + "Create pool with duplicate properties specified separately", + fmt.Sprintf("pool create label --scm-size %s --properties scrub:timed --properties scrub:timed,scrub_freq:1", testSizeStr), + "", + errors.New("more than once"), + }, // Exclude testing with multiple ranks is verified at the control API layer. { "Exclude a target with single target idx", diff --git a/src/control/lib/ui/prop_flags.go b/src/control/lib/ui/prop_flags.go index b8e6438c15d..6193d414d31 100644 --- a/src/control/lib/ui/prop_flags.go +++ b/src/control/lib/ui/prop_flags.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2023 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -158,6 +159,9 @@ func (f *SetPropertiesFlag) UnmarshalFlag(fv string) error { if len(value) == 0 { return propError("value must not be empty") } + if _, set := f.ParsedProps[key]; set { + return propError("%q: same key was specified more than once", key) + } f.ParsedProps[key] = value } diff --git a/src/control/lib/ui/prop_flags_test.go b/src/control/lib/ui/prop_flags_test.go index c013047d893..f5ce13b2ee1 100644 --- a/src/control/lib/ui/prop_flags_test.go +++ b/src/control/lib/ui/prop_flags_test.go @@ -47,6 +47,11 @@ func TestUI_SetPropertiesFlag_UnmarshalFlag(t *testing.T) { fv: strings.Repeat("x", ui.MaxPropKeyLen+1) + ":value", expErr: errors.New("key too long"), }, + "duplicated properties": { + settable: []string{"a", "b"}, + fv: "a:c,b:d,a:d", + expErr: errors.New("more than once"), + }, "valid properties": { settable: []string{"a", "b"}, fv: "b:d,a:c", diff --git a/src/include/daos_prop.h b/src/include/daos_prop.h index 3b7216efd0e..c57399744e6 100644 --- a/src/include/daos_prop.h +++ b/src/include/daos_prop.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2015-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -142,6 +143,8 @@ enum daos_pool_props { DAOS_PROP_PO_MAX, }; +#define DAOS_PROP_PO_NUM (DAOS_PROP_PO_MAX - DAOS_PROP_PO_MIN - 1) + #define DAOS_PROP_PO_EC_CELL_SZ_MIN (1UL << 10) #define DAOS_PROP_PO_EC_CELL_SZ_MAX (1UL << 30) diff --git a/src/mgmt/srv_pool.c b/src/mgmt/srv_pool.c index e7b974a7c40..4070bb9b00b 100644 --- a/src/mgmt/srv_pool.c +++ b/src/mgmt/srv_pool.c @@ -599,7 +599,13 @@ ds_mgmt_pool_set_prop(uuid_t pool_uuid, d_rank_list_t *svc_ranks, int rc; if (prop == NULL || prop->dpp_entries == NULL || prop->dpp_nr < 1) { - D_ERROR("invalid property list\n"); + D_ERROR("no properties in prop list\n"); + rc = -DER_INVAL; + goto out; + } + + if (!daos_prop_valid(prop, true, true)) { + D_ERROR("invalid properties\n"); rc = -DER_INVAL; goto out; } diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index be1e80535cb..f000ac9f4db 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -385,6 +385,11 @@ pool_prop_default_copy(daos_prop_t *prop_def, daos_prop_t *prop) if (prop == NULL || prop->dpp_nr == 0 || prop->dpp_entries == NULL) return 0; + if (!daos_prop_valid(prop, true, true)) { + D_ERROR("invalid pool property\n"); + return -DER_INVAL; + } + for (i = 0; i < prop->dpp_nr; i++) { entry = &prop->dpp_entries[i]; entry_def = daos_prop_entry_get(prop_def, entry->dpe_type); diff --git a/src/utils/daos_hdlr.c b/src/utils/daos_hdlr.c index 3d63aedb655..68f217cd12c 100644 --- a/src/utils/daos_hdlr.c +++ b/src/utils/daos_hdlr.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1490,8 +1491,8 @@ dm_connect(struct cmd_args_s *ap, rc = dm_cont_get_all_props(ap, ca->src_coh, &props, false, false, false); else - rc = dm_cont_get_all_props(ap, ca->src_coh, &props, - true, false, true); + rc = dm_cont_get_all_props(ap, ca->src_coh, &props, true, false, + false); if (rc != 0) { DH_PERROR_DER(ap, rc, "Failed to get container properties"); D_GOTO(err, rc); From 1bb3538cf2852cb8945cd44038ebfc5fe545a97c Mon Sep 17 00:00:00 2001 From: Di Wang Date: Sun, 16 Nov 2025 20:27:37 +0000 Subject: [PATCH 43/43] Merge from release-2.6 Signed-off-by: Di Wang --- src/mgmt/srv_drpc.c | 87 +++++++++++++++++++++++--------------------- src/object/srv_obj.c | 2 +- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/src/mgmt/srv_drpc.c b/src/mgmt/srv_drpc.c index 960f07a6a7b..3fd47be573d 100644 --- a/src/mgmt/srv_drpc.c +++ b/src/mgmt/srv_drpc.c @@ -1748,22 +1748,24 @@ pool_query_free_tier_stats(Mgmt__PoolQueryResp *resp) void ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) { - struct drpc_alloc alloc = PROTO_ALLOCATOR_INIT(alloc); - int rc = 0; - Mgmt__PoolQueryReq *req; - Mgmt__PoolQueryResp resp = MGMT__POOL_QUERY_RESP__INIT; - Mgmt__StorageUsageStats scm = MGMT__STORAGE_USAGE_STATS__INIT; - Mgmt__StorageUsageStats nvme = MGMT__STORAGE_USAGE_STATS__INIT; - Mgmt__PoolRebuildStatus rebuild = MGMT__POOL_REBUILD_STATUS__INIT; - uuid_t uuid; - daos_pool_info_t pool_info = {0}; - d_rank_list_t *svc_ranks; - d_rank_list_t *ranks = NULL; - d_rank_list_t *dead_ranks = NULL; - char *range_list_str = NULL; - char *dead_ranks_str = NULL; - size_t len; - uint8_t *body; + struct drpc_alloc alloc = PROTO_ALLOCATOR_INIT(alloc); + int rc = 0; + Mgmt__PoolQueryReq *req; + Mgmt__PoolQueryResp resp = MGMT__POOL_QUERY_RESP__INIT; + Mgmt__StorageUsageStats scm = MGMT__STORAGE_USAGE_STATS__INIT; + Mgmt__StorageUsageStats nvme = MGMT__STORAGE_USAGE_STATS__INIT; + Mgmt__PoolRebuildStatus rebuild = MGMT__POOL_REBUILD_STATUS__INIT; + uuid_t uuid; + daos_pool_info_t pool_info = {0}; + d_rank_list_t *svc_ranks = NULL; + d_rank_list_t *enabled_ranks = NULL; + d_rank_list_t *disabled_ranks = NULL; + d_rank_list_t *dead_ranks = NULL; + char *enabled_ranks_str = NULL; + char *disabled_ranks_str = NULL; + char *dead_ranks_str = NULL; + size_t len; + uint8_t *body; req = mgmt__pool_query_req__unpack(&alloc.alloc, drpc_req->body.len, drpc_req->body.data); @@ -1792,28 +1794,33 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) D_GOTO(error, rc); } - /* Calculate and stringify rank ranges to return to control plane for display */ - rc = d_rank_list_to_str(ranks, &range_list_str); + rc = d_rank_list_to_str(enabled_ranks, &enabled_ranks_str); if (rc != 0) { DL_ERROR(rc, DF_UUID ": Failed to serialize the list of enabled ranks", DP_UUID(uuid)); - D_GOTO(out_ranks, rc); - } else if (range_list_str == NULL) { - D_STRNDUP(range_list_str, "[]", strlen("[]")); + D_GOTO(error, rc); } + if (enabled_ranks_str != NULL) + D_DEBUG(DB_MGMT, DF_UUID ": list of enabled ranks: %s\n", DP_UUID(uuid), + enabled_ranks_str); - rc = d_rank_list_to_str(dead_ranks, &dead_ranks_str); + rc = d_rank_list_to_str(disabled_ranks, &disabled_ranks_str); if (rc != 0) { DL_ERROR(rc, DF_UUID ": Failed to serialize the list of disabled ranks", DP_UUID(uuid)); - D_GOTO(out_dead_ranks, rc); - } else if (dead_ranks_str == NULL) { - D_STRNDUP(dead_ranks_str, "[]", strlen("[]")); + D_GOTO(error, rc); } - - D_DEBUG(DB_MGMT, DF_UUID ": %s ranks: %s, dead_ranks: %s\n", DP_UUID(uuid), - pool_info.pi_bits & DPI_ENGINES_ENABLED ? "ENABLED" : "DISABLED", range_list_str, - dead_ranks_str); + rc = d_rank_list_to_str(dead_ranks, &dead_ranks_str); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": Failed to serialize the list of dead ranks", DP_UUID(uuid)); + D_GOTO(error, rc); + } + if (disabled_ranks_str != NULL) + D_DEBUG(DB_MGMT, DF_UUID ": list of disabled ranks: %s\n", DP_UUID(uuid), + disabled_ranks_str); + if (dead_ranks_str != NULL) + D_DEBUG(DB_MGMT, DF_UUID ": list of dead ranks: %s\n", DP_UUID(uuid), + dead_ranks_str); /* Populate the response */ resp.query_mask = pool_info.pi_bits; @@ -1825,15 +1832,17 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) resp.svc_ldr = pool_info.pi_leader; resp.svc_reps = req->svc_ranks; resp.n_svc_reps = req->n_svc_ranks; - - resp.version = pool_info.pi_map_ver; - resp.enabled_ranks = (req->query_mask & DPI_ENGINES_ENABLED) ? range_list_str : ""; - resp.disabled_ranks = (req->query_mask & DPI_ENGINES_DISABLED) ? range_list_str : ""; - resp.dead_ranks = dead_ranks_str; + resp.version = pool_info.pi_map_ver; + if (enabled_ranks_str != NULL) + resp.enabled_ranks = enabled_ranks_str; + if (disabled_ranks_str != NULL) + resp.disabled_ranks = disabled_ranks_str; + if (dead_ranks_str != NULL) + resp.dead_ranks = dead_ranks_str; D_ALLOC_ARRAY(resp.tier_stats, DAOS_MEDIA_MAX); if (resp.tier_stats == NULL) - D_GOTO(out_dead_ranks, rc = -DER_NOMEM); + D_GOTO(error, rc = -DER_NOMEM); storage_usage_stats_from_pool_space(&scm, &pool_info.pi_space, DAOS_MEDIA_SCM); @@ -1848,13 +1857,7 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) pool_rebuild_status_from_info(&rebuild, &pool_info.pi_rebuild_st); resp.rebuild = &rebuild; -out_dead_ranks: - d_rank_list_free(dead_ranks); -out_ranks: - d_rank_list_free(ranks); -out_svc_ranks: - d_rank_list_free(svc_ranks); -out: +error: resp.status = rc; len = mgmt__pool_query_resp__get_packed_size(&resp); diff --git a/src/object/srv_obj.c b/src/object/srv_obj.c index da0de35d0e8..96666370d94 100644 --- a/src/object/srv_obj.c +++ b/src/object/srv_obj.c @@ -5379,7 +5379,7 @@ ds_obj_cpd_body_bulk(crt_rpc_t *rpc, struct obj_io_context *ioc, bool leader, } rc = obj_bulk_transfer(rpc, CRT_BULK_GET, true, bulks, NULL, NULL, DAOS_HDL_INVAL, sgls, - count, count, NULL); + count, count, NULL, check_conn(NULL, leader)); if (rc != 0) goto out;