diff --git a/src/container/cli.c b/src/container/cli.c index a0dc96fec00..18b6a75a590 100644 --- a/src/container/cli.c +++ b/src/container/cli.c @@ -190,6 +190,7 @@ dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, int rc = 0; uid_t uid = geteuid(); gid_t gid = getegid(); + uint32_t co_ver = 0; entries = (prop_in == NULL) ? 0 : prop_in->dpp_nr; @@ -224,6 +225,10 @@ dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, entries++; } + if (!daos_prop_has_entry(prop_in, DAOS_PROP_CO_OBJ_VERSION)) { + co_ver = DAOS_POOL_OBJ_VERSION_2; + entries++; + } if (!daos_prop_has_entry(prop_in, DAOS_PROP_CO_ROOTS)) { uint64_t rf; @@ -294,6 +299,11 @@ dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, roots = NULL; /* prop is responsible for it now */ idx++; } + if (co_ver != 0) { + final_prop->dpp_entries[idx].dpe_type = DAOS_PROP_CO_OBJ_VERSION; + final_prop->dpp_entries[idx].dpe_val = co_ver; + idx++; + } } *prop_out = final_prop; @@ -999,7 +1009,7 @@ dc_cont_open(tse_task_t *task) if (tpriv->cont == NULL) D_GOTO(err_pool, rc = -DER_NOMEM); uuid_generate(tpriv->cont->dc_cont_hdl); - tpriv->cont->dc_capas = args->flags; + tpriv->cont->dc_capas = args->flags | DAOS_COO_NEW_LAYOUT; } D_DEBUG(DB_MD, DF_UUID ":%s: opening: hdl=" DF_UUIDF " flags=%x\n", DP_UUID(pool->dp_pool), diff --git a/src/container/srv_container.c b/src/container/srv_container.c index 5dc44e2758d..7f15a275b5d 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -702,10 +702,12 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl, } break; case DAOS_PROP_CO_GLOBAL_VERSION: - case DAOS_PROP_CO_OBJ_VERSION: - D_ERROR("container global/obj %u version could be not set\n", - entry->dpe_type); + D_ERROR("container global %u version could be not set\n", entry->dpe_type); return -DER_INVAL; + case DAOS_PROP_CO_OBJ_VERSION: + /* this is a walkaround for 2.6 only */ + entry_def->dpe_val = entry->dpe_val; + break; default: D_ASSERTF(0, "bad dpt_type %d.\n", entry->dpe_type); break; @@ -760,8 +762,9 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl, /* inherit object version from pool*/ entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_OBJ_VERSION); - if (entry_def) + if (entry_def && entry_def->dpe_val == 0) entry_def->dpe_val = pool_hdl->sph_obj_ver; + /* for new container set HEALTHY status with current pm ver */ entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_STATUS); D_ASSERT(entry_def != NULL); @@ -2272,6 +2275,17 @@ cont_open(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *cont, cr D_GOTO(out, rc); D_ASSERT(prop != NULL); D_ASSERT(prop->dpp_nr <= CONT_PROP_NUM); + if (!(flags & DAOS_COO_NEW_LAYOUT)) { + struct daos_prop_entry *entry; + + entry = daos_prop_entry_get(prop, DAOS_PROP_CO_OBJ_VERSION); + if (entry && entry->dpe_val >= DAOS_POOL_OBJ_VERSION_2) { + D_ERROR(DF_CONT ": refusing attempt to open with new layout: " DF_X64 "\n", + DP_CONT(cont->c_svc->cs_pool_uuid, cont->c_uuid), entry->dpe_val); + rc = -DER_NOTSUPPORTED; + D_GOTO(out, rc); + } + } get_cont_prop_access_info(prop, &owner, &acl); @@ -4612,7 +4626,7 @@ upgrade_cont_cb(daos_handle_t ih, d_iov_t *key, d_iov_t *val, void *varg) } if (DAOS_FAIL_CHECK(DAOS_FORCE_OBJ_UPGRADE)) { - obj_ver = DS_POOL_OBJ_VERSION; + obj_ver = DAOS_POOL_OBJ_VERSION; d_iov_set(&value, &obj_ver, sizeof(obj_ver)); rc = rdb_tx_update(ap->tx, &cont->c_prop, &ds_cont_prop_cont_obj_version, &value); @@ -4707,9 +4721,9 @@ upgrade_cont_cb(daos_handle_t ih, d_iov_t *key, d_iov_t *val, void *varg) entry = daos_prop_entry_get(prop, DAOS_PROP_CO_OBJ_VERSION); D_ASSERT(entry != NULL); - entry->dpe_val = DS_POOL_OBJ_VERSION; + entry->dpe_val = DAOS_POOL_OBJ_VERSION; entry->dpe_flags &= ~DAOS_PROP_ENTRY_NOT_SET; - obj_ver = DS_POOL_OBJ_VERSION; + obj_ver = DAOS_POOL_OBJ_VERSION; d_iov_set(&value, &obj_ver, sizeof(obj_ver)); rc = rdb_tx_update(ap->tx, &cont->c_prop, &ds_cont_prop_cont_obj_version, &value); if (rc) { diff --git a/src/include/daos/pool.h b/src/include/daos/pool.h index e51758e16b1..2747b8c3432 100644 --- a/src/include/daos/pool.h +++ b/src/include/daos/pool.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -85,6 +86,16 @@ */ #define DAOS_POOL_GLOBAL_VERSION 3 +/** + * Each individual object layout format, like oid layout, dkey to group, + * dkey to EC group start. + */ +enum { + DAOS_POOL_OBJ_VERSION_1 = 1, + DAOS_POOL_OBJ_VERSION_2 = 2, + DAOS_POOL_OBJ_VERSION = DAOS_POOL_OBJ_VERSION_2, +}; + int dc_pool_init(void); void dc_pool_fini(void); diff --git a/src/include/daos_cont.h b/src/include/daos_cont.h index 632eafe8130..6e6e5b98adc 100644 --- a/src/include/daos_cont.h +++ b/src/include/daos_cont.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2020-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -73,8 +74,15 @@ extern "C" { */ #define DAOS_COO_EVICT_ALL (1U << 6) +/** + * When clients attempt to open a container using the new hash layout, + * the server will reject the request if this flag is not set, + * even if the container is already in the new layout format. + */ +#define DAOS_COO_NEW_LAYOUT (1U << 7) + /** Number of bits in the container open mode flag, DAOS_COO_ bits */ -#define DAOS_COO_NBITS (7) +#define DAOS_COO_NBITS (8) /** Mask for all of the bits in the container open mode flag, DAOS_COO_ bits */ #define DAOS_COO_MASK ((1U << DAOS_COO_NBITS) - 1) diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index 1b5fcc902c9..7cf244e3cb0 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -30,12 +30,6 @@ /* Pool service (opaque) */ struct ds_pool_svc; -/** - * Each individual object layout format, like oid layout, dkey to group, - * dkey to EC group start. - */ -#define DS_POOL_OBJ_VERSION 1 - /* age of an entry in svc_ops KVS before it may be evicted */ #define DEFAULT_SVC_OPS_ENTRY_AGE_SEC_MAX 300ULL diff --git a/src/pipeline/cli_pipeline.c b/src/pipeline/cli_pipeline.c index b30d8366b79..9d76456f331 100644 --- a/src/pipeline/cli_pipeline.c +++ b/src/pipeline/cli_pipeline.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2021-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -544,7 +545,7 @@ shard_task_sched(tse_task_t *task, void *arg) static int pipeline_create_layout(daos_handle_t coh, struct dc_pool *pool, struct daos_obj_md *obj_md, - struct pl_obj_layout **layout) + struct pl_obj_layout **layout, uint16_t layout_gl_version) { int rc = 0; struct pl_map *map; @@ -555,7 +556,7 @@ pipeline_create_layout(daos_handle_t coh, struct dc_pool *pool, struct daos_obj_ D_GOTO(out, rc = -DER_INVAL); } - rc = pl_obj_place(map, 0, obj_md, 0, NULL, layout); + rc = pl_obj_place(map, layout_gl_version, obj_md, 0, NULL, layout); pl_map_decref(map); if (rc != 0) { D_DEBUG(DB_PL, "Failed to generate object layout\n"); @@ -609,6 +610,7 @@ dc_pipeline_run(tse_task_t *api_task) int total_replicas; int shard; struct pipeline_comp_cb_args comp_cb_args; + uint16_t layout_gl_ver; coh = dc_obj_hdl2cont_hdl(api_args->oh); rc = dc_obj_hdl2obj_md(api_args->oh, &obj_md); @@ -629,7 +631,8 @@ dc_pipeline_run(tse_task_t *api_task) /** get layout */ - rc = pipeline_create_layout(coh, pool, &obj_md, &layout); + layout_gl_ver = dc_obj_hdl2layout_ver(api_args->oh); + rc = pipeline_create_layout(coh, pool, &obj_md, &layout, layout_gl_ver); dc_pool_put(pool); if (rc != 0) D_GOTO(out, rc); @@ -687,7 +690,7 @@ dc_pipeline_run(tse_task_t *api_task) /** object id */ oid.id_pub = obj_md.omd_id; oid.id_shard = shard; - oid.id_layout_ver = dc_obj_hdl2layout_ver(api_args->oh); + oid.id_layout_ver = layout_gl_ver; oid.id_padding = 0; /** queue shard task */ diff --git a/src/placement/jump_map.c b/src/placement/jump_map.c index 179a3788ea5..70057d8e7d8 100644 --- a/src/placement/jump_map.c +++ b/src/placement/jump_map.c @@ -1,6 +1,7 @@ /** * * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -46,7 +47,7 @@ jm_obj_placement_fini(struct jm_obj_placement *jmop) #define LOCAL_PD_ARRAY_SIZE (4) static int jm_obj_pd_init(struct pl_jump_map *jmap, struct daos_obj_md *md, struct pool_domain *root, - struct jm_obj_placement *jmop) + struct jm_obj_placement *jmop, uint32_t layout_ver) { struct pool_domain *pds, *pd; uint8_t *pd_used = NULL; @@ -114,9 +115,9 @@ jm_obj_pd_init(struct pl_jump_map *jmap, struct daos_obj_md *md, struct pool_dom } oid = md->omd_id; - key = oid.hi ^ oid.lo; + key = jm_oid_hash(layout_ver, oid); for (i = 0; i < jmop->jmop_pd_nr; i++) { - key = crc(key, i); + key = (layout_ver <= 1) ? crc(key, i) : jm_crc(key, i, 0xcafebabe); selected_pd = d_hash_jump(key, jmap->jmp_pd_nr); do { selected_pd = selected_pd % jmap->jmp_pd_nr; @@ -242,8 +243,8 @@ layout_find_diff(struct pl_jump_map *jmap, struct pl_obj_layout *original, */ static int jm_obj_placement_init(struct pl_jump_map *jmap, struct daos_obj_md *md, - struct daos_obj_shard_md *shard_md, - struct jm_obj_placement *jmop) + struct daos_obj_shard_md *shard_md, struct jm_obj_placement *jmop, + uint32_t layout_ver) { struct daos_oclass_attr *oc_attr; struct pool_domain *root; @@ -305,7 +306,7 @@ jm_obj_placement_init(struct pl_jump_map *jmap, struct daos_obj_md *md, D_ASSERT(jmop->jmop_grp_nr > 0); D_ASSERT(jmop->jmop_grp_size > 0); - rc = jm_obj_pd_init(jmap, md, root, jmop); + rc = jm_obj_pd_init(jmap, md, root, jmop, layout_ver); if (rc == 0) D_DEBUG(DB_PL, "obj="DF_OID"/ grp_size=%u grp_nr=%d, pd_nr=%u pd_grp_size=%u\n", DP_OID(oid), jmop->jmop_grp_size, jmop->jmop_grp_nr, @@ -404,7 +405,7 @@ obj_remap_shards(struct pl_jump_map *jmap, uint32_t layout_ver, struct daos_obj_ current = remap_list->next; spare_tgt = NULL; oid = md->omd_id; - key = oid.hi ^ oid.lo; + key = jm_oid_hash(layout_ver, oid); spares_left = count_available_spares(jmap, layout, failed_in_layout); rc = pool_map_find_domain(jmap->jmp_map.pl_poolmap, PO_COMP_TP_ROOT, @@ -435,13 +436,16 @@ obj_remap_shards(struct pl_jump_map *jmap, uint32_t layout_ver, struct daos_obj_ dgu = f_shard->fs_data; D_ASSERT(dgu != NULL); - rebuild_key = crc(key, f_shard->fs_shard_idx); + if (layout_ver <= 1) + rebuild_key = crc(key, crc(key, f_shard->fs_shard_idx)); + else /* hash OID differently so we don't land to the same target */ + rebuild_key = jm_crc(oid.lo, oid.hi, 0xDead2Bad); + curr_pd = jm_obj_shard_pd(jmop, shard_id); - get_target(root, curr_pd, layout_ver, &spare_tgt, &spare_dom, - crc(key, rebuild_key), dom_used, dom_full, - dgu->dgu_used, dgu->dgu_real, tgts_used, - shard_id, allow_version, gen_mode, fdom_lvl, - jmop->jmop_grp_size, &spares_left, &spare_avail); + get_target(root, curr_pd, layout_ver, &spare_tgt, &spare_dom, rebuild_key, + dom_used, dom_full, dgu->dgu_used, dgu->dgu_real, tgts_used, + shard_id, allow_version, gen_mode, fdom_lvl, jmop->jmop_grp_size, + &spares_left, &spare_avail); if (layout_ver > 0) { /* * After 2.4 (layout_ver > 0), it will always assign each shard @@ -644,7 +648,8 @@ get_object_layout(struct pl_jump_map *jmap, uint32_t layout_ver, struct pl_obj_l D_GOTO(out, rc = -DER_NOMEM); oid = md->omd_id; - key = oid.hi ^ oid.lo; + key = jm_oid_hash(layout_ver, oid); + if (daos_obj_is_srank(oid)) spec_oid = true; @@ -1012,7 +1017,7 @@ jump_map_obj_place(struct pl_map *map, uint32_t layout_version, struct daos_obj_ D_DEBUG(DB_PL, "Determining location for object: "DF_OID", ver: %d, pda %u\n", DP_OID(oid), md->omd_ver, md->omd_pda); - rc = jm_obj_placement_init(jmap, md, shard_md, &jmop); + rc = jm_obj_placement_init(jmap, md, shard_md, &jmop, layout_version); if (rc) { D_ERROR("jm_obj_placement_init failed, rc "DF_RC"\n", DP_RC(rc)); return rc; @@ -1115,7 +1120,7 @@ jump_map_obj_find_diff(struct pl_map *map, uint32_t layout_ver, struct daos_obj_ } jmap = pl_map2jmap(map); - rc = jm_obj_placement_init(jmap, md, shard_md, &jop); + rc = jm_obj_placement_init(jmap, md, shard_md, &jop, layout_ver); if (rc) { D_ERROR("jm_obj_placement_init failed, rc %d.\n", rc); return rc; diff --git a/src/placement/jump_map.h b/src/placement/jump_map.h index 1fd39d3ce58..330872d842f 100644 --- a/src/placement/jump_map.h +++ b/src/placement/jump_map.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -61,4 +62,22 @@ get_target(struct pool_domain *root, struct pool_domain *curr_pd, uint32_t layou uint8_t *dom_full, uint8_t *dom_cur_grp_used, uint8_t *dom_cur_grp_real, uint8_t *tgts_used, int shard_num, uint32_t allow_version, enum layout_gen_mode gen_mode, pool_comp_type_t fdom_lvl, uint32_t grp_size, uint32_t *spare_left, bool *spare_avail); + +static inline uint64_t +jm_crc(uint64_t val0, uint64_t val1, uint64_t val2) +{ + uint64_t data[3]; + + data[0] = val0; + data[1] = val1; + data[2] = val2; + return crc64_ecma_refl(0xbabecafe, (uint8_t *)data, 24); +} + +static inline uint64_t +jm_oid_hash(uint32_t layout_ver, daos_obj_id_t oid) +{ + return layout_ver <= 1 ? (oid.hi ^ oid.lo) : jm_crc(oid.hi, oid.lo, 0xbabeface); +} + #endif /* __JUMP_MAP_H__ */ diff --git a/src/placement/jump_map_versions.c b/src/placement/jump_map_versions.c index b943949ad6a..f817ed96b04 100644 --- a/src/placement/jump_map_versions.c +++ b/src/placement/jump_map_versions.c @@ -1,6 +1,7 @@ /** * * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -414,6 +415,216 @@ __get_target_v1(struct pool_domain *root_pos, struct pool_domain *curr_pd, } while (!found_target); } +/** + * This function is identical to __get_target_v1, except how they call CRC functions, + * the V1 function may cause a large number of hash collisions in certain cases. + * + * Search for jm_crc() and crc() to see how they differ. + */ +static void +__get_target_v2(struct pool_domain *root_pos, struct pool_domain *curr_pd, + struct pool_target **target, struct pool_domain **dom, uint64_t obj_key, + uint8_t *dom_used, uint8_t *dom_full, uint8_t *dom_cur_grp_used, uint8_t *tgts_used, + int shard_num, uint32_t allow_version, enum layout_gen_mode gen_mode, + pool_comp_type_t fdom_lvl, uint32_t grp_size, bool *pd_ignored) +{ + int range_set; + uint8_t found_target = 0; + struct pool_domain *curr_dom; + struct pool_domain *dom_stack[MAX_STACK] = {0}; + int top = -1; + + curr_dom = curr_pd; + do { + uint32_t avail_doms; + uint64_t key; + + /* Retrieve number of nodes in this domain */ + avail_doms = get_num_domains(curr_dom, allow_version, gen_mode, fdom_lvl); + + /* If choosing target (lowest fault domain level) */ + if (curr_dom->do_comp.co_type == fdom_lvl) { + uint32_t fail_num = 0; + uint32_t tgt_idx; + uint32_t start_tgt; + uint32_t end_tgt; + uint32_t selected_tgt; + + start_tgt = curr_dom->do_targets - root_pos->do_targets; + end_tgt = start_tgt + (curr_dom->do_target_nr - 1); + + range_set = tgt_isset_range(root_pos->do_targets, tgts_used, start_tgt, + end_tgt, allow_version, gen_mode); + if (range_set) { + /* Used up all targets in this domain */ + D_ASSERT(top != -1); + curr_dom = dom_stack[top--]; /* try parent */ + continue; + } + + /* + * Must crc key because jump consistent hash + * requires an even distribution or it will + * not work + */ + key = jm_crc(obj_key, shard_num, fail_num++); + /* Get target for shard */ + selected_tgt = d_hash_jump(key, avail_doms); + do { + selected_tgt = selected_tgt % avail_doms; + /* Retrieve actual target using index */ + *target = + _get_target(curr_dom, selected_tgt, allow_version, gen_mode); + /* Get target id to check if target used */ + tgt_idx = *target - root_pos->do_targets; + selected_tgt++; + } while (isset(tgts_used, tgt_idx)); + + setbit(tgts_used, tgt_idx); + D_DEBUG(DB_PL, "selected tgt %d\n", tgt_idx); + D_ASSERTF(isclr(dom_full, (uint32_t)(curr_dom - root_pos)), + "selected_tgt %u\n", (uint32_t)(curr_dom - root_pos)); + range_set = tgt_isset_range(root_pos->do_targets, tgts_used, start_tgt, + end_tgt, allow_version, gen_mode); + if (range_set) { + /* Used up all targets in this domain */ + setbit(dom_full, curr_dom - root_pos); + D_DEBUG(DB_PL, "dom %d used up\n", (int)(curr_dom - root_pos)); + /* Check and set if all of its parent are full */ + while (top != -1) { + if (is_dom_full(dom_stack[top], root_pos, dom_full, + allow_version, gen_mode)) { + uint32_t off = dom_stack[top] - root_pos; + + D_DEBUG(DB_PL, "dom %u used up\n", off); + setbit(dom_full, off); + } + --top; + } + } + *dom = curr_dom; + /* Found target (which may be available or not) */ + found_target = 1; + } else { + uint32_t selected_dom; + uint32_t fail_num = 0; + uint32_t start_dom; + uint32_t end_dom; + + start_dom = (curr_dom->do_children) - root_pos; + end_dom = start_dom + (curr_dom->do_child_nr - 1); + + D_DEBUG(DB_TRACE, "start_dom %u end_dom %u\n", start_dom, end_dom); + /* Check if all targets under the domain range has been + * used up (occupied), go back to its parent if it does. + */ + range_set = + is_dom_full(curr_dom, root_pos, dom_full, allow_version, gen_mode); + if (range_set) { + if (top == -1) { + if (curr_pd != root_pos) { + /* all domains within the PD are full, ignore the + * PD restrict. + */ + D_DEBUG(DB_PL, + "PD[%d] all doms are full, weak the " + "PD restrict\n", + (int)(curr_dom - root_pos)); + curr_pd = root_pos; + curr_dom = curr_pd; + *pd_ignored = true; + continue; + } + /* shard nr > target nr, no extra target for the shard */ + *target = NULL; + return; + } + setbit(dom_full, curr_dom - root_pos); + D_DEBUG(DB_PL, "used up dom %d\n", (int)(curr_dom - root_pos)); + setbit(dom_cur_grp_used, curr_dom - root_pos); + curr_dom = dom_stack[top--]; + continue; + } + + /* Check if all children has been used for the current group or full */ + range_set = dom_isset_2ranges(root_pos, dom_full, dom_cur_grp_used, + start_dom, end_dom, allow_version, gen_mode); + if (range_set) { + if (top == -1) { + if (curr_pd != root_pos && grp_size > 1) { + /* All domains within the PD are full, ignore the + * PD restrict. For non-replica (grp_size 1) case, + * keep the PD restrict until all targets under the + * domain range has been used up (see above check + * for dom_full bitmap). + */ + D_DEBUG(DB_PL, + "PD[%d] all doms are full, weak the " + "PD restrict\n", + (int)(curr_dom - root_pos)); + curr_pd = root_pos; + curr_dom = curr_pd; + *pd_ignored = true; + continue; + } + *target = NULL; + return; + } + setbit(dom_cur_grp_used, curr_dom - root_pos); + D_DEBUG(DB_PL, "set grp_used %d\n", (int)(curr_dom - root_pos)); + curr_dom = dom_stack[top--]; + continue; + } + + /* Check if all children has been used for the current group or the + * object. + */ + range_set = dom_isset_2ranges(root_pos, dom_used, dom_cur_grp_used, + start_dom, end_dom, allow_version, gen_mode); + if (range_set) { + int idx; + + for (idx = start_dom; idx <= end_dom; ++idx) + if (!isset(dom_full, idx)) { + clrbit(dom_used, idx); + D_DEBUG(DB_PL, "clrbit dom_used %d\n", idx); + } + if (top == -1) + curr_dom = curr_pd; + else + curr_dom = dom_stack[top--]; + continue; + } + + /* Keep choosing the new domain until the one has not been used. */ + do { + struct pool_domain *_dom; + + key = jm_crc(obj_key, shard_num, fail_num++); + selected_dom = d_hash_jump(key, avail_doms); + _dom = _get_dom(curr_dom, selected_dom, allow_version, gen_mode); + if (_dom != NULL) + selected_dom = _dom - curr_dom->do_children; + } while (isset(dom_used, start_dom + selected_dom) || + isset(dom_cur_grp_used, start_dom + selected_dom)); + + D_ASSERTF(isclr(dom_full, start_dom + selected_dom), "selected_dom %u\n", + selected_dom); + /* Mark this domain as used */ + if (curr_dom == curr_pd && curr_pd != root_pos) + setbit(dom_used, (int)(curr_dom - root_pos)); + D_DEBUG(DB_PL, "selected dom %d\n", start_dom + selected_dom); + setbit(dom_used, start_dom + selected_dom); + setbit(dom_cur_grp_used, start_dom + selected_dom); + D_ASSERT(top < MAX_STACK - 1); + top++; + dom_stack[top] = curr_dom; + curr_dom = &curr_dom->do_children[selected_dom]; + obj_key = jm_crc(obj_key, curr_dom->do_comp.co_id, 0xbabecafe); + } + } while (!found_target); +} + /* Only reset all used dom/tgts bits if all domain has been used. */ static void dom_reset_bit(struct pool_domain *dom, uint8_t *dom_bits, struct pool_domain *root, @@ -604,11 +815,11 @@ reset_dom_cur_grp_v1(struct pool_domain *root, struct pool_domain *curr_pd, } static void -get_target_v1(struct pool_domain *root, struct pool_domain *curr_pd, struct pool_target **target, - struct pool_domain **dom, uint64_t key, uint8_t *dom_used, uint8_t *dom_full, - uint8_t *dom_cur_grp_used, uint8_t *dom_cur_grp_real, uint8_t *tgts_used, - int shard_num, uint32_t allow_version, enum layout_gen_mode gen_mode, - pool_comp_type_t fdom_lvl, uint32_t grp_size) +get_target_new(struct pool_domain *root, struct pool_domain *curr_pd, uint32_t layout_ver, + struct pool_target **target, struct pool_domain **dom, uint64_t key, + uint8_t *dom_used, uint8_t *dom_full, uint8_t *dom_cur_grp_used, + uint8_t *dom_cur_grp_real, uint8_t *tgts_used, int shard_num, uint32_t allow_version, + enum layout_gen_mode gen_mode, pool_comp_type_t fdom_lvl, uint32_t grp_size) { struct pool_target *found = NULL; bool pd_ignored; @@ -622,9 +833,17 @@ get_target_v1(struct pool_domain *root, struct pool_domain *curr_pd, struct pool */ while (found == NULL) { pd_ignored = false; - __get_target_v1(root, curr_pd, &found, dom, key, dom_used, dom_full, - dom_cur_grp_used, tgts_used, shard_num, allow_version, gen_mode, - fdom_lvl, grp_size, &pd_ignored); + if (layout_ver == 1) { + __get_target_v1(root, curr_pd, &found, dom, key, dom_used, dom_full, + dom_cur_grp_used, tgts_used, shard_num, allow_version, + gen_mode, fdom_lvl, grp_size, &pd_ignored); + } else { /* Fix CRC calls */ + D_ASSERT(layout_ver == 2); + __get_target_v2(root, curr_pd, &found, dom, key, dom_used, dom_full, + dom_cur_grp_used, tgts_used, shard_num, allow_version, + gen_mode, fdom_lvl, grp_size, &pd_ignored); + } + if (found == NULL) { if (pd_ignored) reset_dom_cur_grp_v1(root, root, dom_cur_grp_used, dom_cur_grp_real, @@ -901,14 +1120,15 @@ get_target(struct pool_domain *root, struct pool_domain *curr_pd, uint32_t layou } break; case 1: + case 2: /** * New layout(>= 1) will always try to assign target to the shard due to the high * overhead of the EC degraded fetch, so it might put multiple shards in the * same target, i.e. it will never check the spare target here. */ - get_target_v1(root, curr_pd, target, dom, key, dom_used, dom_full, dom_cur_grp_used, - dom_cur_grp_real, tgts_used, shard_num, allow_version, gen_mode, - fdom_lvl, grp_size); + get_target_new(root, curr_pd, layout_ver, target, dom, key, dom_used, dom_full, + dom_cur_grp_used, dom_cur_grp_real, tgts_used, shard_num, + allow_version, gen_mode, fdom_lvl, grp_size); if (spare_avail) *spare_avail = true; break; diff --git a/src/placement/tests/jump_map_place_obj.c b/src/placement/tests/jump_map_place_obj.c index 8e257aad82f..b2e5e1d21e7 100644 --- a/src/placement/tests/jump_map_place_obj.c +++ b/src/placement/tests/jump_map_place_obj.c @@ -1196,10 +1196,10 @@ down_back_to_up_in_same_order(void **state) * healthy targets because of the retry/collision mechanism of the jump * map algorithm. * Due to layout colocation, if the oid has been changed, then it could - * be 2 or even 3 as well, with current oid setting, this is 1. + * be 2 or even 3 as well, with current oid setting, this is 2. */ - assert_int_equal(1, ctx.reint.out_nr); - jtc_assert_rebuild_reint_new(ctx, 1, 0, 1, 1); + assert_int_equal(2, ctx.reint.out_nr); + jtc_assert_rebuild_reint_new(ctx, 2, 0, 2, 2); /* Take second downed target up */ jtc_set_status_on_target(&ctx, UP, orig_shard_targets[1]); @@ -1674,14 +1674,14 @@ placement_handles_multiple_states_with_addition(void **state) rebuilding = jtc_get_layout_rebuild_count(&ctx); /* 1 each for down, up, new ... maybe? */ - assert_true(rebuilding == 2 || rebuilding == 3); + assert_true(rebuilding == 2 || rebuilding == 3 || rebuilding == 4); /* Both DOWN and UP target will be remapped during remap */ - assert_int_equal(ctx.rebuild.out_nr, 2); + assert_int_equal(ctx.rebuild.out_nr, 3); /* Adding new targets might make original shard to be remapped * to new location. */ - assert_true(ctx.reint.out_nr == 1 || ctx.reint.out_nr == 2); + assert_true(ctx.reint.out_nr == 1 || ctx.reint.out_nr == 2 || ctx.reint.out_nr == 3); /* JCH might cause multiple shards remap to the new target */ assert_true(ctx.new.out_nr >= 1); diff --git a/src/placement/tests/place_obj_common.h b/src/placement/tests/place_obj_common.h index 537c02a8de7..537fbe963a2 100644 --- a/src/placement/tests/place_obj_common.h +++ b/src/placement/tests/place_obj_common.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -28,7 +29,7 @@ extern bool g_verbose; #define is_true assert_true #define is_false assert_false -#define PLT_LAYOUT_VERSION 1 +#define PLT_LAYOUT_VERSION 2 extern bool fail_domain_node; void print_layout(struct pl_obj_layout *layout); diff --git a/src/pool/srv_layout.c b/src/pool/srv_layout.c index dbddb92ef6f..1456cbeaf46 100644 --- a/src/pool/srv_layout.c +++ b/src/pool/srv_layout.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2017-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -133,7 +134,7 @@ struct daos_prop_entry pool_prop_entries_default[DAOS_PROP_PO_NUM] = { }, { .dpe_type = DAOS_PROP_PO_OBJ_VERSION, - .dpe_val = DS_POOL_OBJ_VERSION, + .dpe_val = DAOS_POOL_OBJ_VERSION, }, { .dpe_type = DAOS_PROP_PO_PERF_DOMAIN, diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index d207f4876c9..339a236e281 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -665,7 +665,7 @@ pool_prop_write(struct rdb_tx *tx, const rdb_path_t *kvs, daos_prop_t *prop) rc = rdb_tx_update(tx, kvs, &ds_pool_prop_svc_redun_fac, &value); break; case DAOS_PROP_PO_OBJ_VERSION: - if (entry->dpe_val > DS_POOL_OBJ_VERSION) { + if (entry->dpe_val > DAOS_POOL_OBJ_VERSION) { rc = -DER_INVAL; break; } @@ -6030,7 +6030,7 @@ __ds_pool_mark_upgrade_completed(uuid_t pool_uuid, struct pool_svc *svc, int rc) obj_version = (uint32_t)fail_val; } else { - obj_version = DS_POOL_OBJ_VERSION; + obj_version = DAOS_POOL_OBJ_VERSION; } d_iov_set(&value, &obj_version, sizeof(obj_version)); @@ -6091,10 +6091,9 @@ pool_check_upgrade_object_layout(struct rdb_tx *tx, struct pool_svc *svc, else if (rc == -DER_NONEXIST) current_layout_ver = 0; - if (current_layout_ver < DS_POOL_OBJ_VERSION) { - rc = ds_rebuild_schedule(svc->ps_pool, svc->ps_pool->sp_map_version, - upgrade_eph, DS_POOL_OBJ_VERSION, NULL, - RB_OP_UPGRADE, 0); + if (current_layout_ver < DAOS_POOL_OBJ_VERSION) { + rc = ds_rebuild_schedule(svc->ps_pool, svc->ps_pool->sp_map_version, upgrade_eph, + DAOS_POOL_OBJ_VERSION, NULL, RB_OP_UPGRADE, 0); if (rc == 0) *scheduled_layout_upgrade = true; }