diff --git a/src/common/tests_dmg_helpers.c b/src/common/tests_dmg_helpers.c index dcf1b274116..08825366133 100644 --- a/src/common/tests_dmg_helpers.c +++ b/src/common/tests_dmg_helpers.c @@ -1206,6 +1206,80 @@ dmg_pool_list(const char *dmg_config_file, const char *group, return rc; } +int +dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force) +{ + char uuid_str[DAOS_UUID_STR_SIZE]; + int argcount = 0; + char **args = NULL; + struct json_object *dmg_out = NULL; + int rc = 0; + + uuid_unparse_lower(uuid, uuid_str); + args = cmd_push_arg(args, &argcount, "%s ", uuid_str); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + + if (grp != NULL) { + args = cmd_push_arg(args, &argcount, "--sys=%s ", grp); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + } + + if (force) { + args = cmd_push_arg(args, &argcount, "--force"); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + } + + rc = daos_dmg_json_pipe("pool rebuild stop", dmg_config_file, args, argcount, &dmg_out); + if (rc != 0) { + D_ERROR("dmg pool rebuild stop failed\n"); + goto out_json; + } + +out_json: + if (dmg_out != NULL) + json_object_put(dmg_out); + cmd_free_args(args, argcount); +out: + return rc; +} + +int +dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp) +{ + char uuid_str[DAOS_UUID_STR_SIZE]; + int argcount = 0; + char **args = NULL; + struct json_object *dmg_out = NULL; + int rc = 0; + + uuid_unparse_lower(uuid, uuid_str); + args = cmd_push_arg(args, &argcount, "%s ", uuid_str); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + + if (grp != NULL) { + args = cmd_push_arg(args, &argcount, "--sys=%s ", grp); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + } + + rc = daos_dmg_json_pipe("pool rebuild start", dmg_config_file, args, argcount, &dmg_out); + if (rc != 0) { + D_ERROR("dmg pool rebuild start failed\n"); + goto out_json; + } + +out_json: + if (dmg_out != NULL) + json_object_put(dmg_out); + cmd_free_args(args, argcount); +out: + return rc; +} + static int parse_device_info(struct json_object *smd_dev, device_list *devices, char *host, int dev_length, int *disks) diff --git a/src/include/daos/common.h b/src/include/daos/common.h index ba8dfc04667..2225a417c47 100644 --- a/src/include/daos/common.h +++ b/src/include/daos/common.h @@ -789,9 +789,6 @@ enum { #define DAOS_RDB_SKIP_APPENDENTRIES_FAIL (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1a) #define DAOS_FORCE_REFRESH_POOL_MAP (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1b) -#define DAOS_REBUILD_ADMIN_STOP (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1c) -#define DAOS_REBUILD_ADMIN_START (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1d) - #define DAOS_FORCE_CAPA_FETCH (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1e) #define DAOS_FORCE_PROP_VERIFY (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1f) diff --git a/src/include/daos/tests_lib.h b/src/include/daos/tests_lib.h index 8c9ed5bf1f9..72438598297 100644 --- a/src/include/daos/tests_lib.h +++ b/src/include/daos/tests_lib.h @@ -484,6 +484,25 @@ dmg_pool_set_prop(const char *dmg_config_file, int dmg_pool_get_prop(const char *dmg_config_file, const char *label, const uuid_t uuid, const char *name, char **value); +/** + * Interactively stop a pool's currently-running rebuild. + * \param dmg_config_file [IN] DMG config file. + * \param uuid [IN] UUID of the pool. + * \param grp [IN] Process set name of the DAOS servers managing the pool. + * \param force [IN] forcibly stop a rebuild that is failing. + */ +int +dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force); + +/** + * Interactively start/resume a pool's rebuilding. + * \param dmg_config_file [IN] DMG config file. + * \param uuid [IN] UUID of the pool. + * \param grp [IN] Process set name of the DAOS servers managing the pool. + */ +int + dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp); + /** * List all disks in the specified DAOS system. * diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c index c961cff23cf..cf3e34a6834 100644 --- a/src/rebuild/srv.c +++ b/src/rebuild/srv.c @@ -874,9 +874,6 @@ enum { RB_BCAST_QUERY, }; -/* for interactive mode rebuild fault-injection testing */ -static uuid_t test_stop_start_puuid; - /* administrator (via dmg command) asks to stop the currently-running rebuild for pool */ int ds_rebuild_admin_stop(struct ds_pool *pool, uint32_t force) @@ -921,9 +918,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, unsigned int total; struct sched_req_attr attr = {0}; d_rank_t myrank; - unsigned int nsleeps = 0; - bool fi_admin_stop_fi = false; - bool fi_admin_stop = false; int rc; rc = crt_group_size(pool->sp_group, &total); @@ -948,18 +942,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, bool rebuild_abort = false; int i; - if (!fi_admin_stop_fi) - fi_admin_stop_fi = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_STOP); - fi_admin_stop = (fi_admin_stop_fi && (op == RB_OP_REBUILD)); - - if (fi_admin_stop && (nsleeps > 0)) { - uuid_copy(test_stop_start_puuid, rgt->rgt_pool_uuid); - - /* What the real RPC handler will call. We will exit below due to rgt_abort - */ - ds_pool_rebuild_stop(test_stop_start_puuid, 0 /* force */, NULL /* hint */); - } - ABT_rwlock_rdlock(pool->sp_lock); rc = map_ranks_init(pool->sp_map, PO_COMP_ST_UP | PO_COMP_ST_DOWN | @@ -1055,8 +1037,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, sleep: update_and_warn_for_slow_engines(rgt); sched_req_sleep(rgt->rgt_ult, RBLD_CHECK_INTV); - if (fi_admin_stop) - nsleeps++; } sched_req_put(rgt->rgt_ult); @@ -2064,25 +2044,13 @@ rebuild_ults(void *arg) while (DAOS_FAIL_CHECK(DAOS_REBUILD_HANG)) ABT_thread_yield(); - /* Stay alive for fault-injection possibility (need a ULT for DAOS_REBUILD_ADMIN_START) */ - while (true) { - bool fi_admin_start = false; - + while (!d_list_empty(&rebuild_gst.rg_queue_list) || + !d_list_empty(&rebuild_gst.rg_running_list)) { if (rebuild_gst.rg_abort) { D_INFO("abort rebuilds\n"); break; } - fi_admin_start = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_START); - if (fi_admin_start) { - if (!uuid_is_null(test_stop_start_puuid)) { - D_INFO("fi_admin_start=%d, apply to pool " DF_UUID "\n", - fi_admin_start, DP_UUID(test_stop_start_puuid)); - ds_pool_rebuild_start(test_stop_start_puuid, NULL /* hint */); - uuid_clear(test_stop_start_puuid); - } - } - if (d_list_empty(&rebuild_gst.rg_queue_list) || rebuild_gst.rg_inflight >= REBUILD_MAX_INFLIGHT) { D_INFO("in-flight rebuild %u - going to sleep for 5 seconds\n", @@ -3027,8 +2995,6 @@ init(void) { int rc; - uuid_clear(test_stop_start_puuid); - D_INIT_LIST_HEAD(&rebuild_gst.rg_tgt_tracker_list); D_INIT_LIST_HEAD(&rebuild_gst.rg_global_tracker_list); D_INIT_LIST_HEAD(&rebuild_gst.rg_completed_list); diff --git a/src/tests/suite/daos_rebuild_common.c b/src/tests/suite/daos_rebuild_common.c index f65f0ad4f41..8f1bc7eb944 100644 --- a/src/tests/suite/daos_rebuild_common.c +++ b/src/tests/suite/daos_rebuild_common.c @@ -62,6 +62,8 @@ rebuild_exclude_tgt(test_arg_t **args, int arg_cnt, d_rank_t rank, for (i = 0; i < arg_cnt; i++) { rc = dmg_pool_exclude(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, rank, tgt_idx); + print_message("dmg pool exclude rank %u tgt_idx=%d " DF_UUID ", rc=%d\n", rank, + tgt_idx, DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } } @@ -83,6 +85,9 @@ rebuild_reint_tgt(test_arg_t **args, int args_cnt, d_rank_t rank, if (!args[i]->pool.destroyed) { rc = dmg_pool_reintegrate(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, rank, tgt_idx); + print_message("dmg pool reintegrate rank %u tgt_idx=%d " DF_UUID + ", rc=%d\n", + rank, tgt_idx, DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } sleep(2); @@ -100,6 +105,8 @@ rebuild_extend_tgt(test_arg_t **args, int args_cnt, d_rank_t rank, if (!args[i]->pool.destroyed) { rc = dmg_pool_extend(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, &rank, 1); + print_message("dmg pool extend rank %u " DF_UUID ", rc=%d\n", rank, + DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } sleep(2); @@ -117,6 +124,8 @@ rebuild_drain_tgt(test_arg_t **args, int args_cnt, d_rank_t rank, if (!args[i]->pool.destroyed) { rc = dmg_pool_drain(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, rank, tgt_idx); + print_message("dmg pool drain rank %u tgt_idx=%d " DF_UUID ", rc=%d\n", + rank, tgt_idx, DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } sleep(2); @@ -162,16 +171,24 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, par_barrier(PAR_COMM_WORLD); for (i = 0; i < args_cnt; i++) - if (args[i]->rebuild_cb) + if (args[i]->rebuild_cb) { + print_message("call rebuild_cb for exclude rebuilding pool " DF_UUID + "\n", + DP_UUID(args[i]->pool.pool_uuid)); args[i]->rebuild_cb(args[i]); + } if (args[0]->myrank == 0 && !args[0]->no_rebuild) test_rebuild_wait(args, args_cnt); par_barrier(PAR_COMM_WORLD); for (i = 0; i < args_cnt; i++) { - if (args[i]->rebuild_post_cb) + if (args[i]->rebuild_post_cb) { + print_message( + "call rebuild_post_cb for exclude rebuilt pool " DF_UUID "\n", + DP_UUID(args[i]->pool.pool_uuid)); args[i]->rebuild_post_cb(args[i]); + } } return; @@ -179,6 +196,7 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, for (i = 0; i < rank_nr; i++) { int j; + const char *op_type_str = NULL; /* No concurrent drain/extend/reintegration are allowed, so * it has to reintegrate/extend one by one. @@ -187,15 +205,18 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, if (args[0]->myrank == 0) { switch (op_type) { case RB_OP_TYPE_REINT: + op_type_str = "reintegrate"; rebuild_reint_tgt(args, args_cnt, ranks[i], tgts ? tgts[i] : -1, kill); break; case RB_OP_TYPE_ADD: + op_type_str = "extend"; rebuild_extend_tgt(args, args_cnt, ranks[i], tgts ? tgts[i] : -1, args[i]->pool.pool_size); break; case RB_OP_TYPE_DRAIN: + op_type_str = "drain"; rebuild_drain_tgt(args, args_cnt, ranks[i], tgts ? tgts[i] : -1); break; @@ -208,20 +229,30 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, D_ASSERT(op_type != RB_OP_TYPE_RECLAIM); break; default: + op_type_str = "UNKNOWN"; break; } } par_barrier(PAR_COMM_WORLD); for (j = 0; j < args_cnt; j++) - if (args[j]->rebuild_cb) + if (args[j]->rebuild_cb) { + print_message("call rebuild_cb for %s rebuilding pool " DF_UUID + "\n", + op_type_str, DP_UUID(args[i]->pool.pool_uuid)); + args[j]->rebuild_cb(args[j]); + } if (args[0]->myrank == 0 && !args[0]->no_rebuild) test_rebuild_wait(args, args_cnt); for (j = 0; j < args_cnt; j++) { - if (args[j]->rebuild_post_cb) + if (args[j]->rebuild_post_cb) { + print_message("call rebuild_post_cb for %s rebuilt pool " DF_UUID + "\n", + op_type_str, DP_UUID(args[i]->pool.pool_uuid)); args[j]->rebuild_post_cb(args[j]); + } } } } @@ -970,8 +1001,8 @@ reintegrate_inflight_io(void *data) } ioreq_fini(&req); - sleep(12); print_message("sleep 12 seconds to wait for the stable epoch update.\n"); + sleep(12); if (arg->myrank == 0) daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, 0, 0, NULL); @@ -1182,57 +1213,75 @@ get_killing_rank_by_oid(test_arg_t *arg, daos_obj_id_t oid, int data_nr, *ranks_num = idx; } -/* inject fault on leader engine to stop an in-progress rebuild */ +static int +rebuild_stop_with_dmg_internal(const char *cfg, const uuid_t uuid, const char *grp, bool force) +{ + int rc; + + rc = dmg_pool_rebuild_stop(cfg, uuid, grp, force); + print_message("dmg pool rebuild stop " DF_UUID ", force=%d, rc=%d\n", DP_UUID(uuid), force, + rc); + assert_rc_equal(rc, 0); + return 0; +} + +/* stop an in-progress rebuild with dmg pool rebuild stop command */ int -fi_rebuild_stop(void *data) +rebuild_stop_with_dmg(void *data) { test_arg_t *arg = data; - d_rank_t leader; - int rc; - if (arg->myrank != 0) - return 0; + print_message("wait for rebuild to start for pool " DF_UUID "\n", + DP_UUID(arg->pool.pool_uuid)); + test_rebuild_wait_to_start(&arg, 1); + sleep(4); - rc = test_get_leader(arg, &leader); - if (rc) { - print_message("get pool " DF_UUIDF " leader failed: %d\n", - DP_UUID(arg->pool.pool_uuid), rc); - assert_rc_equal(rc, 0); - } + return rebuild_stop_with_dmg_internal(arg->dmg_config, arg->pool.pool_uuid, arg->group, + false); +} + +/* stop an in-progress rebuild with dmg pool rebuild stop command (force stop option) */ +int +rebuild_force_stop_with_dmg(void *data) +{ + test_arg_t *arg = data; - printf("%s(): pool " DF_UUIDF ": send ADMIN_STOP to leader rank %d\n", __FUNCTION__, - DP_UUID(arg->pool.pool_uuid), leader); - test_set_engine_fail_loc(arg, leader, DAOS_REBUILD_ADMIN_STOP | DAOS_FAIL_ONCE); + print_message("wait for rebuild to start for pool " DF_UUID "\n", + DP_UUID(arg->pool.pool_uuid)); + test_rebuild_wait_to_start(&arg, 1); + sleep(4); + return rebuild_stop_with_dmg_internal(arg->dmg_config, arg->pool.pool_uuid, arg->group, + true); +} + +/* start/reesume a stopped rebuild with dmg pool rebuild start command */ +int +rebuild_start_with_dmg(void *data) +{ + test_arg_t *arg = data; + int rc; + + rc = dmg_pool_rebuild_start(arg->dmg_config, arg->pool.pool_uuid, arg->group); + print_message("dmg pool rebuild start " DF_UUID ", rc=%d\n", DP_UUID(arg->pool.pool_uuid), + rc); + assert_rc_equal(rc, 0); return 0; } -/* resume after a stopped rebuild, and optionally inject a fault to start a new rebuild */ int -fi_rebuild_resume_wait(void *data) +rebuild_resume_wait(void *data) { test_arg_t *arg = data; - struct daos_rebuild_status *rst = &arg->pool.pool_info.pi_rebuild_st; - + struct daos_rebuild_status *rst = &arg->pool.pool_info.pi_rebuild_st; bool skip_restart = false; - d_rank_t leader; int rc; - if (arg->myrank != 0) - return 0; - - if (arg->rebuild_cb == fi_rebuild_resume_wait && arg->rebuild_cb_arg) + if (arg->rebuild_cb == rebuild_resume_wait && arg->rebuild_cb_arg) skip_restart = *((bool *)arg->rebuild_cb_arg); - if (arg->rebuild_post_cb == fi_rebuild_resume_wait && arg->rebuild_post_cb_arg) + if (arg->rebuild_post_cb == rebuild_resume_wait && arg->rebuild_post_cb_arg) skip_restart = *((bool *)arg->rebuild_post_cb_arg); - rc = test_get_leader(arg, &leader); - if (rc) { - print_message("get pool " DF_UUIDF " leader failed: %d\n", - DP_UUID(arg->pool.pool_uuid), rc); - assert_rc_equal(rc, 0); - } - /* Verify that the stop resulted in the correct rebuild status */ print_message("check: stopped rebuild rs_errno=%d (expect %d), rs_state=%d (expect %d)\n", rst->rs_errno, -DER_OP_CANCELED, rst->rs_state, DRS_NOT_STARTED); @@ -1243,10 +1292,8 @@ fi_rebuild_resume_wait(void *data) if (skip_restart) return 0; - /* Issue start fault injection to re-run the rebuild to successful completion */ - printf("%s(): pool " DF_UUIDF ": send ADMIN_START to leader rank %d\n", __FUNCTION__, - DP_UUID(arg->pool.pool_uuid), leader); - test_set_engine_fail_loc(arg, leader, DAOS_REBUILD_ADMIN_START | DAOS_FAIL_ONCE); + rc = rebuild_start_with_dmg(data); + assert_rc_equal(rc, 0); /* Verify that the start resulted in a fully completed rebuild. * NB: loop to get past the immediate remnant state from the previous stop (OP_CANCELED) diff --git a/src/tests/suite/daos_rebuild_simple.c b/src/tests/suite/daos_rebuild_simple.c index 85c05dc41ad..7571fef749e 100644 --- a/src/tests/suite/daos_rebuild_simple.c +++ b/src/tests/suite/daos_rebuild_simple.c @@ -108,8 +108,8 @@ rebuild_dkeys(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -170,8 +170,8 @@ rebuild_akeys(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, tgt, false); arg->rebuild_cb = NULL; @@ -222,8 +222,8 @@ rebuild_indexes(void **state) /* Rebuild rank 1 */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -283,8 +283,8 @@ rebuild_snap_update_recs(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -349,8 +349,8 @@ rebuild_snap_punch_recs(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -409,8 +409,8 @@ rebuild_snap_update_keys(void **state) } if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -517,8 +517,8 @@ rebuild_snap_punch_keys(void **state) } if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -606,8 +606,8 @@ rebuild_snap_punch_empty(void **state) /* stop exclude rebuild, but skip the start, then directly reintegrate */ if (arg->interactive_rebuild) { bool skip_restart = true; - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; arg->rebuild_post_cb_arg = &skip_restart; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); @@ -655,8 +655,8 @@ rebuild_snap_punch_empty(void **state) /* from a stopped (not restarted) exclude, directly reintegrate (stop+start this rebuild) */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt); rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX); @@ -705,8 +705,8 @@ rebuild_multiple(void **state) } if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -763,8 +763,8 @@ rebuild_large_rec(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -803,8 +803,8 @@ rebuild_objects(void **state) rebuild_io(arg, oids, OBJ_NR); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -844,6 +844,12 @@ rebuild_sx_object_internal(void **state, daos_oclass_id_t oclass, if (!test_runable(arg, 4)) return; + if (arg->interactive_rebuild && !wait_rebuild) { + print_message("SKIP due to interactive_rebuild enabled, but not tested here\n"); + skip(); + return; + } + oid = daos_test_oid_gen(arg->coh, oclass, 0, 0, arg->myrank); ioreq_init(&req, arg->coh, oid, DAOS_IOD_ARRAY, arg); print_message("insert 100 dkeys\n"); @@ -874,30 +880,35 @@ rebuild_sx_object_internal(void **state, daos_oclass_id_t oclass, get_killing_rank_by_oid(arg, oid, 1, 0, &rank, &rank_nr); /** exclude the target of this obj's replicas */ - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool exclude rank %u " DF_UUID "\n", rank, DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_exclude(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) + rebuild_stop_with_dmg(arg); + /* wait until exclude rebuild done */ if (wait_rebuild) { test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); } - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool reintegrate rank %u " DF_UUID "\n", rank, + DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_reintegrate(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) + rebuild_stop_with_dmg(arg); + /* wait until reintegration rebuild is done */ if (wait_rebuild) { test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); } print_message("lookup 100 dkeys\n"); @@ -978,27 +989,32 @@ rebuild_large_object(void **state) ioreq_fini(&req); } - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool exclude rank %u " DF_UUID "\n", rank, DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_exclude(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) + rebuild_stop_with_dmg(arg); + /* wait until exclude rebuild done */ test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool reintegrate rank %u " DF_UUID "\n", rank, + DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_reintegrate(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) + rebuild_stop_with_dmg(arg); + /* wait until reintegration rebuild is done */ test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); print_message("success\n"); } @@ -1094,8 +1110,8 @@ rebuild_large_snap(void **state) /* stop and resume the exclude rebuild */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); ioreq_fini(&req); @@ -1142,8 +1158,8 @@ rebuild_full_shards(void **state) /* rebuild and reintegration to use full shards */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } /* stop and resume the first exclude rebuild (but not the second one) */ rebuild_single_pool_target(arg, 0, -1, false); @@ -1153,8 +1169,8 @@ rebuild_full_shards(void **state) /* stop and resume the first reintegrate rebuild (but not the second one) */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } reintegrate_single_pool_target(arg, 0, -1); arg->rebuild_cb = NULL; @@ -1196,8 +1212,8 @@ rebuild_punch_recs(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], -1, false); arg->rebuild_cb = NULL; @@ -1253,8 +1269,8 @@ rebuild_multiple_group(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -1264,10 +1280,6 @@ rebuild_multiple_group(void **state) if (rc != 0) assert_rc_equal(rc, -DER_NOSYS); - if (arg->interactive_rebuild) { - arg->rebuild_cb = NULL; - arg->rebuild_post_cb = NULL; - } reintegrate_with_inflight_io(arg, &oid, kill_rank, -1); rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX); @@ -1311,8 +1323,8 @@ rebuild_with_large_offset(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -1363,8 +1375,8 @@ rebuild_with_large_key(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -1432,8 +1444,8 @@ rebuild_with_dfs_open_create_punch(void **state) dfs_obj2id(dir, &oid); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rank = get_rank_by_oid_shard(arg, oid, 0); rebuild_single_pool_rank(arg, rank, false); diff --git a/src/tests/suite/daos_test.h b/src/tests/suite/daos_test.h index f7ddeb7ff84..81fa42d9f02 100644 --- a/src/tests/suite/daos_test.h +++ b/src/tests/suite/daos_test.h @@ -397,6 +397,8 @@ int test_pool_get_info(test_arg_t *arg, daos_pool_info_t *pinfo, d_rank_list_t * int test_get_leader(test_arg_t *arg, d_rank_t *rank); bool test_rebuild_query(test_arg_t **args, int args_cnt); void test_rebuild_wait(test_arg_t **args, int args_cnt); +void + test_rebuild_wait_to_start(test_arg_t **args, int args_cnt); int daos_pool_set_prop(const uuid_t pool_uuid, const char *name, const char *value); @@ -456,22 +458,34 @@ void rebuild_add_back_tgts(test_arg_t *arg, d_rank_t failed_rank, int rebuild_pool_disconnect_internal(void *data); int rebuild_pool_connect_internal(void *data); - -int rebuild_sub_setup(void **state); -int rebuild_sub_rf1_setup(void **state); -int rebuild_sub_rf0_setup(void **state); -int rebuild_sub_teardown(void **state); -int rebuild_small_sub_setup(void **state); -int rebuild_small_sub_rf1_setup(void **state); -int rebuild_small_sub_rf0_setup(void **state); -int rebuild_sub_3nodes_rf0_setup(void **state); -int rebuild_sub_6nodes_rf1_setup(void **state); -int rebuild_sub_setup_common(void **state, daos_size_t pool_size, int node_nr, uint32_t rf); - int -fi_rebuild_stop(void *data); +rebuild_sub_setup(void **state); +int +rebuild_sub_rf1_setup(void **state); +int +rebuild_sub_rf0_setup(void **state); +int +rebuild_sub_teardown(void **state); +int +rebuild_small_sub_setup(void **state); +int +rebuild_small_sub_rf1_setup(void **state); +int +rebuild_small_sub_rf0_setup(void **state); +int +rebuild_sub_3nodes_rf0_setup(void **state); +int +rebuild_sub_6nodes_rf1_setup(void **state); +int +rebuild_sub_setup_common(void **state, daos_size_t pool_size, int node_nr, uint32_t rf); +int +rebuild_stop_with_dmg(void *data); +int +rebuild_force_stop_with_dmg(void *data); +int +rebuild_start_with_dmg(void *data); int - fi_rebuild_resume_wait(void *data); + rebuild_resume_wait(void *data); int get_server_config(char *host, char *server_config_file); int get_log_file(char *host, char *server_config_file, diff --git a/src/tests/suite/daos_test_common.c b/src/tests/suite/daos_test_common.c index 2f6b3fd3691..5d34e3a62c6 100644 --- a/src/tests/suite/daos_test_common.c +++ b/src/tests/suite/daos_test_common.c @@ -791,7 +791,30 @@ test_pool_get_info(test_arg_t *arg, daos_pool_info_t *pinfo, d_rank_list_t **eng } static bool -rebuild_pool_wait(test_arg_t *arg) +rebuild_pool_started(test_arg_t *arg) +{ + daos_pool_info_t pinfo = {0}; + struct daos_rebuild_status *rst; + int rc; + + pinfo.pi_bits = DPI_REBUILD_STATUS; + rc = test_pool_get_info(arg, &pinfo, NULL /* engine_ranks */); + rst = &pinfo.pi_rebuild_st; + + if (rc != 0) { + print_message("pool query for rebuild status failed, rc=%d, pool " DF_UUIDF "\n", + rc, DP_UUID(arg->pool.pool_uuid)); + return false; + } else { + bool started = (rst->rs_state == DRS_IN_PROGRESS); + print_message("rebuild for pool " DF_UUIDF "has %sstarted\n", + DP_UUID(arg->pool.pool_uuid), started ? "" : "not yet "); + return started; + } +} + +static bool +rebuild_pool_done(test_arg_t *arg) { daos_pool_info_t pinfo = {0}; struct daos_rebuild_status *rst; @@ -862,6 +885,31 @@ test_get_last_svr_rank(test_arg_t *arg) return arg->srv_nnodes - disable_nodes - 1; } +bool +test_rebuild_started(test_arg_t **args, int args_cnt) +{ + bool all_started = true; + int i; + + for (i = 0; i < args_cnt; i++) { + bool started = true; + + if (!args[i]->pool.destroyed) + started = rebuild_pool_started(args[i]); + + if (!started) + all_started = false; + } + return all_started; +} + +void +test_rebuild_wait_to_start(test_arg_t **args, int args_cnt) +{ + while (!test_rebuild_started(args, args_cnt)) + sleep(2); +} + bool test_rebuild_query(test_arg_t **args, int args_cnt) { @@ -872,7 +920,7 @@ test_rebuild_query(test_arg_t **args, int args_cnt) bool done = true; if (!args[i]->pool.destroyed) - done = rebuild_pool_wait(args[i]); + done = rebuild_pool_done(args[i]); if (!done) all_done = false;