DAOS-17357 rebuild: dmg helpers for rebuild stop/start commands

src/common/tests_dmg_helpers.c

@@ -1206,6 +1206,80 @@ dmg_pool_list(const char *dmg_config_file, const char *group,
 	return rc;
 }
 
+int
+dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force)
+{
+	char                uuid_str[DAOS_UUID_STR_SIZE];
+	int                 argcount = 0;
+	char              **args     = NULL;
+	struct json_object *dmg_out  = NULL;
+	int                 rc       = 0;
+
+	uuid_unparse_lower(uuid, uuid_str);
+	args = cmd_push_arg(args, &argcount, "%s ", uuid_str);
+	if (args == NULL)
+		D_GOTO(out, rc = -DER_NOMEM);
+
+	if (grp != NULL) {
+		args = cmd_push_arg(args, &argcount, "--sys=%s ", grp);
+		if (args == NULL)
+			D_GOTO(out, rc = -DER_NOMEM);
+	}
+
+	if (force) {
+		args = cmd_push_arg(args, &argcount, "--force");
+		if (args == NULL)
+			D_GOTO(out, rc = -DER_NOMEM);
+	}
+
+	rc = daos_dmg_json_pipe("pool rebuild stop", dmg_config_file, args, argcount, &dmg_out);
+	if (rc != 0) {
+		D_ERROR("dmg pool rebuild stop failed\n");
+		goto out_json;
+	}
+
+out_json:
+	if (dmg_out != NULL)
+		json_object_put(dmg_out);
+	cmd_free_args(args, argcount);
+out:
+	return rc;
+}
+
+int
+dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp)
+{
+	char                uuid_str[DAOS_UUID_STR_SIZE];
+	int                 argcount = 0;
+	char              **args     = NULL;
+	struct json_object *dmg_out  = NULL;
+	int                 rc       = 0;
+
+	uuid_unparse_lower(uuid, uuid_str);
+	args = cmd_push_arg(args, &argcount, "%s ", uuid_str);
+	if (args == NULL)
+		D_GOTO(out, rc = -DER_NOMEM);
+
+	if (grp != NULL) {
+		args = cmd_push_arg(args, &argcount, "--sys=%s ", grp);
+		if (args == NULL)
+			D_GOTO(out, rc = -DER_NOMEM);
+	}
+
+	rc = daos_dmg_json_pipe("pool rebuild start", dmg_config_file, args, argcount, &dmg_out);
+	if (rc != 0) {
+		D_ERROR("dmg pool rebuild start failed\n");
+		goto out_json;
+	}
+
+out_json:
+	if (dmg_out != NULL)
+		json_object_put(dmg_out);
+	cmd_free_args(args, argcount);
+out:
+	return rc;
+}
+
 static int
 parse_device_info(struct json_object *smd_dev, device_list *devices,
 		  char *host, int dev_length, int *disks)

src/include/daos/common.h

@@ -789,9 +789,6 @@ enum {
 #define DAOS_RDB_SKIP_APPENDENTRIES_FAIL (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1a)
 #define DAOS_FORCE_REFRESH_POOL_MAP	  (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1b)
 
-#define DAOS_REBUILD_ADMIN_STOP           (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1c)
-#define DAOS_REBUILD_ADMIN_START          (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1d)
-
 #define DAOS_FORCE_CAPA_FETCH		(DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1e)
 #define DAOS_FORCE_PROP_VERIFY		(DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1f)
 

src/include/daos/tests_lib.h

@@ -484,6 +484,25 @@ dmg_pool_set_prop(const char *dmg_config_file,
 int dmg_pool_get_prop(const char *dmg_config_file, const char *label, const uuid_t uuid,
 		      const char *name, char **value);
 
+/**
+ * Interactively stop a pool's currently-running rebuild.
+ * \param dmg_config_file [IN] DMG config file.
+ * \param uuid            [IN] UUID of the pool.
+ * \param grp             [IN] Process set name of the DAOS servers managing the pool.
+ * \param force           [IN] forcibly stop a rebuild that is failing.
+ */
+int
+dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force);
+
+/**
+ * Interactively start/resume a pool's rebuilding.
+ * \param dmg_config_file [IN] DMG config file.
+ * \param uuid            [IN] UUID of the pool.
+ * \param grp             [IN] Process set name of the DAOS servers managing the pool.
+ */
+int
+    dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp);
+
 /**
  * List all disks in the specified DAOS system.
  *

src/rebuild/srv.c

@@ -874,9 +874,6 @@ enum {
 	RB_BCAST_QUERY,
 };
 
-/* for interactive mode rebuild fault-injection testing */
-static uuid_t test_stop_start_puuid;
-
 /* administrator (via dmg command) asks to stop the currently-running rebuild for pool */
 int
 ds_rebuild_admin_stop(struct ds_pool *pool, uint32_t force)
@@ -921,9 +918,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op,
 	unsigned int          total;
 	struct sched_req_attr attr = {0};
 	d_rank_t              myrank;
-	unsigned int          nsleeps          = 0;
-	bool                  fi_admin_stop_fi = false;
-	bool                  fi_admin_stop    = false;
 	int                   rc;
 
 	rc = crt_group_size(pool->sp_group, &total);
@@ -948,18 +942,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op,
 		bool                        rebuild_abort = false;
 		int                         i;
 
-		if (!fi_admin_stop_fi)
-			fi_admin_stop_fi = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_STOP);
-		fi_admin_stop = (fi_admin_stop_fi && (op == RB_OP_REBUILD));
-
-		if (fi_admin_stop && (nsleeps > 0)) {
-			uuid_copy(test_stop_start_puuid, rgt->rgt_pool_uuid);
-
-			/* What the real RPC handler will call. We will exit below due to rgt_abort
-			 */
-			ds_pool_rebuild_stop(test_stop_start_puuid, 0 /* force */, NULL /* hint */);
-		}
-
 		ABT_rwlock_rdlock(pool->sp_lock);
 		rc = map_ranks_init(pool->sp_map,
 				    PO_COMP_ST_UP | PO_COMP_ST_DOWN |
@@ -1055,8 +1037,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op,
 sleep:
 		update_and_warn_for_slow_engines(rgt);
 		sched_req_sleep(rgt->rgt_ult, RBLD_CHECK_INTV);
-		if (fi_admin_stop)
-			nsleeps++;
 	}
 
 	sched_req_put(rgt->rgt_ult);
@@ -2064,25 +2044,13 @@ rebuild_ults(void *arg)
 	while (DAOS_FAIL_CHECK(DAOS_REBUILD_HANG))
 		ABT_thread_yield();
 
-	/* Stay alive for fault-injection possibility (need a ULT for DAOS_REBUILD_ADMIN_START) */
-	while (true) {
-		bool fi_admin_start = false;
-
+	while (!d_list_empty(&rebuild_gst.rg_queue_list) ||
+	       !d_list_empty(&rebuild_gst.rg_running_list)) {
 		if (rebuild_gst.rg_abort) {
 			D_INFO("abort rebuilds\n");
 			break;
 		}
 
-		fi_admin_start = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_START);
-		if (fi_admin_start) {
-			if (!uuid_is_null(test_stop_start_puuid)) {
-				D_INFO("fi_admin_start=%d, apply to pool " DF_UUID "\n",
-				       fi_admin_start, DP_UUID(test_stop_start_puuid));
-				ds_pool_rebuild_start(test_stop_start_puuid, NULL /* hint */);
-				uuid_clear(test_stop_start_puuid);
-			}
-		}
-
 		if (d_list_empty(&rebuild_gst.rg_queue_list) ||
 		    rebuild_gst.rg_inflight >= REBUILD_MAX_INFLIGHT) {
 			D_INFO("in-flight rebuild %u - going to sleep for 5 seconds\n",
@@ -3027,8 +2995,6 @@ init(void)
 {
 	int rc;
 
-	uuid_clear(test_stop_start_puuid);
-
 	D_INIT_LIST_HEAD(&rebuild_gst.rg_tgt_tracker_list);
 	D_INIT_LIST_HEAD(&rebuild_gst.rg_global_tracker_list);
 	D_INIT_LIST_HEAD(&rebuild_gst.rg_completed_list);