DAOS-17996 test: enhance test_dmg_system_reformat

src/tests/ftest/control/dmg_system_reformat.py

@@ -1,14 +1,18 @@
 """
   (C) Copyright 2020-2024 Intel Corporation.
+  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
+import os
 import time
 
 from apricot import TestWithServers
 from avocado.core.exceptions import TestFail
 from exception_utils import CommandFailure
-from general_utils import journalctl_time
+from general_utils import check_file_exists, journalctl_time
+from ior_utils import write_data
+from run_utils import run_remote
 from test_utils_pool import add_pool, get_size_params
 
 
@@ -21,12 +25,6 @@ class DmgSystemReformatTest(TestWithServers):
     :avocado: recursive
     """
 
-    def setUp(self):
-        """Set up each test case."""
-        # Create test-case-specific DAOS log files
-        self.update_log_file_names()
-        super().setUp()
-
     def test_dmg_system_reformat(self):
         """
         JIRA ID: DAOS-5415
@@ -40,70 +38,86 @@ def test_dmg_system_reformat(self):
         """
         dmg = self.get_dmg_command().copy()
 
-        # Create pool using 90% of the available NVMe capacity
-        pools = [add_pool(self, dmg=dmg)]
-
-        self.log.info("Check that new pool will fail with DER_NOSPACE")
-        dmg.exit_status_exception = False
-        pools.append(add_pool(self, create=False, **get_size_params(pools[0])))
-        try:
-            pools[-1].create()
-        except TestFail as error:
-            self.log.info("Pool create failed: %s", str(error))
-            if "-1007" not in str(error):
-                self.fail("Pool create did not fail due to DER_NOSPACE!")
-        dmg.exit_status_exception = True
-
-        self.log.info("Stop running engine instances: 'dmg system stop'")
-        dmg.system_stop(force=True)
-        if dmg.result.exit_status != 0:
-            self.fail("Detected issues performing a system stop: {}".format(
-                dmg.result.stderr_text))
+        self.log_step("Create pool using 90% of the available NVMe capacity")
+        pool = add_pool(self, connect=False, dmg=dmg)
 
-        # Remove pools and disable removing pools that about to be removed by formatting
-        for pool in pools:
-            pool.skip_cleanup()
-        pools = []
+        self.log_step("Write data to all targets in the pool")
+        container = self.get_container(pool)
+        write_data(self, container)
+
+        self.log_step("Check that new pool will fail with DER_NOSPACE")
+        with dmg.no_exception():
+            pool2 = add_pool(self, connect=False, create=False, **get_size_params(pool))
+            try:
+                pool2.create()
+                self.fail("Pool create was expected to fail with DER_NOSPACE!")
+            except TestFail as error:
+                self.log.info("Pool create failed: %s", str(error))
+                if "-1007" not in str(error):
+                    self.fail("Pool create did not fail due to DER_NOSPACE!")
+
+        self.log_step("Verify expected metadata exists before erase")
+        # Check this upfront so later when we check it is gone we know the erase worked
+        scm_mount = self.server_managers[0].get_config_value("scm_mount")
+        superblock_path = os.path.join(scm_mount, "superblock")
+        control_metadata_path = self.server_managers[0].manager.job.control_metadata.path.value
+        run_remote(self.log, self.hostlist_servers, f"sudo ls -l {scm_mount}")
+        _, missing_hosts = check_file_exists(self.hostlist_servers, superblock_path, sudo=True)
+        if missing_hosts:
+            self.fail(f"superblock does not exist before erase: {superblock_path}")
+        if control_metadata_path:
+            self.log.info("control_metadata_path = '%s'", control_metadata_path)
+            _, missing_hosts = check_file_exists(
+                self.hostlist_servers, control_metadata_path, sudo=True)
+            if missing_hosts:
+                self.fail(f"control metadata does not exist before erase: {control_metadata_path}")
+
+        self.log_step("Stop running engine instances: 'dmg system stop'")
+        dmg.system_stop(force=True)
 
         # Perform a dmg system erase to allow the dmg storage format to succeed
-        self.log.info("Perform dmg system erase on all system ranks:")
+        self.log_step("Perform dmg system erase on all system ranks:")
         dmg.system_erase()
-        if dmg.result.exit_status != 0:
-            self.fail("Issues performing system erase: {}".format(
-                dmg.result.stderr_text))
-
-        self.log.info("Perform dmg storage format on all system ranks:")
 
+        # Disable pool and container cleanup in teardown since this pool was erased
+        pool.skip_cleanup()
+        pool = None
+        container.skip_cleanup()
+        container = None
+
+        self.log_step("Verify metadata is erased")
+        _, missing_hosts = check_file_exists(self.hostlist_servers, superblock_path, sudo=True)
+        if missing_hosts != self.hostlist_servers:
+            self.fail(f"superblock still exists after erase: {superblock_path}")
+        if control_metadata_path:
+            _, missing_hosts = check_file_exists(
+                self.hostlist_servers, control_metadata_path, sudo=True)
+            if missing_hosts != self.hostlist_servers:
+                self.fail(f"control metadata still exists after erase: {control_metadata_path}")
+
+        self.log_step("Perform dmg storage format on all system ranks:")
         # Calling storage format after system stop too soon would fail, so
         # wait 10 sec and retry up to 4 times.
-        count = 0
-        while count < 4:
+        for _ in range(4):
             try:
                 dmg.storage_format(force=True)
-                if dmg.result.exit_status != 0:
-                    self.fail(
-                        "Issues performing storage format --force: {}".format(
-                            dmg.result.stderr_text))
                 break
             except CommandFailure as error:
                 self.log.info("Storage format failed. Wait 10 sec and retry. %s", error)
-                count += 1
                 time.sleep(10)
 
-        # Check that engine starts up again
+        self.log_step("Verify engines start again")
         self.log.info("<SERVER> Waiting for the engines to start")
         self.server_managers[-1].manager.timestamps["start"] = journalctl_time()
         self.server_managers[-1].detect_engine_start()
 
-        # Check that we have cleared storage by checking pool list
+        self.log_step("Verify no pools are in the pool list after reformat")
         if dmg.get_pool_list_uuids():
             self.fail("Detected pools in storage after reformat: {}".format(
                 dmg.result.stdout_text))
 
-        # Create last pool now that memory has been wiped.
-        pools.append(add_pool(self, connect=False, dmg=dmg))
-
-        # Lastly, verify that last created pool is in the list
-        pool_uuids = dmg.get_pool_list_uuids()
+        self.log_step("Create a new pool and verify only the new pool is in the pool list")
+        pool = add_pool(self, connect=False, dmg=dmg)
+        pool_uuids = list(map(str.lower, dmg.get_pool_list_uuids()))
         self.assertEqual(
-            pool_uuids[0].lower(), pools[-1].uuid.lower(), "{} missing from list".format(pools[-1]))
+            pool_uuids, [pool.uuid.lower()], f"unexpected pool list output: {pool_uuids}")

src/tests/ftest/control/dmg_system_reformat.yaml

@@ -1,13 +1,27 @@
 hosts:
   test_servers: 1
   test_clients: 1
+
 timeout: 300
+
 server_config:
   name: daos_server
   engines_per_host: 1
   engines:
     0:
       storage: auto
   system_ram_reserved: 32
+
 pool:
   size: 90%
+
+container:
+  type: POSIX
+
+ior_write:
+  ppn: 16
+  flags: "-k -v -w -W -G 1"
+  api: DFS
+  transfer_size: 1M
+  block_size: 64M  # Aggregate 1G
+  dfs_oclass: SX