From 72ac624a4259e4cf24158094aa2e7df04933ffe3 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Fri, 19 Sep 2025 15:47:40 +0000 Subject: [PATCH] DAOS-17996 test: enhance test_dmg_system_reformat - Verify dmg system erase actually removes metadata Skip-unit-tests: true Skip-fault-injection-test: true Signed-off-by: Dalton Bohning --- .../ftest/control/dmg_system_reformat.py | 116 ++++++++++-------- .../ftest/control/dmg_system_reformat.yaml | 14 +++ 2 files changed, 79 insertions(+), 51 deletions(-) diff --git a/src/tests/ftest/control/dmg_system_reformat.py b/src/tests/ftest/control/dmg_system_reformat.py index 887c23bc0f1..d0d25c6089f 100644 --- a/src/tests/ftest/control/dmg_system_reformat.py +++ b/src/tests/ftest/control/dmg_system_reformat.py @@ -1,14 +1,18 @@ """ (C) Copyright 2020-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ +import os import time from apricot import TestWithServers from avocado.core.exceptions import TestFail from exception_utils import CommandFailure -from general_utils import journalctl_time +from general_utils import check_file_exists, journalctl_time +from ior_utils import write_data +from run_utils import run_remote from test_utils_pool import add_pool, get_size_params @@ -21,12 +25,6 @@ class DmgSystemReformatTest(TestWithServers): :avocado: recursive """ - def setUp(self): - """Set up each test case.""" - # Create test-case-specific DAOS log files - self.update_log_file_names() - super().setUp() - def test_dmg_system_reformat(self): """ JIRA ID: DAOS-5415 @@ -40,70 +38,86 @@ def test_dmg_system_reformat(self): """ dmg = self.get_dmg_command().copy() - # Create pool using 90% of the available NVMe capacity - pools = [add_pool(self, dmg=dmg)] - - self.log.info("Check that new pool will fail with DER_NOSPACE") - dmg.exit_status_exception = False - pools.append(add_pool(self, create=False, **get_size_params(pools[0]))) - try: - pools[-1].create() - except TestFail as error: - self.log.info("Pool create failed: %s", str(error)) - if "-1007" not in str(error): - self.fail("Pool create did not fail due to DER_NOSPACE!") - dmg.exit_status_exception = True - - self.log.info("Stop running engine instances: 'dmg system stop'") - dmg.system_stop(force=True) - if dmg.result.exit_status != 0: - self.fail("Detected issues performing a system stop: {}".format( - dmg.result.stderr_text)) + self.log_step("Create pool using 90% of the available NVMe capacity") + pool = add_pool(self, connect=False, dmg=dmg) - # Remove pools and disable removing pools that about to be removed by formatting - for pool in pools: - pool.skip_cleanup() - pools = [] + self.log_step("Write data to all targets in the pool") + container = self.get_container(pool) + write_data(self, container) + + self.log_step("Check that new pool will fail with DER_NOSPACE") + with dmg.no_exception(): + pool2 = add_pool(self, connect=False, create=False, **get_size_params(pool)) + try: + pool2.create() + self.fail("Pool create was expected to fail with DER_NOSPACE!") + except TestFail as error: + self.log.info("Pool create failed: %s", str(error)) + if "-1007" not in str(error): + self.fail("Pool create did not fail due to DER_NOSPACE!") + + self.log_step("Verify expected metadata exists before erase") + # Check this upfront so later when we check it is gone we know the erase worked + scm_mount = self.server_managers[0].get_config_value("scm_mount") + superblock_path = os.path.join(scm_mount, "superblock") + control_metadata_path = self.server_managers[0].manager.job.control_metadata.path.value + run_remote(self.log, self.hostlist_servers, f"sudo ls -l {scm_mount}") + _, missing_hosts = check_file_exists(self.hostlist_servers, superblock_path, sudo=True) + if missing_hosts: + self.fail(f"superblock does not exist before erase: {superblock_path}") + if control_metadata_path: + self.log.info("control_metadata_path = '%s'", control_metadata_path) + _, missing_hosts = check_file_exists( + self.hostlist_servers, control_metadata_path, sudo=True) + if missing_hosts: + self.fail(f"control metadata does not exist before erase: {control_metadata_path}") + + self.log_step("Stop running engine instances: 'dmg system stop'") + dmg.system_stop(force=True) # Perform a dmg system erase to allow the dmg storage format to succeed - self.log.info("Perform dmg system erase on all system ranks:") + self.log_step("Perform dmg system erase on all system ranks:") dmg.system_erase() - if dmg.result.exit_status != 0: - self.fail("Issues performing system erase: {}".format( - dmg.result.stderr_text)) - - self.log.info("Perform dmg storage format on all system ranks:") + # Disable pool and container cleanup in teardown since this pool was erased + pool.skip_cleanup() + pool = None + container.skip_cleanup() + container = None + + self.log_step("Verify metadata is erased") + _, missing_hosts = check_file_exists(self.hostlist_servers, superblock_path, sudo=True) + if missing_hosts != self.hostlist_servers: + self.fail(f"superblock still exists after erase: {superblock_path}") + if control_metadata_path: + _, missing_hosts = check_file_exists( + self.hostlist_servers, control_metadata_path, sudo=True) + if missing_hosts != self.hostlist_servers: + self.fail(f"control metadata still exists after erase: {control_metadata_path}") + + self.log_step("Perform dmg storage format on all system ranks:") # Calling storage format after system stop too soon would fail, so # wait 10 sec and retry up to 4 times. - count = 0 - while count < 4: + for _ in range(4): try: dmg.storage_format(force=True) - if dmg.result.exit_status != 0: - self.fail( - "Issues performing storage format --force: {}".format( - dmg.result.stderr_text)) break except CommandFailure as error: self.log.info("Storage format failed. Wait 10 sec and retry. %s", error) - count += 1 time.sleep(10) - # Check that engine starts up again + self.log_step("Verify engines start again") self.log.info(" Waiting for the engines to start") self.server_managers[-1].manager.timestamps["start"] = journalctl_time() self.server_managers[-1].detect_engine_start() - # Check that we have cleared storage by checking pool list + self.log_step("Verify no pools are in the pool list after reformat") if dmg.get_pool_list_uuids(): self.fail("Detected pools in storage after reformat: {}".format( dmg.result.stdout_text)) - # Create last pool now that memory has been wiped. - pools.append(add_pool(self, connect=False, dmg=dmg)) - - # Lastly, verify that last created pool is in the list - pool_uuids = dmg.get_pool_list_uuids() + self.log_step("Create a new pool and verify only the new pool is in the pool list") + pool = add_pool(self, connect=False, dmg=dmg) + pool_uuids = list(map(str.lower, dmg.get_pool_list_uuids())) self.assertEqual( - pool_uuids[0].lower(), pools[-1].uuid.lower(), "{} missing from list".format(pools[-1])) + pool_uuids, [pool.uuid.lower()], f"unexpected pool list output: {pool_uuids}") diff --git a/src/tests/ftest/control/dmg_system_reformat.yaml b/src/tests/ftest/control/dmg_system_reformat.yaml index ca86e2aa8d8..eb7a7a741ad 100644 --- a/src/tests/ftest/control/dmg_system_reformat.yaml +++ b/src/tests/ftest/control/dmg_system_reformat.yaml @@ -1,7 +1,9 @@ hosts: test_servers: 1 test_clients: 1 + timeout: 300 + server_config: name: daos_server engines_per_host: 1 @@ -9,5 +11,17 @@ server_config: 0: storage: auto system_ram_reserved: 32 + pool: size: 90% + +container: + type: POSIX + +ior_write: + ppn: 16 + flags: "-k -v -w -W -G 1" + api: DFS + transfer_size: 1M + block_size: 64M # Aggregate 1G + dfs_oclass: SX