diff --git a/.github/workflows/bash_unit_testing.yml b/.github/workflows/bash_unit_testing.yml index 964f89bf0b7..48b5d4ed149 100644 --- a/.github/workflows/bash_unit_testing.yml +++ b/.github/workflows/bash_unit_testing.yml @@ -4,7 +4,7 @@ on: pull_request: concurrency: - group: bash-unit-test-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true defaults: diff --git a/.github/workflows/bullseye-coverage.yml b/.github/workflows/bullseye-coverage.yml index 1ae0cb19a80..d9a1e9b79e1 100644 --- a/.github/workflows/bullseye-coverage.yml +++ b/.github/workflows/bullseye-coverage.yml @@ -47,7 +47,7 @@ on: concurrency: - group: bullseye-coverage-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true defaults: @@ -366,7 +366,7 @@ jobs: if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: check_name: ${{ env.STAGE_NAME }} Test Results github_token: ${{ secrets.GITHUB_TOKEN }} @@ -374,14 +374,14 @@ jobs: - name: Publish artifacts if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} artifacts path: ${{ env.STAGE_NAME }}/** - name: Upload test results if: (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} test-results path: ${{ env.STAGE_NAME }}/**/results.xml @@ -634,7 +634,7 @@ jobs: if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: check_name: ${{ env.STAGE_NAME }} Test Results github_token: ${{ secrets.GITHUB_TOKEN }} @@ -642,14 +642,14 @@ jobs: - name: Publish artifacts if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} artifacts path: ${{ env.STAGE_NAME }}/** - name: Upload test results if: (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} test-results path: ${{ env.STAGE_NAME }}/**/results.xml diff --git a/.github/workflows/ci2.yml b/.github/workflows/ci2.yml index c0e8b79d110..99d61f6dcb5 100644 --- a/.github/workflows/ci2.yml +++ b/.github/workflows/ci2.yml @@ -4,7 +4,7 @@ on: pull_request: concurrency: - group: ci2-${{ github.head_ref }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true permissions: {} @@ -68,7 +68,7 @@ jobs: - name: Publish NLT test results if: always() # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} files: nlt-junit.xml diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index 581b52ad25d..9f273af14ff 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -18,6 +18,10 @@ on: - requirements-utest.txt - utils/build.config +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + permissions: {} jobs: @@ -139,7 +143,7 @@ jobs: - name: Publish NLT test results if: always() # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} files: nlt-junit.xml diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 40c350b0400..a4befc36af7 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -9,6 +9,10 @@ on: - 'release/*' pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + permissions: {} jobs: @@ -21,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python environment @@ -44,7 +48,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Run @@ -62,7 +66,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Check DAOS logging macro use. @@ -73,7 +77,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Check DAOS ftest tags. run: \[ ! -x src/tests/ftest/tags.py \] || ./src/tests/ftest/tags.py lint --verbose @@ -82,7 +86,7 @@ jobs: name: Flake8 check steps: - name: Check out source repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python environment @@ -115,7 +119,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Install doxygen @@ -125,7 +129,7 @@ jobs: - name: Run check run: doxygen Doxyfile - name: 'Upload Artifact' - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: API Documentation path: docs/doxygen/html/ @@ -136,7 +140,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 @@ -156,11 +160,11 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install extra python packages run: python3 -m pip install --requirement utils/cq/requirements.txt - name: Run check - uses: codespell-project/actions-codespell@406322ec52dd7b488e48c1c4b82e2a8b3a1bf630 # master + uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # master with: skip: ./src/control/vendor,./src/control/go.sum,./.git,./utils/*.patch ignore_words_file: ci/codespell.ignores @@ -168,18 +172,26 @@ jobs: clang-format: name: Clang Format - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Check out source repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 - name: Pull via git run: git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Get merge-base ref + id: get_merge_base + run: | + ref="$(git merge-base HEAD ${{ github.event.pull_request.base.sha || github.ref }})" + echo "ref=$ref" >> $GITHUB_OUTPUT - name: Run check in docker uses: ./.github/actions/clang-format with: - target: origin/${{ github.event.pull_request.base.ref }} + target: ${{ steps.get_merge_base.outputs.ref }} - name: Export changes - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 if: failure() with: name: format-patch-for-pr-${{ github.event.pull_request.number }} @@ -190,7 +202,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Check out source repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python environment @@ -207,7 +219,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Check out source repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index 108c764f81a..c33fe62c222 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -62,7 +62,7 @@ jobs: # uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: SARIF file path: results.sarif @@ -71,6 +71,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7 + uses: github/codeql-action/upload-sarif@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: sarif_file: results.sarif diff --git a/.github/workflows/rpm-build-and-test.yml b/.github/workflows/rpm-build-and-test.yml index de197693b54..70488c67c9a 100644 --- a/.github/workflows/rpm-build-and-test.yml +++ b/.github/workflows/rpm-build-and-test.yml @@ -54,7 +54,7 @@ on: - 'release/*' concurrency: - group: rpm-build-and-test-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true defaults: @@ -374,7 +374,7 @@ jobs: if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: check_name: ${{ env.STAGE_NAME }} Test Results github_token: ${{ secrets.GITHUB_TOKEN }} @@ -382,14 +382,14 @@ jobs: - name: Publish artifacts if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} artifacts path: ${{ env.STAGE_NAME }}/** - name: Upload test results if: (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} test-results path: ${{ env.STAGE_NAME }}/**/results.xml @@ -642,7 +642,7 @@ jobs: if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' # yamllint disable-line rule:line-length - uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 + uses: EnricoMi/publish-unit-test-result-action@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 with: check_name: ${{ env.STAGE_NAME }} Test Results github_token: ${{ secrets.GITHUB_TOKEN }} @@ -650,14 +650,14 @@ jobs: - name: Publish artifacts if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} artifacts path: ${{ env.STAGE_NAME }}/** - name: Upload test results if: (success() || failure()) && steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ env.STAGE_NAME }} test-results path: ${{ env.STAGE_NAME }}/**/results.xml diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index f2ca5ef18bc..4a708cdcbf8 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -11,6 +11,16 @@ on: branches: ["master", "release/**"] pull_request: branches: ["master", "release/**"] + paths: + - .github/workflows/trivy.yml + - requirements-*.txt + - src/client/java/**/pom.xml + - src/control/go.mod + - src/control/vendor/** + - utils/build.config + - utils/rpms/** + - utils/scripts/install-*.sh + - utils/trivy/** # Declare default permissions as nothing. permissions: {} @@ -39,7 +49,7 @@ jobs: cp utils/trivy/.trivyignore report/trivyignore.txt - name: Upload the report to the GitHub artifact store - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: path: report/* name: trivy-report-daos @@ -58,7 +68,7 @@ jobs: trivy-config: 'utils/trivy/trivy.yaml' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7 + uses: github/codeql-action/upload-sarif@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: sarif_file: 'trivy-results.sarif' diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index a9c82d02d60..7885b529581 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -5,6 +5,10 @@ on: permissions: {} +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: Run_in_docker: if: github.repository == 'daos-stack/daos' diff --git a/TAG b/TAG index ca1fdbfd4c1..59c033b06fe 100644 --- a/TAG +++ b/TAG @@ -1 +1 @@ -2.6.4-rc2 +2.6.4-rc5 diff --git a/ci/provisioning/post_provision_config_common_functions.sh b/ci/provisioning/post_provision_config_common_functions.sh index 5f17916e934..63d88cad536 100755 --- a/ci/provisioning/post_provision_config_common_functions.sh +++ b/ci/provisioning/post_provision_config_common_functions.sh @@ -411,7 +411,7 @@ post_provision_config_nodes() { # shellcheck disable=SC2001 if [ ${#inst_rpms[@]} -gt 0 ]; then - if ! retry_dnf 360 install "${inst_rpms[@]/%/${DAOS_VERSION:-}}"; then + if ! retry_dnf 360 install "${inst_rpms[@]/%/${DAOS_VERSION:-}}" --setopt=allow_vendor_change=True; then rc=${PIPESTATUS[0]} dump_repos echo "Failed to install packages" diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 47bacfe4ec3..2b271d922ff 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -99,7 +99,7 @@ function mount_nvme_drive { nvme_class="/sys/class/nvme/" function nvme_limit { set +x - if [ ! -d /sys/class/nvme ]; then + if [ ! -d "${nvme_class}" ] || [ -z "$(ls -A "${nvme_class}")" ]; then echo "No NVMe devices found" return fi diff --git a/debian/changelog b/debian/changelog index 219e1430fc7..2ab4c2132b7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,21 @@ +daos (2.6.4-7) unstable; urgency=medium + [ Mohamad Chaarawi ] + * Fix libfabric packaging + + -- Mohamad Chaarawi Fri, 31 Oct 2025 20:00:00 -0500 + +daos (2.6.4-6) unstable; urgency=medium + [ Phillip Henderson ] + * Fourth release candidate for 2.6.4 + + -- Phillip Henderson Thu, 23 Oct 2025 16:23:00 -0500 + +daos (2.6.4-4) unstable; urgency=medium + [ Phillip Henderson ] + * Third release candidate for 2.6.4 + + -- Phillip Henderson Fri, 17 Oct 2025 15:09:00 -0500 + daos (2.6.4-2) unstable; urgency=medium [ Dalton Bohning ] * Second release candidate for 2.6.4 diff --git a/docs/release/release_notes.md b/docs/release/release_notes.md index 7c4d2beb743..0f7cded220f 100644 --- a/docs/release/release_notes.md +++ b/docs/release/release_notes.md @@ -2,6 +2,67 @@ We are pleased to announce the release of DAOS version 2.6. +## DAOS Version 2.6.4 (2025-10-29) + +The DAOS 2.6.4 release includes the daos-2.6.4-7 RPM packages and their prerequisites. +It contains the following updates on top of DAOS 2.6.3: + +* Libfabric has been updated to version 1.20.0. +* Mercury has been upgraded to version 2.4.0-7. +* Argobots has been upgraded to version 1.2-3. +* Libisal has been upgraded to version 2.31.1-7. +* pmemobj has been upgraded to version 2.1.0-6. + +### Bug fixes and improvements + +This release includes fixes and improvements, including: + +* Avoid potential races between rebuild and EC aggregation by serializing these two operations; + this prevents inconsistencies between data and parity. +* Store EC aggregation epoch in RDB so the rebuild system retains the value after system restart. +* Add two new DFUSE options: --dump-handles to dump pool, container, and DFS handles when mounting + dfuse on a client. Other clients can use the dumped information to mount dfuse by using the + --read-handles option. This mechanism uses the local2global and global2local to serialize and + share handles from a root process to other processes. +* Correct how the JCH placement algorithm invokes CRC functions and improve data balancing. +* Fixes for EC object consistency verification so it can detect inconsistencies between data + and parity. +* Merge small or fragmented buffers on the client side to reduce IOV count and mitigate resource + exhaustion during network bulk transfers. +* Enable EC\_xP3 and use EC\_xP3 for automatic object-class selection. +* Fix a bug in the rebuild system related to waking up throttled ULTs, improving rebuild + performance. +* Fix pool/container handle behavior after fork. +* Multiple fixes for DTX background services to avoid orphan DTX entries and inconsistent DTX + participant states caused by intermittent network failures. +* Workaround for SPDK AMD IOMMU support. +* Fix a race condition between fetch and aggregation that could cause degraded fetch to return + an incorrect view of data. +* Reset the DTX base UUID after fork to avoid parent and child processes generating identical + DTX IDs. +* Fix a race between DTX refresh and abort that could leave a transaction in an unexpected state. +* Multiple fixes for EC degraded fetch of single values. +* Fix size queries for array values protected by erasure coding. +* Allow modification of a pool's redundancy factor on existing pools. +* Fix a race between DTX aggregation and container close that could cause an assertion failure. +* Multiple fixes in the rebuild system to prevent rebuild hangs on unstable networks. +* Increase the retry delay for collective RPCs to avoid generating large bursts of network traffic. +* Set the DFS chunk size on container creation (if not set by the caller) to a multiple of the full + EC stripe length to avoid misalignment. This ensures that the default DFS chunk size does not + create partial stripe updates if the IO size is full stripe. This auto setting works for + containers created with rd_fac 1 or 2. EC for rd_fac 3 was newly added and missed in this + optimization so users would have to set the chunk-size manually for that. This will be fixed next + release. +* Remove the separate RPM build for raft. +* Add a new environment variable, D\_QUOTA\_BULKS. When enabled, it limits the number of active + Mercury bulk allocations by postponing those that exceed the limit. +* DMG's exclude, drain, and reintegrate operations accept --ranks or --rank-hosts in ranged + format, allowing the same operation to be applied to those ranks across all pools they belong to. +* Ensure server-side verification does not cause a target to be disabled because of checksum + errors that are unrelated to the device. +* Avoid returning non-retryable errors to clients when an SSD is marked FAULTY, allowing + applications to continue accessing data in degraded mode. + ## DAOS Version 2.6.3 (2025-03-26) The DAOS 2.6.3 release includes the daos-2.6.3-4 RPM packages and its prerequisites. diff --git a/src/bio/bio_internal.h b/src/bio/bio_internal.h index d57ea15201c..04cb6f73613 100644 --- a/src/bio/bio_internal.h +++ b/src/bio/bio_internal.h @@ -385,6 +385,7 @@ struct bio_xs_context { struct bio_xs_blobstore *bxc_xs_blobstores[SMD_DEV_TYPE_MAX]; struct bio_dma_buffer *bxc_dma_buf; unsigned int bxc_self_polling:1; /* for standalone VOS */ + unsigned int bxc_skip_draining : 1; }; /* Per VOS instance I/O context */ diff --git a/src/bio/bio_wal.c b/src/bio/bio_wal.c index c960eb9685d..fa9438393ca 100644 --- a/src/bio/bio_wal.c +++ b/src/bio/bio_wal.c @@ -1310,44 +1310,63 @@ tx_known_committed(struct wal_super_info *si, uint64_t tx_id) return (wal_id_cmp(si, tx_id, si->si_commit_id) <= 0); } +static inline void +dump_tx_hdr(struct wal_super_info *si, struct wal_trans_head *hdr) +{ + D_ERROR("[WAL header] ckp_id:" DF_U64 "/%u, commit_id:" DF_U64 "/%u\n", si->si_ckp_id, + si->si_ckp_blks, si->si_commit_id, si->si_commit_blks); + D_ERROR("[TX header] magic:%x, gen:%u, id:" DF_U64 ", ents:%u, payload:%u\n", hdr->th_magic, + hdr->th_gen, hdr->th_id, hdr->th_tot_ents, hdr->th_tot_payload); +} + static int verify_tx_hdr(struct wal_super_info *si, struct wal_trans_head *hdr, uint64_t tx_id) { bool committed = tx_known_committed(si, tx_id); + int rc = 0; if (hdr->th_magic != WAL_HDR_MAGIC) { D_CDEBUG(committed, DLOG_ERR, DB_IO, "Mismatched WAL head magic, %x != %x\n", hdr->th_magic, WAL_HDR_MAGIC); - return committed ? -DER_INVAL : 1; + rc = committed ? -DER_INVAL : 1; + goto out; } if (hdr->th_gen != si->si_header.wh_gen) { D_CDEBUG(committed, DLOG_ERR, DB_IO, "Mismatched WAL generation, %u != %u\n", hdr->th_gen, si->si_header.wh_gen); - return committed ? -DER_INVAL : 1; + rc = committed ? -DER_INVAL : 1; + goto out; } if (id2seq(hdr->th_id) < id2seq(tx_id)) { D_CDEBUG(committed, DLOG_ERR, DB_IO, "Stale sequence number detected, %u < %u\n", id2seq(hdr->th_id), id2seq(tx_id)); - return committed ? -DER_INVAL : 1; + rc = committed ? -DER_INVAL : 1; + goto out; } else if (id2seq(hdr->th_id) > id2seq(tx_id)) { D_ERROR("Invalid sequence number detected, %u > %u\n", id2seq(hdr->th_id), id2seq(tx_id)); - return -DER_INVAL; + rc = -DER_INVAL; + goto out; } if (hdr->th_id != tx_id) { D_ERROR("Mismatched transaction ID. "DF_U64" != "DF_U64"\n", hdr->th_id, tx_id); - return -DER_INVAL; + rc = -DER_INVAL; + goto out; } if (hdr->th_tot_ents == 0) { D_ERROR("Invalid entry number\n"); - return -DER_INVAL; + rc = -DER_INVAL; + goto out; } +out: + if (rc < 0) + dump_tx_hdr(si, hdr); - return 0; + return rc; } static int @@ -1907,7 +1926,7 @@ bio_wal_replay(struct bio_meta_context *mc, struct bio_wal_rp_stats *wrs, wrs->wrs_tx_cnt = total_tx; } } else { - D_ERROR("WAL replay failed, "DF_RC"\n", DP_RC(rc)); + DL_ERROR(rc, "WAL replay failed, nr_replayed:%u", nr_replayed); } D_FREE(dbuf); @@ -1953,19 +1972,21 @@ bio_wal_checkpoint(struct bio_meta_context *mc, uint64_t tx_id, uint64_t *purged unmap_start = id2off(wal_next_id(si, si->si_ckp_id, si->si_ckp_blks)); unmap_end = id2off(wal_next_id(si, tx_id, blk_desc.bd_blks)); - /* Unmap the checkpointed regions */ - rc = unmap_wal(mc, unmap_start, unmap_end, purged_blks); - if (rc) /* Flush the WAL header anyway */ - D_ERROR("Unmap checkpointed region failed. "DF_RC"\n", DP_RC(rc)); - si->si_ckp_id = tx_id; si->si_ckp_blks = blk_desc.bd_blks; - wakeup_reserve_waiters(si, false); - /* Flush the WAL header */ rc = bio_wal_flush_header(mc); - if (rc) - D_ERROR("Flush WAL header failed. "DF_RC"\n", DP_RC(rc)); + if (rc) { + DL_ERROR(rc, "Flush WAL header failed."); + goto out; + } + + /* Unmap the checkpointed region */ + rc = unmap_wal(mc, unmap_start, unmap_end, purged_blks); + if (rc) /* Wakeup waiters even if unmap failed */ + DL_ERROR(rc, "Unmap checkpointed region failed."); + + wakeup_reserve_waiters(si, false); out: D_FREE(buf); return rc; diff --git a/src/bio/bio_xstream.c b/src/bio/bio_xstream.c index 0c2e5469404..9177372b498 100644 --- a/src/bio/bio_xstream.c +++ b/src/bio/bio_xstream.c @@ -2,7 +2,6 @@ * (C) Copyright 2018-2024 Intel Corporation. * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * - * * SPDX-License-Identifier: BSD-2-Clause-Patent */ #define D_LOGFAC DD_FAC(bio) @@ -1507,6 +1506,8 @@ bio_xsctxt_free(struct bio_xs_context *ctxt) rc = xs_poll_completion(ctxt, &cp_arg.cca_inflights, bio_spdk_subsys_timeout); DL_CDEBUG(rc == 0, DB_MGMT, DLOG_ERR, rc, "SPDK subsystems finalized"); + if (rc != 0) + ctxt->bxc_skip_draining = 1; nvme_glb.bd_init_thread = NULL; @@ -1518,18 +1519,21 @@ bio_xsctxt_free(struct bio_xs_context *ctxt) ABT_mutex_unlock(nvme_glb.bd_mutex); if (ctxt->bxc_thread != NULL) { - D_DEBUG(DB_MGMT, "Finalizing SPDK thread, tgt_id:%d", - ctxt->bxc_tgt_id); + D_DEBUG(DB_MGMT, "Finalizing SPDK thread, tgt_id:%d, skip_draining:%u", + ctxt->bxc_tgt_id, ctxt->bxc_skip_draining); - /* Don't drain events if spdk_subsystem_fini() timeout */ - while (rc == 0 && !spdk_thread_is_idle(ctxt->bxc_thread)) + /* + * Don't drain events if we are asked to skip this (usually + * due to an earlier error). + */ + while (!ctxt->bxc_skip_draining && !spdk_thread_is_idle(ctxt->bxc_thread)) spdk_thread_poll(ctxt->bxc_thread, 0, 0); D_DEBUG(DB_MGMT, "SPDK thread finalized, tgt_id:%d", ctxt->bxc_tgt_id); spdk_thread_exit(ctxt->bxc_thread); - while (rc == 0 && !spdk_thread_is_exited(ctxt->bxc_thread)) + while (!ctxt->bxc_skip_draining && !spdk_thread_is_exited(ctxt->bxc_thread)) spdk_thread_poll(ctxt->bxc_thread, 0, 0); spdk_thread_destroy(ctxt->bxc_thread); ctxt->bxc_thread = NULL; @@ -1615,7 +1619,12 @@ bio_xsctxt_alloc(struct bio_xs_context **pctxt, int tgt_id, bool self_polling) if (cp_arg.cca_rc != 0) { rc = cp_arg.cca_rc; - D_ERROR("failed to init bdevs, rc:%d\n", rc); + DL_ERROR(rc, "failed to init bdevs"); + /* + * We're afraid that draining the thread might never + * complete (DAOS-17442). + */ + ctxt->bxc_skip_draining = 1; goto out; } diff --git a/src/bio/smd/smd_device.c b/src/bio/smd/smd_device.c index 5b44b38c984..cc14733b255 100644 --- a/src/bio/smd/smd_device.c +++ b/src/bio/smd/smd_device.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2018-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -218,7 +219,7 @@ smd_dev_get_by_tgt(uint32_t tgt_id, enum smd_dev_type st, struct smd_dev_info ** } static int -smd_dev_list_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) +smd_dev_list_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts) { struct smd_trav_data *td = args; struct smd_dev_info *info; diff --git a/src/bio/smd/smd_pool.c b/src/bio/smd/smd_pool.c index 94312e960e1..03bca402ddf 100644 --- a/src/bio/smd/smd_pool.c +++ b/src/bio/smd/smd_pool.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2018-2025 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -336,7 +337,7 @@ smd_rdb_get_blob_sz(uuid_t pool_id, uint64_t *blob_sz) } static int -smd_pool_list_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) +smd_pool_list_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts) { struct smd_trav_data *td = args; struct smd_pool_info *info; diff --git a/src/bio/smd/tests/smd_ut.c b/src/bio/smd/tests/smd_ut.c index 71d2028d194..01172853d43 100644 --- a/src/bio/smd/tests/smd_ut.c +++ b/src/bio/smd/tests/smd_ut.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2018-2025 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -159,7 +160,7 @@ db_traverse(struct sys_db *db, char *table, sys_db_trav_cb_t cb, void *args) d_iov_t key; d_iov_set(&key, chain->uc_key, chain->uc_key_size); - cb(db, table, &key, args); + cb(db, table, &key, args, NULL); } return 0; } diff --git a/src/chk/chk_common.c b/src/chk/chk_common.c index 39821ec6328..03a8db41487 100644 --- a/src/chk/chk_common.c +++ b/src/chk/chk_common.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2022-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -332,20 +333,15 @@ chk_ranks_dump(uint32_t rank_nr, d_rank_t *ranks) D_INFO("Ranks List:\n"); while (rank_nr >= 8) { - D_INFO("%8u %8u %8u %8u %8u %8u %8u %8u\n", - ranks[0], ranks[1], ranks[2], ranks[3], + D_INFO("%8u%8u%8u%8u%8u%8u%8u%8u\n", ranks[0], ranks[1], ranks[2], ranks[3], ranks[4], ranks[5], ranks[6], ranks[7]); rank_nr -= 8; ranks += 8; } if (rank_nr > 0) { - rc = snprintf(ptr, 79, "%8u", ranks[0]); - D_ASSERT(rc > 0); - ptr += rc; - - for (i = 1; i < rank_nr; i++) { - rc = snprintf(ptr, 79 - 8 * i, " %8u", ranks[i]); + for (i = 0; i < rank_nr; i++) { + rc = snprintf(ptr, 79 - 8 * i, "%8u", ranks[i]); D_ASSERT(rc > 0); ptr += rc; } @@ -529,7 +525,7 @@ chk_pool_stop_all(struct chk_instance *ins, uint32_t status, int *ret) } int -chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) +chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts) { struct chk_traverse_pools_args *ctpa = args; char *uuid_str = key->iov_buf; @@ -559,7 +555,7 @@ chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) } int -chk_pools_cleanup_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) +chk_pools_cleanup_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts) { char *uuid_str = key->iov_buf; struct chk_bookmark cbk; @@ -569,8 +565,11 @@ chk_pools_cleanup_cb(struct sys_db *db, char *table, d_iov_t *key, void *args) D_GOTO(out, rc = 0); rc = chk_bk_fetch_pool(&cbk, uuid_str); - if (rc == 0) + if (rc == 0) { rc = chk_bk_delete_pool(uuid_str); + if (rc == 0) + *acts |= VOS_ITER_CB_DELETE; + } out: return rc == -DER_NONEXIST ? 0 : rc; @@ -610,6 +609,8 @@ chk_pools_load_list(struct chk_instance *ins, uint64_t gen, uint32_t flags, struct chk_bookmark cbk; char uuid_str[DAOS_UUID_STR_SIZE]; d_rank_t myrank = dss_self_rank(); + d_iov_t kiov; + d_iov_t riov; int i; int rc = 0; @@ -659,6 +660,16 @@ chk_pools_load_list(struct chk_instance *ins, uint64_t gen, uint32_t flags, if (rc == 0 && cbk.cb_phase == CHK__CHECK_SCAN_PHASE__CSP_DONE && ins->ci_is_leader) continue; + /* There may be repeated pool(s) in the list, filter out. */ + d_iov_set(&kiov, pools[i], sizeof(uuid_t)); + d_iov_set(&riov, NULL, 0); + rc = dbtree_lookup(ins->ci_pool_hdl, &kiov, &riov); + if (unlikely(rc == 0)) + continue; + + if (unlikely(rc != -DER_NONEXIST)) + break; + /* * Here, we only update the pool bookmark in DRAM, the caller will store the update * persistently sometime later. @@ -677,7 +688,7 @@ chk_pools_load_list(struct chk_instance *ins, uint64_t gen, uint32_t flags, } int -chk_pools_load_from_db(struct sys_db *db, char *table, d_iov_t *key, void *args) +chk_pools_load_from_db(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts) { struct chk_traverse_pools_args *ctpa = args; struct chk_instance *ins = ctpa->ctpa_ins; diff --git a/src/chk/chk_internal.h b/src/chk/chk_internal.h index 9ab16b060b3..dc221ebf312 100644 --- a/src/chk/chk_internal.h +++ b/src/chk/chk_internal.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2022-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -688,6 +689,7 @@ extern btr_ops_t chk_pending_ops; extern btr_ops_t chk_rank_ops; extern btr_ops_t chk_cont_ops; +/* clang-format off */ /* chk_common.c */ void chk_ranks_dump(uint32_t rank_nr, d_rank_t *ranks); @@ -703,16 +705,17 @@ void chk_pool_stop_one(struct chk_instance *ins, uuid_t uuid, uint32_t status, u void chk_pool_stop_all(struct chk_instance *ins, uint32_t status, int *ret); -int chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args); +int chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts); -int chk_pools_cleanup_cb(struct sys_db *db, char *table, d_iov_t *key, void *args); +int chk_pools_cleanup_cb(struct sys_db *db, char *table, d_iov_t *key, void *args, unsigned *acts); int chk_pool_start_one(struct chk_instance *ins, uuid_t uuid, uint64_t gen); int chk_pools_load_list(struct chk_instance *ins, uint64_t gen, uint32_t flags, int pool_nr, uuid_t pools[], uint32_t *phase); -int chk_pools_load_from_db(struct sys_db *db, char *table, d_iov_t *key, void *args); +int chk_pools_load_from_db(struct sys_db *db, char *table, d_iov_t *key, void *args, + unsigned *acts); int chk_pools_update_bk(struct chk_instance *ins, uint32_t phase); @@ -883,6 +886,7 @@ int chk_traverse_pools(sys_db_trav_cb_t cb, void *args); void chk_vos_init(void); void chk_vos_fini(void); +/* clang-format on */ static inline bool chk_is_ins_reset(struct chk_instance *ins, uint32_t flags) diff --git a/src/client/dfs/cont.c b/src/client/dfs/cont.c index 7eb910d4283..b3c133a8580 100644 --- a/src/client/dfs/cont.c +++ b/src/client/dfs/cont.c @@ -27,8 +27,8 @@ suggest_dfs_cs(daos_handle_t poh, daos_prop_t *prop, uint64_t rf, daos_oclass_id daos_size_t base; int rc; - /** No EC above RF 2, use default CS */ - if (rf > 2) { + /** No EC above RF 3, use default CS */ + if (rf > 3) { *cs = DFS_DEFAULT_CHUNK_SIZE; return 0; } @@ -126,9 +126,13 @@ dfs_cont_create(daos_handle_t poh, uuid_t *cuuid, dfs_attr_t *attr, daos_handle_ struct daos_prop_co_roots roots; int rc, rc2; struct daos_prop_entry *dpe; + struct daos_prop_entry *roots_entry = NULL; + struct daos_prop_entry *layout_entry = NULL; struct timespec now; uint32_t cid_tf; uint32_t pa_domain; + uint32_t prop_nr = 0; + uint32_t props_to_add = 0; int cont_tf; if (cuuid == NULL) @@ -138,10 +142,23 @@ dfs_cont_create(daos_handle_t poh, uuid_t *cuuid, dfs_attr_t *attr, daos_handle_ return EINVAL; } - if (attr != NULL && attr->da_props != NULL) - prop = daos_prop_alloc(attr->da_props->dpp_nr + 2); - else - prop = daos_prop_alloc(2); + if (attr != NULL && attr->da_props != NULL) { + prop_nr = attr->da_props->dpp_nr; + + roots_entry = daos_prop_entry_get(attr->da_props, DAOS_PROP_CO_ROOTS); + if (roots_entry == NULL) + props_to_add++; + + layout_entry = daos_prop_entry_get(attr->da_props, DAOS_PROP_CO_LAYOUT_TYPE); + if (layout_entry == NULL) + props_to_add++; + } else { + /* roots and layout are still needed */ + props_to_add = 2; + } + prop_nr += props_to_add; + + prop = daos_prop_alloc(prop_nr); if (prop == NULL) return ENOMEM; @@ -267,13 +284,29 @@ dfs_cont_create(daos_handle_t poh, uuid_t *cuuid, dfs_attr_t *attr, daos_handle_ /* store SB & root OIDs as container property */ roots.cr_oids[2] = roots.cr_oids[3] = DAOS_OBJ_NIL; - prop->dpp_entries[prop->dpp_nr - 2].dpe_type = DAOS_PROP_CO_ROOTS; - rc = daos_prop_entry_set_ptr(&prop->dpp_entries[prop->dpp_nr - 2], &roots, sizeof(roots)); + + if (roots_entry == NULL) { + /* need to add roots prop to list */ + roots_entry = &prop->dpp_entries[prop->dpp_nr - props_to_add]; + roots_entry->dpe_type = DAOS_PROP_CO_ROOTS; + props_to_add--; + } + /* if an existing roots prop was passed in, it will be overwritten */ + rc = daos_prop_entry_set_ptr(roots_entry, &roots, sizeof(roots)); if (rc) D_GOTO(err_prop, rc = daos_der2errno(rc)); - prop->dpp_entries[prop->dpp_nr - 1].dpe_type = DAOS_PROP_CO_LAYOUT_TYPE; - prop->dpp_entries[prop->dpp_nr - 1].dpe_val = DAOS_PROP_CO_LAYOUT_POSIX; + if (layout_entry == NULL) { + /* need to add layout prop to list */ + layout_entry = &prop->dpp_entries[prop->dpp_nr - props_to_add]; + layout_entry->dpe_type = DAOS_PROP_CO_LAYOUT_TYPE; + props_to_add--; + } + + /* layout passed in will be overwritten */ + layout_entry->dpe_val = DAOS_PROP_CO_LAYOUT_POSIX; + + D_ASSERT(props_to_add == 0); rc = daos_cont_create(poh, cuuid, prop, NULL); if (rc) { diff --git a/src/client/pydaos/raw/daos_cref.py b/src/client/pydaos/raw/daos_cref.py index ceb2a5b6885..3878912b698 100644 --- a/src/client/pydaos/raw/daos_cref.py +++ b/src/client/pydaos/raw/daos_cref.py @@ -1,5 +1,6 @@ """ (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -51,7 +52,8 @@ class RebuildStatus(ctypes.Structure): ("rs_seconds", ctypes.c_uint32), ("rs_errno", ctypes.c_uint32), ("rs_state", ctypes.c_uint32), - ("rs_padding32", ctypes.c_uint32), + ("rs_max_supported_layout_ver", ctypes.c_uint16), + ("rs_padding16", ctypes.c_uint16), ("rs_fail_rank", ctypes.c_uint32), ("rs_toberb_obj_nr", ctypes.c_uint64), ("rs_obj_nr", ctypes.c_uint64), diff --git a/src/common/prop.c b/src/common/prop.c index 5b927120c1b..a29d33948de 100644 --- a/src/common/prop.c +++ b/src/common/prop.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -135,7 +136,7 @@ daos_prop_merge2(daos_prop_t *old_prop, daos_prop_t *new_prop, daos_prop_t **out int rc; uint32_t result_nr; uint32_t i, result_i; - struct daos_prop_entry *entry; + struct daos_prop_entry *entry; if (old_prop == NULL || new_prop == NULL) { D_ERROR("NULL input\n"); @@ -143,8 +144,8 @@ daos_prop_merge2(daos_prop_t *old_prop, daos_prop_t *new_prop, daos_prop_t **out } /* - * We might override some values in the old prop. Need to account for that in the final prop - * count. + * We might override some values in the old prop. Need to account for that in the + * final prop count. */ result_nr = old_prop->dpp_nr; for (i = 0; i < new_prop->dpp_nr; i++) { @@ -162,8 +163,8 @@ daos_prop_merge2(daos_prop_t *old_prop, daos_prop_t *new_prop, daos_prop_t **out result_i = 0; for (i = 0; i < old_prop->dpp_nr; i++, result_i++) { - rc = daos_prop_entry_copy(&old_prop->dpp_entries[i], - &result->dpp_entries[result_i]); + rc = + daos_prop_entry_copy(&old_prop->dpp_entries[i], &result->dpp_entries[result_i]); if (rc != 0) goto err; } @@ -244,10 +245,14 @@ daos_prop_owner_group_valid(d_string_t owner) bool daos_prop_valid(daos_prop_t *prop, bool pool, bool input) { - uint32_t type; - uint64_t val; - struct daos_acl *acl_ptr; - int i; + uint32_t type; + uint64_t val; + struct daos_acl *acl_ptr; + int i; + const uint32_t seen_nr = pool ? DAOS_PROP_PO_NUM : DAOS_PROP_CO_NUM; + bool seen[seen_nr]; + + memset(seen, 0, sizeof(seen)); if (prop == NULL) { D_ERROR("NULL properties\n"); @@ -270,6 +275,8 @@ daos_prop_valid(daos_prop_t *prop, bool pool, bool input) for (i = 0; i < prop->dpp_nr; i++) { struct daos_co_status co_status; int rc; + int type_min; + int type_idx; type = prop->dpp_entries[i].dpe_type; if (pool) { @@ -278,6 +285,7 @@ daos_prop_valid(daos_prop_t *prop, bool pool, bool input) D_ERROR("invalid type %d for pool.\n", type); return false; } + type_min = DAOS_PROP_PO_MIN; } else { if (type <= DAOS_PROP_CO_MIN || type >= DAOS_PROP_CO_MAX) { @@ -285,7 +293,17 @@ daos_prop_valid(daos_prop_t *prop, bool pool, bool input) type); return false; } + type_min = DAOS_PROP_CO_MIN; } + + /* detect duplicate types */ + type_idx = type - type_min - 1; + if (seen[type_idx]) { + D_ERROR("type %d (idx=%d) appears more than once\n", type, type_idx); + return false; + } + seen[type_idx] = true; + /* for output parameter need not check entry value */ if (!input) continue; diff --git a/src/common/tests/prop_tests.c b/src/common/tests/prop_tests.c index 2f33f568da3..0c2c51f5c72 100644 --- a/src/common/tests/prop_tests.c +++ b/src/common/tests/prop_tests.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2020-2022 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -374,18 +375,153 @@ test_daos_prop_from_str(void **state) daos_prop_free(prop); } +static void +test_daos_prop_valid_null(void **state) +{ + assert_false(daos_prop_valid(NULL, false, false)); +} + +static void +test_daos_prop_valid_too_many(void **state) +{ + daos_prop_t *prop = daos_prop_alloc(0); + uint32_t nr = prop->dpp_nr; + + prop->dpp_nr = DAOS_PROP_ENTRIES_MAX_NR + 1; + + assert_false(daos_prop_valid(prop, true, false)); + + prop->dpp_nr = nr; + daos_prop_free(prop); +} + +static void +test_daos_prop_valid_inconsistent_nr_entries(void **state) +{ + const int nr = 2; + daos_prop_t *prop = daos_prop_alloc(nr); + struct daos_prop_entry *entries = prop->dpp_entries; + + D_PRINT("zero entries, but non-null array\n"); + prop->dpp_nr = 0; + assert_false(daos_prop_valid(prop, false, false)); + + D_PRINT("non-zero entries, but null array\n"); + prop->dpp_nr = nr; + prop->dpp_entries = NULL; + assert_false(daos_prop_valid(prop, false, false)); + + /* cleanup */ + prop->dpp_entries = entries; + daos_prop_free(prop); +} + +static daos_prop_t * +get_prop_with_duplicates(int total_nr, bool pool) +{ + daos_prop_t *prop = daos_prop_alloc(total_nr); + int i; + uint32_t prop_min; + + assert_true(total_nr >= 2); + + if (pool) + prop_min = DAOS_PROP_PO_MIN; + else + prop_min = DAOS_PROP_CO_MIN; + + for (i = 0; i < total_nr; i++) + prop->dpp_entries[i].dpe_type = prop_min + i + 1; + + /* last one is a duplicate type */ + prop->dpp_entries[total_nr - 1].dpe_type = prop->dpp_entries[total_nr - 2].dpe_type; + + return prop; +} + +static void +check_daos_prop_valid_duplicate_types(bool pool) +{ + daos_prop_t *prop; + + prop = get_prop_with_duplicates(5, pool); + + assert_false(daos_prop_valid(prop, pool, false)); + + daos_prop_free(prop); +} + +static void +test_daos_prop_valid_duplicate_types(void **state) +{ + D_PRINT("check pool prop with duplicate types\n"); + check_daos_prop_valid_duplicate_types(true); + + D_PRINT("check container prop with duplicate types\n"); + check_daos_prop_valid_duplicate_types(false); +} + +static void +test_daos_prop_valid_pool_success_no_val_check(void **state) +{ + const int nr = DAOS_PROP_PO_MAX - DAOS_PROP_PO_MIN - 1; + daos_prop_t *prop = daos_prop_alloc(nr); + int i; + + for (i = 0; i < nr; i++) + prop->dpp_entries[i].dpe_type = DAOS_PROP_PO_MIN + i + 1; + + assert_true(daos_prop_valid(prop, true, false)); + + daos_prop_free(prop); +} + +static void +test_daos_prop_valid_cont_success_no_val_check(void **state) +{ + const int nr = DAOS_PROP_CO_MAX - DAOS_PROP_CO_MIN - 1; + daos_prop_t *prop = daos_prop_alloc(nr); + int i; + + for (i = 0; i < nr; i++) + prop->dpp_entries[i].dpe_type = DAOS_PROP_CO_MIN + i + 1; + + assert_true(daos_prop_valid(prop, false, false)); + + daos_prop_free(prop); +} + +static int +suite_setup(void **state) +{ + return d_log_init(); +} + +static int +suite_teardown(void **state) +{ + d_log_fini(); + return 0; +} + int main(void) { const struct CMUnitTest tests[] = { - cmocka_unit_test(test_daos_prop_merge_null), - cmocka_unit_test(test_daos_prop_merge_empty), - cmocka_unit_test(test_daos_prop_merge_add_only), - cmocka_unit_test(test_daos_prop_merge_total_update), - cmocka_unit_test(test_daos_prop_merge_subset_update), - cmocka_unit_test(test_daos_prop_merge_add_and_update), - cmocka_unit_test(test_daos_prop_from_str), + cmocka_unit_test(test_daos_prop_merge_null), + cmocka_unit_test(test_daos_prop_merge_empty), + cmocka_unit_test(test_daos_prop_merge_add_only), + cmocka_unit_test(test_daos_prop_merge_total_update), + cmocka_unit_test(test_daos_prop_merge_subset_update), + cmocka_unit_test(test_daos_prop_merge_add_and_update), + cmocka_unit_test(test_daos_prop_from_str), + cmocka_unit_test(test_daos_prop_valid_null), + cmocka_unit_test(test_daos_prop_valid_too_many), + cmocka_unit_test(test_daos_prop_valid_inconsistent_nr_entries), + cmocka_unit_test(test_daos_prop_valid_duplicate_types), + cmocka_unit_test(test_daos_prop_valid_pool_success_no_val_check), + cmocka_unit_test(test_daos_prop_valid_cont_success_no_val_check), }; - return cmocka_run_group_tests_name("common_prop", tests, NULL, NULL); + return cmocka_run_group_tests_name("common_prop", tests, suite_setup, suite_teardown); } diff --git a/src/container/cli.c b/src/container/cli.c index 18b6a75a590..70287280cd0 100644 --- a/src/container/cli.c +++ b/src/container/cli.c @@ -178,8 +178,8 @@ daos_prop_has_entry(daos_prop_t *prop, uint32_t entry_type) * The newly allocated prop is expected to be freed by the cont create callback. */ static int -dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, - daos_prop_t *prop_in) +dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, daos_prop_t *prop_in, + uint32_t co_ver) { char *owner = NULL; char *owner_grp = NULL; @@ -189,8 +189,7 @@ dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, uint32_t entries; int rc = 0; uid_t uid = geteuid(); - gid_t gid = getegid(); - uint32_t co_ver = 0; + gid_t gid = getegid(); entries = (prop_in == NULL) ? 0 : prop_in->dpp_nr; @@ -226,8 +225,10 @@ dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, entries++; } if (!daos_prop_has_entry(prop_in, DAOS_PROP_CO_OBJ_VERSION)) { - co_ver = DAOS_POOL_OBJ_VERSION_2; - entries++; + if (co_ver >= DAOS_POOL_OBJ_VERSION_2) + entries++; + else /* don't specify version if server is old */ + co_ver = 0; } if (!daos_prop_has_entry(prop_in, DAOS_PROP_CO_ROOTS)) { @@ -351,7 +352,8 @@ dc_cont_create(tse_task_t *task) D_GOTO(err_task, rc = -DER_NO_HDL); /** duplicate properties and add extra ones (e.g. ownership) */ - rc = dup_cont_create_props(args->poh, &rpc_prop, args->prop); + rc = dup_cont_create_props(args->poh, &rpc_prop, args->prop, + pool->dp_max_supported_layout_ver); if (rc != 0) D_GOTO(err_pool, rc); @@ -1009,7 +1011,9 @@ dc_cont_open(tse_task_t *task) if (tpriv->cont == NULL) D_GOTO(err_pool, rc = -DER_NOMEM); uuid_generate(tpriv->cont->dc_cont_hdl); - tpriv->cont->dc_capas = args->flags | DAOS_COO_NEW_LAYOUT; + tpriv->cont->dc_capas = args->flags; + if (pool->dp_max_supported_layout_ver >= DAOS_POOL_OBJ_VERSION_2) + tpriv->cont->dc_capas |= DAOS_COO_NEW_LAYOUT; } D_DEBUG(DB_MD, DF_UUID ":%s: opening: hdl=" DF_UUIDF " flags=%x\n", DP_UUID(pool->dp_pool), diff --git a/src/container/srv_container.c b/src/container/srv_container.c index 0971752888e..b2319271fb6 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -101,10 +101,16 @@ cont_svc_init(struct cont_svc *svc, const uuid_t pool_uuid, uint64_t id, D_GOTO(err, rc = dss_abterr2der(rc)); } + rc = ABT_mutex_create(&svc->cs_ec_agg_mutex); + if (rc != ABT_SUCCESS) { + D_ERROR("failed to create cs_ec_agg_mutex: %d\n", rc); + D_GOTO(err_rwlock, rc = dss_abterr2der(rc)); + } + /* cs_root */ rc = rdb_path_init(&svc->cs_root); if (rc != 0) - goto err_lock; + goto err_mutex; rc = rdb_path_push(&svc->cs_root, &rdb_path_root_key); if (rc != 0) goto err_root; @@ -143,7 +149,9 @@ cont_svc_init(struct cont_svc *svc, const uuid_t pool_uuid, uint64_t id, rdb_path_fini(&svc->cs_uuids); err_root: rdb_path_fini(&svc->cs_root); -err_lock: +err_mutex: + ABT_mutex_free(&svc->cs_ec_agg_mutex); +err_rwlock: ABT_rwlock_free(&svc->cs_lock); err: return rc; @@ -157,6 +165,7 @@ cont_svc_fini(struct cont_svc *svc) rdb_path_fini(&svc->cs_uuids); rdb_path_fini(&svc->cs_root); ABT_rwlock_free(&svc->cs_lock); + ABT_mutex_free(&svc->cs_ec_agg_mutex); } int @@ -705,7 +714,6 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl, D_ERROR("container global %u version could be not set\n", entry->dpe_type); return -DER_INVAL; case DAOS_PROP_CO_OBJ_VERSION: - /* this is a walkaround for 2.6 only */ entry_def->dpe_val = entry->dpe_val; break; default: @@ -760,10 +768,14 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl, if (entry_def) entry_def->dpe_val = pool_hdl->sph_global_ver; - /* inherit object version from pool*/ + /* + * New container creation by clients will specify the object version. + * If not specified (dpe_val == 0), it indicates a client from before + * DAOS 2.6.4, so use VERSION 1 for backward compatibility. + */ entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_OBJ_VERSION); if (entry_def && entry_def->dpe_val == 0) - entry_def->dpe_val = pool_hdl->sph_obj_ver; + entry_def->dpe_val = DAOS_POOL_OBJ_VERSION_1; /* for new container set HEALTHY status with current pm ver */ entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_STATUS); @@ -1986,6 +1998,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) return; } + ABT_mutex_lock(svc->cs_ec_agg_mutex); d_list_for_each_entry_safe(ec_agg, tmp, &svc->cs_ec_agg_list, ea_list) { if (ec_agg->ea_deleted) { d_list_del(&ec_agg->ea_list); @@ -2068,6 +2081,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) } ec_agg->ea_current_eph = min_eph; } + ABT_mutex_unlock(svc->cs_ec_agg_mutex); map_ranks_fini(&fail_ranks); } diff --git a/src/container/srv_epoch.c b/src/container/srv_epoch.c index a504dd32145..5d5e01e4469 100644 --- a/src/container/srv_epoch.c +++ b/src/container/srv_epoch.c @@ -220,9 +220,8 @@ snap_oit_create(struct rdb_tx *tx, struct cont *cont, uuid_t coh_uuid, out = crt_reply_get(rpc); rc = out->tso_rc; if (rc != 0) { - D_ERROR(DF_CONT": snapshot notify failed on %d targets\n", - DP_CONT(cont->c_svc->cs_pool_uuid, cont->c_uuid), rc); - rc = -DER_IO; + D_ERROR(DF_CONT " snapshot notify failed: " DF_RC "\n", + DP_CONT(cont->c_svc->cs_pool_uuid, cont->c_uuid), DP_RC(rc)); goto out_rpc; } *epoch = in->tsi_epoch; diff --git a/src/container/srv_internal.h b/src/container/srv_internal.h index bb257a73212..aaae6d772bf 100644 --- a/src/container/srv_internal.h +++ b/src/container/srv_internal.h @@ -93,6 +93,7 @@ struct cont_svc { /* Manage the EC aggregation epoch */ struct sched_request *cs_ec_leader_ephs_req; d_list_t cs_ec_agg_list; /* link cont_ec_agg */ + ABT_mutex cs_ec_agg_mutex; /* protect cs_ec_agg_list */ }; /* Container descriptor */ diff --git a/src/container/srv_target.c b/src/container/srv_target.c index f55cdf36eeb..20ae80246b4 100644 --- a/src/container/srv_target.c +++ b/src/container/srv_target.c @@ -62,7 +62,7 @@ agg_rate_ctl(void *arg) * XXX temporary workaround: EC aggregation needs to be paused during rebuilding * to avoid the race between EC rebuild and EC aggregation. **/ - if (pool->sp_rebuilding && cont->sc_ec_agg_active && !param->ap_vos_agg) + if (ds_pool_is_rebuilding(pool) && cont->sc_ec_agg_active && !param->ap_vos_agg) return -1; /* When system is idle or under space pressure, let aggregation run in tight mode */ @@ -182,10 +182,10 @@ cont_aggregate_runnable(struct ds_cont_child *cont, struct sched_request *req, return false; } - if (pool->sp_rebuilding && !vos_agg) { - D_DEBUG(DB_EPC, DF_CONT": skip EC aggregation during rebuild %d.\n", - DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), - pool->sp_rebuilding); + if (ds_pool_is_rebuilding(pool) && !vos_agg) { + D_DEBUG(DB_EPC, DF_CONT ": skip EC aggregation during rebuild %d, %d.\n", + DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), pool->sp_rebuilding, + pool->sp_rebuild_scan); return false; } @@ -500,7 +500,8 @@ cont_aggregate_interval(struct ds_cont_child *cont, cont_aggregate_cb_t cb, /* sleep 18 seconds for EC aggregation ULT if the pool is in rebuilding, * if no space pressure. */ - if (cont->sc_pool->spc_pool->sp_rebuilding && !param->ap_vos_agg && msecs != 200) + if (ds_pool_is_rebuilding(cont->sc_pool->spc_pool) && !param->ap_vos_agg && + msecs != 200) msecs = 18000; sched_req_sleep(req, msecs); @@ -918,6 +919,57 @@ ds_cont_child_reset_ec_agg_eph_all(struct ds_pool_child *pool_child) cont_child->sc_ec_agg_eph = cont_child->sc_ec_agg_eph_boundary; } +#define WAIT_EC_PAUSE_MAX 600 + +void +ds_cont_child_wait_ec_agg_pause(struct ds_pool_child *pool_child, int wait_timeout) +{ + uint64_t start_time = daos_wallclock_secs(); + int wait_intv = 10; + int waited = 0; + + D_DEBUG(DB_MD, DF_UUID "[%d]: wait for pausing EC aggregation\n", + DP_UUID(pool_child->spc_uuid), dss_get_module_info()->dmi_tgt_id); + + if (wait_timeout == 0 || wait_timeout > WAIT_EC_PAUSE_MAX) + wait_timeout = WAIT_EC_PAUSE_MAX; /* 10 minutes by default */ + + while (1) { + struct ds_cont_child *coc; + bool paused = true; + + /* Wait for pausing aggregation + * XXX: There is no global barrier so we always wait for at least 10 seconds to + * lower the chance that remote targets are still running EC aggregation. + */ + if (wait_intv > wait_timeout - waited) + wait_intv = wait_timeout - waited; + + dss_sleep(wait_intv * 1000); + d_list_for_each_entry(coc, &pool_child->spc_cont_list, sc_link) { + if (ds_cont_child_ec_aggregating(coc)) { + /* Aggregation is active on this container */ + paused = false; + break; + } + } + if (paused) + return; + + waited = daos_wallclock_secs() - start_time; + if (waited >= wait_timeout) { + D_WARN("can't pause EC aggregation after %d seconds\n", waited); + return; /* XXX what can I do? */ + } + + if (waited % 60 == 0) { + D_WARN(DF_UUID "[%d]: waited %d secs for EC aggregation to pause\n", + DP_UUID(pool_child->spc_uuid), dss_get_module_info()->dmi_tgt_id, + waited); + } + } +} + static int cont_child_start(struct ds_pool_child *pool_child, const uuid_t co_uuid, bool *started, struct ds_cont_child **cont_out) @@ -2154,7 +2206,8 @@ ds_cont_tgt_snapshot_notify_aggregator(crt_rpc_t *source, crt_rpc_t *result, out_source = crt_reply_get(source); out_result = crt_reply_get(result); - out_result->tso_rc += out_source->tso_rc; + if (out_result->tso_rc >= 0 && out_source->tso_rc < 0) + out_result->tso_rc = out_source->tso_rc; return 0; } diff --git a/src/control/cmd/dmg/pool.go b/src/control/cmd/dmg/pool.go index a9cef94f5be..37e6bc9d24b 100644 --- a/src/control/cmd/dmg/pool.go +++ b/src/control/cmd/dmg/pool.go @@ -307,7 +307,7 @@ func (cmd *poolCreateCmd) Execute(args []string) error { User: cmd.UserName.String(), UserGroup: cmd.GroupName.String(), NumSvcReps: cmd.NumSvcReps, - Properties: cmd.Properties.ToSet, + Properties: cmd.Properties.ToSet.Slice(), Ranks: cmd.RankList.Ranks(), } @@ -767,7 +767,7 @@ type poolSetPropCmd struct { poolCmd Args struct { - Props PoolSetPropsFlag `positional-arg-name:"" required:"1"` + Props PoolSetPropsFlag `positional-arg-name:"" required:"1"` } `positional-args:"yes"` } @@ -787,7 +787,7 @@ func (cmd *poolSetPropCmd) Execute(_ []string) error { req := &control.PoolSetPropReq{ ID: cmd.PoolID().String(), - Properties: cmd.Args.Props.ToSet, + Properties: cmd.Args.Props.ToSet.Slice(), } err := control.PoolSetProp(cmd.MustLogCtx(), cmd.ctlInvoker, req) diff --git a/src/control/cmd/dmg/pool_properties.go b/src/control/cmd/dmg/pool_properties.go index 10921ef8cae..108c943c029 100644 --- a/src/control/cmd/dmg/pool_properties.go +++ b/src/control/cmd/dmg/pool_properties.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2022 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -7,16 +8,46 @@ package main import ( + "fmt" + "sort" + "github.com/jessevdk/go-flags" "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ui" ) +// PoolPropSet is a non-duplicated set of pool properties. +type PoolPropSet map[string]*daos.PoolProperty + +// Slice returns a sorted slice of the pool properties in the set. +func (pps PoolPropSet) Slice() []*daos.PoolProperty { + slice := make([]*daos.PoolProperty, 0, len(pps)) + for _, prop := range pps { + slice = append(slice, prop) + } + sort.Slice(slice, func(i, j int) bool { + return slice[i].Name < slice[j].Name + }) + + return slice +} + +// Add adds a property to the set, or overrides the existing value. +func (pps PoolPropSet) Add(prop *daos.PoolProperty) { + pps[prop.Name] = prop +} + +// HasProp checks if the named prop has already been added to the set. +func (pps PoolPropSet) HasProp(name string) bool { + _, found := pps[name] + return found +} + type PoolSetPropsFlag struct { ui.SetPropertiesFlag - ToSet []*daos.PoolProperty + ToSet PoolPropSet } func (f *PoolSetPropsFlag) UnmarshalFlag(fv string) error { @@ -28,17 +59,23 @@ func (f *PoolSetPropsFlag) UnmarshalFlag(fv string) error { return err } + if f.ToSet == nil { + f.ToSet = make(PoolPropSet) + } for _, key := range propHdlrs.Keys() { val, parsed := f.ParsedProps[key] if !parsed { continue } + if f.ToSet.HasProp(key) { + return fmt.Errorf("%q: same key was specified more than once", key) + } hdlr := propHdlrs[key] p := hdlr.GetProperty(key) if err := p.SetValue(val); err != nil { return err } - f.ToSet = append(f.ToSet, p) + f.ToSet.Add(p) } return nil diff --git a/src/control/cmd/dmg/pool_test.go b/src/control/cmd/dmg/pool_test.go index 70362cea777..ab2a89fc485 100644 --- a/src/control/cmd/dmg/pool_test.go +++ b/src/control/cmd/dmg/pool_test.go @@ -516,9 +516,27 @@ func TestPoolCommands(t *testing.T) { strings.Join([]string{ printRequest(t, &control.PoolCreateReq{ Properties: []*daos.PoolProperty{ + propWithVal("label", "label"), propWithVal("scrub", "timed"), propWithVal("scrub_freq", "1"), + }, + User: eUsr.Username + "@", + UserGroup: eGrp.Name + "@", + Ranks: []ranklist.Rank{}, + TierBytes: []uint64{uint64(testSize), 0}, + }), + }, " "), + nil, + }, + { + "Create pool with properties specified separately", + fmt.Sprintf("pool create label --scm-size %s --properties scrub:timed --properties scrub_freq:1", testSizeStr), + strings.Join([]string{ + printRequest(t, &control.PoolCreateReq{ + Properties: []*daos.PoolProperty{ propWithVal("label", "label"), + propWithVal("scrub", "timed"), + propWithVal("scrub_freq", "1"), }, User: eUsr.Username + "@", UserGroup: eGrp.Name + "@", @@ -528,6 +546,12 @@ func TestPoolCommands(t *testing.T) { }, " "), nil, }, + { + "Create pool with duplicate properties specified separately", + fmt.Sprintf("pool create label --scm-size %s --properties scrub:timed --properties scrub:timed,scrub_freq:1", testSizeStr), + "", + errors.New("more than once"), + }, // Exclude testing with multiple ranks is verified at the control API layer. { "Exclude a target with single target idx", @@ -733,6 +757,20 @@ func TestPoolCommands(t *testing.T) { }, " "), nil, }, + { + "Set pool properties with semi-colon separated self_heal value", + `pool set-prop 031bcaf8-f0f5-42ef-b3c5-ee048676dceb self_heal:exclude;rebuild,space_rb:42`, + strings.Join([]string{ + printRequest(t, &control.PoolSetPropReq{ + ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb", + Properties: []*daos.PoolProperty{ + propWithVal("self_heal", "exclude;rebuild"), + propWithVal("space_rb", "42"), + }, + }), + }, " "), + nil, + }, { "Set pool properties with pool flag", "pool set-prop 031bcaf8-f0f5-42ef-b3c5-ee048676dceb label:foo,space_rb:42", @@ -809,6 +847,20 @@ func TestPoolCommands(t *testing.T) { }, " "), nil, }, + { + "Get pool properties", + "pool get-prop 031bcaf8-f0f5-42ef-b3c5-ee048676dceb label,self_heal", + strings.Join([]string{ + printRequest(t, &control.PoolGetPropReq{ + ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb", + Properties: []*daos.PoolProperty{ + propWithVal("label", ""), + propWithVal("self_heal", ""), + }, + }), + }, " "), + nil, + }, { "Get pool ACL", "pool get-acl 031bcaf8-f0f5-42ef-b3c5-ee048676dceb", diff --git a/src/control/lib/control/pool.go b/src/control/lib/control/pool.go index a77b90efa19..b20dbbf7ef7 100644 --- a/src/control/lib/control/pool.go +++ b/src/control/lib/control/pool.go @@ -13,6 +13,7 @@ import ( "fmt" "math" "os/user" + "slices" "sort" "strings" "time" @@ -660,14 +661,23 @@ type PoolGetPropReq struct { // PoolGetProp sends a pool get-prop request to the pool service leader. func PoolGetProp(ctx context.Context, rpcClient UnaryInvoker, req *PoolGetPropReq) ([]*daos.PoolProperty, error) { - // Get all by default. if len(req.Properties) == 0 { + // Get all by default. allProps := daos.PoolProperties() req.Properties = make([]*daos.PoolProperty, 0, len(allProps)) for _, key := range allProps.Keys() { hdlr := allProps[key] req.Properties = append(req.Properties, hdlr.GetProperty(key)) } + } else { + for _, prop := range req.Properties { + if prop == nil { + return nil, errors.New("nil property") + } + } + slices.SortFunc(req.Properties, func(a, b *daos.PoolProperty) int { + return strings.Compare(a.Name, b.Name) + }) } pbReq := &mgmtpb.PoolGetPropReq{ @@ -707,6 +717,7 @@ func PoolGetProp(ctx context.Context, rpcClient UnaryInvoker, req *PoolGetPropRe pbMap[prop.GetNumber()] = prop } + // We've already sorted req.Properties above. resp := make([]*daos.PoolProperty, 0, len(req.Properties)) for _, prop := range req.Properties { pbProp, found := pbMap[prop.Number] diff --git a/src/control/lib/daos/pool_property.go b/src/control/lib/daos/pool_property.go index 0d8a8d1e24c..bdb4117e327 100644 --- a/src/control/lib/daos/pool_property.go +++ b/src/control/lib/daos/pool_property.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2023 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -59,9 +60,9 @@ func PoolProperties() PoolPropertyMap { case PoolSelfHealingDelayRebuild: return "delay_rebuild" case PoolSelfHealingAutoExclude | PoolSelfHealingAutoRebuild: - return "exclude,rebuild" + return "exclude;rebuild" case PoolSelfHealingAutoExclude | PoolSelfHealingDelayRebuild: - return "exclude,delay_rebuild" + return "exclude;delay_rebuild" default: return "unknown" } @@ -71,10 +72,10 @@ func PoolProperties() PoolPropertyMap { "exclude": PoolSelfHealingAutoExclude, "rebuild": PoolSelfHealingAutoRebuild, "delay_rebuild": PoolSelfHealingDelayRebuild, - "exclude,rebuild": PoolSelfHealingAutoExclude | PoolSelfHealingAutoRebuild, - "rebuild,exclude": PoolSelfHealingAutoExclude | PoolSelfHealingAutoRebuild, - "delay_rebuild,exclude": PoolSelfHealingAutoExclude | PoolSelfHealingDelayRebuild, - "exclude,delay_rebuild": PoolSelfHealingAutoExclude | PoolSelfHealingDelayRebuild, + "exclude;rebuild": PoolSelfHealingAutoExclude | PoolSelfHealingAutoRebuild, + "rebuild;exclude": PoolSelfHealingAutoExclude | PoolSelfHealingAutoRebuild, + "delay_rebuild;exclude": PoolSelfHealingAutoExclude | PoolSelfHealingDelayRebuild, + "exclude;delay_rebuild": PoolSelfHealingAutoExclude | PoolSelfHealingDelayRebuild, }, }, "space_rb": { diff --git a/src/control/lib/daos/pool_property_test.go b/src/control/lib/daos/pool_property_test.go index e9742a07c05..9e1a4b2e040 100644 --- a/src/control/lib/daos/pool_property_test.go +++ b/src/control/lib/daos/pool_property_test.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2023 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -174,23 +175,23 @@ func TestControl_PoolProperties(t *testing.T) { expStr: "self_heal:delay_rebuild", expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"delay_rebuild"}`), }, - "self_heal-exclude,rebuild": { + "self_heal-exclude;rebuild": { name: "self_heal", - value: "exclude,rebuild", - expStr: "self_heal:exclude,rebuild", - expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude,rebuild"}`), + value: "exclude;rebuild", + expStr: "self_heal:exclude;rebuild", + expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude;rebuild"}`), }, - "self_heal-rebuild,exclude": { + "self_heal-rebuild;exclude": { name: "self_heal", - value: "rebuild,exclude", - expStr: "self_heal:exclude,rebuild", - expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude,rebuild"}`), + value: "rebuild;exclude", + expStr: "self_heal:exclude;rebuild", + expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude;rebuild"}`), }, - "self_heal-exclude,delay_rebuild": { + "self_heal-exclude;delay_rebuild": { name: "self_heal", - value: "exclude,delay_rebuild", - expStr: "self_heal:exclude,delay_rebuild", - expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude,delay_rebuild"}`), + value: "exclude;delay_rebuild", + expStr: "self_heal:exclude;delay_rebuild", + expJson: []byte(`{"name":"self_heal","description":"Self-healing policy","value":"exclude;delay_rebuild"}`), }, "self_heal-invalid": { name: "self_heal", diff --git a/src/control/lib/ui/prop_flags.go b/src/control/lib/ui/prop_flags.go index b8e6438c15d..6193d414d31 100644 --- a/src/control/lib/ui/prop_flags.go +++ b/src/control/lib/ui/prop_flags.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2023 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -158,6 +159,9 @@ func (f *SetPropertiesFlag) UnmarshalFlag(fv string) error { if len(value) == 0 { return propError("value must not be empty") } + if _, set := f.ParsedProps[key]; set { + return propError("%q: same key was specified more than once", key) + } f.ParsedProps[key] = value } diff --git a/src/control/lib/ui/prop_flags_test.go b/src/control/lib/ui/prop_flags_test.go index 4e9424a6680..f5ce13b2ee1 100644 --- a/src/control/lib/ui/prop_flags_test.go +++ b/src/control/lib/ui/prop_flags_test.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2023 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -46,6 +47,11 @@ func TestUI_SetPropertiesFlag_UnmarshalFlag(t *testing.T) { fv: strings.Repeat("x", ui.MaxPropKeyLen+1) + ":value", expErr: errors.New("key too long"), }, + "duplicated properties": { + settable: []string{"a", "b"}, + fv: "a:c,b:d,a:d", + expErr: errors.New("more than once"), + }, "valid properties": { settable: []string{"a", "b"}, fv: "b:d,a:c", @@ -105,7 +111,6 @@ func TestUI_SetPropertiesFlag_UnmarshalFlag(t *testing.T) { } -/* NB: Doesn't work with go < 1.18 func FuzzUI_SetPropertiesFlag_UnmarshalFlag(f *testing.F) { f.Fuzz(func(t *testing.T, fv string) { f := ui.SetPropertiesFlag{} @@ -118,7 +123,6 @@ func FuzzUI_SetPropertiesFlag_UnmarshalFlag(f *testing.F) { } }) } -*/ func TestUI_SetPropertiesFlag_Complete(t *testing.T) { comps := ui.CompletionMap{ diff --git a/src/control/system/member.go b/src/control/system/member.go index 9ff5b6f720d..7edd34e1498 100644 --- a/src/control/system/member.go +++ b/src/control/system/member.go @@ -1,5 +1,6 @@ // // (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -161,6 +162,14 @@ func (ms MemberState) isTransitionIllegal(to MemberState) bool { MemberStateReady: true, MemberStateJoined: true, }, + MemberStateExcluded: { + MemberStateErrored: true, + MemberStateStopping: true, + MemberStateStopped: true, + MemberStateStarting: true, + MemberStateReady: true, + MemberStateUnresponsive: true, + }, }[ms][to] } diff --git a/src/control/system/member_internal_test.go b/src/control/system/member_internal_test.go new file mode 100644 index 00000000000..cf3aafad07c --- /dev/null +++ b/src/control/system/member_internal_test.go @@ -0,0 +1,198 @@ +// +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package system + +import ( + "fmt" + "testing" + + "github.com/daos-stack/daos/src/control/common/test" +) + +func TestSystem_MemberState_isTransitionIllegal(t *testing.T) { + nextState := func(state MemberState) MemberState { + if state == MemberStateUnknown { + return state + 1 + } + return state << 1 + } + + for name, tc := range map[string]struct { + start MemberState + expLegalStates []MemberState + }{ + "unknown": { + start: MemberStateUnknown, + expLegalStates: []MemberState{}, + }, + "admin excluded": { + start: MemberStateAdminExcluded, + expLegalStates: []MemberState{}, + }, + "await format": { + start: MemberStateAwaitFormat, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateJoined, + MemberStateReady, + MemberStateStarting, + MemberStateStopped, + MemberStateStopping, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "starting": { + start: MemberStateStarting, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateJoined, + MemberStateReady, + MemberStateStopped, + MemberStateStopping, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "ready": { + start: MemberStateReady, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateJoined, + MemberStateStarting, + MemberStateStopped, + MemberStateStopping, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "joined": { + start: MemberStateJoined, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateExcluded, + MemberStateStarting, + MemberStateStopped, + MemberStateStopping, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "errored": { + start: MemberStateErrored, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateExcluded, + MemberStateStarting, + MemberStateStopped, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "unresponsive": { + start: MemberStateUnresponsive, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateExcluded, + MemberStateStarting, + MemberStateStopped, + MemberStateUnknown, + }, + }, + "checker": { + start: MemberStateCheckerStarted, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateErrored, + MemberStateExcluded, + MemberStateStarting, + MemberStateStopping, + MemberStateStopped, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "excluded": { + start: MemberStateExcluded, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateJoined, + MemberStateCheckerStarted, + MemberStateUnknown, + }, + }, + "stopping": { + start: MemberStateStopping, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateExcluded, + MemberStateJoined, + MemberStateReady, + MemberStateStarting, + MemberStateStopped, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + "stopped": { + start: MemberStateStopped, + expLegalStates: []MemberState{ + MemberStateAdminExcluded, + MemberStateAwaitFormat, + MemberStateCheckerStarted, + MemberStateErrored, + MemberStateExcluded, + MemberStateJoined, + MemberStateReady, + MemberStateStarting, + MemberStateStopping, + MemberStateUnresponsive, + MemberStateUnknown, + }, + }, + } { + expLegal := func(state MemberState) bool { + for _, l := range tc.expLegalStates { + if l == state { + return true + } + } + return false + } + + for end := MemberStateUnknown; end < MemberStateMax; end = nextState(end) { + caseName := fmt.Sprintf("%s/%s -> %s", name, tc.start, end) + + t.Run(caseName, func(t *testing.T) { + result := tc.start.isTransitionIllegal(end) + + test.AssertEqual(t, !expLegal(end), result, "") + }) + } + } +} diff --git a/src/control/system/raft/database.go b/src/control/system/raft/database.go index 5aedf7cd460..926d35610a6 100644 --- a/src/control/system/raft/database.go +++ b/src/control/system/raft/database.go @@ -54,6 +54,7 @@ type ( Barrier(time.Duration) raft.Future Shutdown() raft.Future State() raft.RaftState + GetConfiguration() raft.ConfigurationFuture } // syncRaft provides a wrapper for synchronized access to the @@ -734,6 +735,21 @@ func (db *Database) RemoveMember(m *system.Member) error { return db.submitMemberUpdate(raftOpRemoveMember, &memberUpdate{Member: m}) } +func raftSvcHasVoter(svc raftService, voterAddr raft.ServerAddress) (bool, error) { + cfgFuture := svc.GetConfiguration() + if cfgFuture.Error() != nil { + return false, errors.Wrapf(cfgFuture.Error(), "get raft configuration") + } + + cfg := cfgFuture.Configuration() + for _, rs := range cfg.Servers { + if rs.Address == voterAddr { + return true, nil + } + } + return false, nil +} + func (db *Database) manageVoter(vc *system.Member, op raftOp) error { // Ignore self as a voter candidate. if common.CmpTCPAddr(db.replicaAddr, vc.Addr) { @@ -749,12 +765,36 @@ func (db *Database) manageVoter(vc *system.Member, op raftOp) error { rsa := raft.ServerAddress(vc.Addr.String()) switch op { - case raftOpAddMember: - case raftOpUpdateMember, raftOpRemoveMember: - // If we're updating an existing member, we need to kick it out of the - // raft cluster and then re-add it so that it doesn't hijack the campaign. - db.log.Debugf("removing %s as a current raft voter", vc) + case raftOpAddMember, raftOpUpdateMember: + return db.raft.withReadLock(func(svc raftService) error { + if hasVoter, err := raftSvcHasVoter(svc, rsa); err != nil { + return errors.Wrapf(err, "check if %s is already a raft voter", rsa) + } else if !hasVoter { + db.log.Debugf("adding %s as a new raft voter", vc) + return svc.AddVoter(rsi, rsa, 0, 0).Error() + } + + // It's already a voter + return nil + }) + case raftOpRemoveMember: + // We only need to remove a voter if all of its members are gone. if err := db.raft.withReadLock(func(svc raftService) error { + members, err := db.FindMembersByAddr(vc.Addr) + if err != nil { + return errors.Wrapf(err, "look up members at address %s", rsa) + } + for _, m := range members { + if m.UUID == vc.UUID { + // This is the member we're planning to remove + continue + } + + // There are members other than this one + return nil + } + + db.log.Debugf("removing %s as a current raft voter", vc) return svc.RemoveServer(rsi, 0, 0).Error() }); err != nil { return errors.Wrapf(err, "failed to remove %q as a raft replica", vc.Addr) @@ -763,13 +803,6 @@ func (db *Database) manageVoter(vc *system.Member, op raftOp) error { return errors.Errorf("unhandled manageVoter op: %s", op) } - db.log.Debugf("adding %s as a new raft voter", vc) - if err := db.raft.withReadLock(func(svc raftService) error { - return svc.AddVoter(rsi, rsa, 0, 0).Error() - }); err != nil { - return errors.Wrapf(err, "failed to add %q as raft replica", vc.Addr) - } - return nil } diff --git a/src/control/system/raft/mocks.go b/src/control/system/raft/mocks.go index 9f6b9a846e8..6104a3532a0 100644 --- a/src/control/system/raft/mocks.go +++ b/src/control/system/raft/mocks.go @@ -26,13 +26,15 @@ type ( err error index uint64 response interface{} + config raft.Configuration } mockRaftServiceConfig struct { - LeaderCh <-chan bool - ServerAddress raft.ServerAddress - State raft.RaftState - LeadershipTransferErr error - BarrierReturn raft.Future + LeaderCh <-chan bool + ServerAddress raft.ServerAddress + State raft.RaftState + LeadershipTransferErr error + BarrierReturn raft.Future + GetConfigurationReturn raft.ConfigurationFuture } mockRaftService struct { cfg mockRaftServiceConfig @@ -42,9 +44,10 @@ type ( ) // mockRaftFuture implements raft.Future, raft.IndexFuture, and raft.ApplyFuture -func (mrf *mockRaftFuture) Error() error { return mrf.err } -func (mrf *mockRaftFuture) Index() uint64 { return mrf.index } -func (mrf *mockRaftFuture) Response() interface{} { return mrf.response } +func (mrf *mockRaftFuture) Error() error { return mrf.err } +func (mrf *mockRaftFuture) Index() uint64 { return mrf.index } +func (mrf *mockRaftFuture) Response() interface{} { return mrf.response } +func (mrf *mockRaftFuture) Configuration() raft.Configuration { return mrf.config } func (mrs *mockRaftService) Apply(cmd []byte, timeout time.Duration) raft.ApplyFuture { mrs.fsm.Apply(&raft.Log{Data: cmd}) @@ -95,6 +98,13 @@ func (mrs *mockRaftService) Barrier(time.Duration) raft.Future { return mrs.cfg.BarrierReturn } +func (mrs *mockRaftService) GetConfiguration() raft.ConfigurationFuture { + if mrs.cfg.GetConfigurationReturn == nil { + return &mockRaftFuture{} + } + return mrs.cfg.GetConfigurationReturn +} + // resetCalls resets call counts for the mock. func (mrs *mockRaftService) resetCalls() { mrs.addVoterCalledForAddrs = nil diff --git a/src/engine/init.c b/src/engine/init.c index 130b54773cb..2727568f678 100644 --- a/src/engine/init.c +++ b/src/engine/init.c @@ -1,6 +1,7 @@ /* * (C) Copyright 2016-2024 Intel Corporation. * (C) Copyright 2025 Google LLC + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -158,6 +159,27 @@ hlc_recovery_end(uint64_t bound) } } +static int +init_tls_support(void) +{ + int rc; + + rc = ds_tls_key_create(); + if (rc != 0) + return daos_errno2der(rc); + + dss_register_key(&daos_srv_modkey); + + return 0; +} + +static void +fini_tls_support(void) +{ + dss_unregister_key(&daos_srv_modkey); + ds_tls_key_delete(); +} + /* * Register the dbtree classes used by native server-side modules (e.g., * ds_pool, ds_cont, etc.). Unregistering is currently not supported. @@ -694,10 +716,6 @@ dss_crt_hlc_error_cb(void *arg) static void server_id_cb(uint32_t *tid, uint64_t *uid) { - - if (server_init_state != DSS_INIT_STATE_SET_UP) - return; - if (uid != NULL && dss_abt_init) { ABT_unit_type type = ABT_UNIT_TYPE_EXT; int rc; @@ -749,10 +767,15 @@ server_init(int argc, char *argv[]) gethostname(dss_hostname, DSS_HOSTNAME_MAX_LEN); + /* Must initialize TLS support before using server_id_cb. */ + rc = init_tls_support(); + if (rc != 0) + return rc; + daos_debug_set_id_cb(server_id_cb); rc = daos_debug_init_ex(DAOS_LOG_DEFAULT, DLOG_INFO); if (rc != 0) - return rc; + goto exit_tls_support; /** initialize server topology data - this is needed to set up the number of targets */ rc = dss_topo_init(); @@ -925,6 +948,8 @@ server_init(int argc, char *argv[]) /* dss_topo_fini cleans itself if it fails */ exit_debug_init: daos_debug_fini(); +exit_tls_support: + fini_tls_support(); return rc; } @@ -985,6 +1010,8 @@ server_fini(bool force) D_INFO("dss_top_fini() done\n"); daos_debug_fini(); D_INFO("daos_debug_fini() done\n"); + fini_tls_support(); + D_INFO("fini_tls_support() done\n"); } static void diff --git a/src/engine/module.c b/src/engine/module.c index d43387573e4..847d9d800d3 100644 --- a/src/engine/module.c +++ b/src/engine/module.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -244,12 +245,6 @@ dss_module_init_all(uint64_t *mod_facs) uint64_t fac; int rc = 0; - /* - * first register global tls accessible to all modules, - * it'll be first initialized and last finalized. - */ - dss_register_key(&daos_srv_modkey); - /* lookup the module from the loaded module list */ D_MUTEX_LOCK(&loaded_mod_list_lock); d_list_for_each_entry_safe(lmod, tmp, &loaded_mod_list, lm_lk) { diff --git a/src/engine/srv.c b/src/engine/srv.c index 68c325d5f96..a82121af88e 100644 --- a/src/engine/srv.c +++ b/src/engine/srv.c @@ -571,8 +571,10 @@ dss_srv_handler(void *arg) } } - if (dss_xstream_exiting(dx)) + if (dss_xstream_exiting(dx)) { + rc = 0; break; + } ABT_thread_yield(); } @@ -609,6 +611,7 @@ dss_srv_handler(void *arg) ABT_cond_signal(xstream_data.xd_ult_init); ABT_mutex_unlock(xstream_data.xd_mutex); } + DL_CDEBUG(rc == 0, DLOG_INFO, DLOG_ERR, rc, "stopping"); } static inline struct dss_xstream * @@ -1067,8 +1070,10 @@ dss_start_xs_id(int tag, int xs_id) } rc = dss_start_one_xstream(obj->cpuset, tag, xs_id); - if (rc) + if (rc != 0) { + DL_ERROR(rc, "failed to start one xstream: tag=%x xs_id=%d", tag, xs_id); return rc; + } return 0; } @@ -1318,7 +1323,6 @@ enum { XD_INIT_MUTEX, XD_INIT_ULT_INIT, XD_INIT_ULT_BARRIER, - XD_INIT_TLS_REG, XD_INIT_TLS_INIT, XD_INIT_SYS_DB, XD_INIT_XSTREAMS, @@ -1356,9 +1360,6 @@ dss_srv_fini(bool force) case XD_INIT_TLS_INIT: vos_standalone_tls_fini(); /* fall through */ - case XD_INIT_TLS_REG: - ds_tls_key_delete(); - /* fall through */ case XD_INIT_ULT_BARRIER: ABT_cond_free(&xstream_data.xd_ult_barrier); /* fall through */ @@ -1458,15 +1459,6 @@ dss_srv_init(void) } xstream_data.xd_init_step = XD_INIT_ULT_BARRIER; - /* register xstream-local storage key */ - rc = ds_tls_key_create(); - if (rc) { - rc = dss_abterr2der(rc); - D_ERROR("Failed to register storage key: "DF_RC"\n", DP_RC(rc)); - D_GOTO(failed, rc); - } - xstream_data.xd_init_step = XD_INIT_TLS_REG; - /* initialize xstream-local storage */ rc = vos_standalone_tls_init(DAOS_SERVER_TAG - DAOS_TGT_TAG); if (rc) { diff --git a/src/gurt/misc.c b/src/gurt/misc.c index 641869db9ba..094b3cc1a56 100644 --- a/src/gurt/misc.c +++ b/src/gurt/misc.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -904,43 +905,83 @@ d_rank_range_list_create_from_ranks(d_rank_list_t *rank_list) return range_list; } +static inline int +ranks2str(char *buf, size_t buf_size, uint32_t rank_lo, uint32_t rank_hi, size_t *buf_len) +{ + int len; + + if (rank_lo == rank_hi) + len = snprintf(buf, buf_size, "%u", rank_lo); + else + len = snprintf(buf, buf_size, "%u-%u", rank_lo, rank_hi); + + if (len < 0) + return -DER_INVAL; + if (len >= buf_size) + return -DER_TRUNC; + + *buf_len = (size_t)len; + return -DER_SUCCESS; +} + int d_rank_range_list_str(d_rank_range_list_t *list, char **ranks_str) { - const size_t MAXBYTES = 512u; - size_t remaining = MAXBYTES - 2u; + const size_t MINBYTES = 256u; char *line; - char *linepos; + size_t line_size; + char *buf; + size_t buf_size; int i; - int len; int rc; D_ASSERT(list != NULL); - D_ALLOC(line, MAXBYTES); + D_ALLOC_ARRAY(line, MINBYTES); if (line == NULL) D_GOTO(error, rc = -DER_NOMEM); + line_size = MINBYTES; + buf = line; + buf_size = line_size; - *line = '['; - linepos = line + 1; + buf[0] = '['; + buf++; + buf_size--; for (i = 0; i < list->rrl_nr; i++) { - uint32_t lo = list->rrl_ranges[i].lo; - uint32_t hi = list->rrl_ranges[i].hi; - bool lastrange = (i == (list->rrl_nr - 1)); + size_t len = 0; + uint32_t rank_lo = list->rrl_ranges[i].lo; + uint32_t rank_hi = list->rrl_ranges[i].hi; + bool is_last_iter = (i == (list->rrl_nr - 1)); + + rc = ranks2str(buf, buf_size, rank_lo, rank_hi, &len); + if (rc == -DER_INVAL) + D_GOTO(error, rc); + if (rc == -DER_TRUNC || + (rc == -DER_SUCCESS && buf_size < len + sizeof("]") + (is_last_iter ? 0 : 1))) { + int buf_idx = buf - line; + + D_REALLOC_ARRAY(buf, line, line_size, line_size + MINBYTES); + if (buf == NULL) + D_GOTO(error, rc = -DER_NOMEM); + line = buf; + line_size += MINBYTES; + buf = line + buf_idx; + buf_size += MINBYTES; + + rc = ranks2str(buf, buf_size, rank_lo, rank_hi, &len); + } + D_ASSERT(rc == -DER_SUCCESS && + buf_size >= len + sizeof("]") + (is_last_iter ? 0 : 1)); - if (lo == hi) - len = snprintf(linepos, remaining, "%u%s", lo, lastrange ? "" : ","); - else - len = snprintf(linepos, remaining, "%u-%u%s", lo, hi, lastrange ? "" : ","); - if (len < 0) - D_GOTO(error, rc = -DER_INVAL); - if (len >= remaining) - D_GOTO(error, rc = -DER_TRUNC); - - remaining -= len; - linepos += len; + if (!is_last_iter) { + buf[len] = ','; + len++; + } + + buf += len; + buf_size -= len; } - memcpy(linepos, "]", 2u); + memcpy(buf, "]", sizeof("]")); *ranks_str = line; D_GOTO(out, rc = -DER_SUCCESS); diff --git a/src/gurt/tests/test_gurt.c b/src/gurt/tests/test_gurt.c index e683a49629b..f685b759d34 100644 --- a/src/gurt/tests/test_gurt.c +++ b/src/gurt/tests/test_gurt.c @@ -2520,6 +2520,8 @@ test_d_rank_list_to_str(void **state) d_rank_list_t *ranks; char *ranks_str = NULL; int i; + char *line; + char *buf; int rc; // Test with null list @@ -2565,16 +2567,31 @@ test_d_rank_list_to_str(void **state) D_FREE(ranks_str); d_rank_list_free(ranks); - // Test truncate error + // Test with long list ranks = d_rank_list_alloc(1024); assert_non_null(ranks); - for (i = 0; i < 1024; ++i) + D_ALLOC(line, 1024 * 5); + assert_non_null(line); + buf = line; + buf[0] = '['; + buf++; + for (i = 0; i < 1024; ++i) { + int len; + ranks->rl_ranks[i] = 2 * i + 1; + len = sprintf(buf, "%d,", ranks->rl_ranks[i]); + assert_true(len > 0); + buf += len; + } + memcpy(buf - 1, "]", sizeof("]")); rc = d_rank_list_to_str(ranks, &ranks_str); - assert_int_equal(rc, -DER_TRUNC); - assert_null(ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_non_null(ranks_str); + assert_string_equal(ranks_str, line); + D_FREE(line); + D_FREE(ranks_str); d_rank_list_free(ranks); } @@ -2642,6 +2659,8 @@ test_d_rank_range_list_str(void **state) d_rank_range_list_t *range_list; char *ranks_str = NULL; int i; + char *line; + char *buf; int rc; // Test with empty list @@ -2683,18 +2702,32 @@ test_d_rank_range_list_str(void **state) D_FREE(ranks_str); d_rank_range_list_free(range_list); - // Test truncate error + // Test with lot of ranks range_list = d_rank_range_list_alloc(1024); assert_non_null(range_list); + D_ALLOC(line, 1024 * 5); + assert_non_null(line); + buf = line; + buf[0] = '['; + buf++; for (i = 0; i < 1024; ++i) { + int len; + range_list->rrl_ranges[i].lo = i; range_list->rrl_ranges[i].hi = i; + len = sprintf(buf, "%d,", i); + assert_true(len > 0); + buf += len; } + memcpy(buf - 1, "]", sizeof("]")); rc = d_rank_range_list_str(range_list, &ranks_str); - assert_int_equal(rc, -DER_TRUNC); - assert_null(ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_non_null(ranks_str); + assert_string_equal(ranks_str, line); + D_FREE(line); + D_FREE(ranks_str); d_rank_range_list_free(range_list); } diff --git a/src/include/daos/pool.h b/src/include/daos/pool.h index 036122133f2..63445377d0c 100644 --- a/src/include/daos/pool.h +++ b/src/include/daos/pool.h @@ -35,6 +35,7 @@ /** pool query request bits */ #define DAOS_PO_QUERY_SPACE (1ULL << 0) #define DAOS_PO_QUERY_REBUILD_STATUS (1ULL << 1) +#define DAOS_PO_QUERY_REBULD_MAX_LAYOUT_VER (1ULL << 2) #define PROP_BIT_START 16 #define DAOS_PO_QUERY_PROP_BIT_START PROP_BIT_START #define DAOS_PO_QUERY_PROP_LABEL (1ULL << (PROP_BIT_START + 0)) @@ -136,6 +137,8 @@ struct dc_pool { /* pool redunc factor */ uint32_t dp_rf; + /* Maximum supported layout version */ + uint16_t dp_max_supported_layout_ver; }; static inline unsigned int diff --git a/src/include/daos/sys_db.h b/src/include/daos/sys_db.h index 850e9c4203a..91d85e735ab 100644 --- a/src/include/daos/sys_db.h +++ b/src/include/daos/sys_db.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2022 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -12,8 +13,8 @@ #include struct sys_db; -typedef int (*sys_db_trav_cb_t)(struct sys_db *db, char *table, d_iov_t *key, - void *args); +typedef int (*sys_db_trav_cb_t)(struct sys_db *db, char *table, d_iov_t *key, void *args, + unsigned *acts); #define SYS_DB_NAME_SZ 32 diff --git a/src/include/daos_pool.h b/src/include/daos_pool.h index 219dc13fad7..3f7ef632423 100644 --- a/src/include/daos_pool.h +++ b/src/include/daos_pool.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2020-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -116,8 +117,10 @@ struct daos_rebuild_status { int32_t rs_state; int32_t rs_done; }; + /** Maximum supported layout version */ + uint16_t rs_max_supported_layout_ver; /** padding of rebuild status */ - int32_t rs_padding32; + int16_t rs_padding16; /** Failure on which rank */ int32_t rs_fail_rank; @@ -159,6 +162,8 @@ enum daos_pool_info_bit { DPI_ENGINES_DISABLED = 1ULL << 3, /** true to include (in \a ranks) engines marked DEAD by SWIM. */ DPI_ENGINES_DEAD = 1ULL << 4, + /** true to query pool's maximum supported rebuild layout version */ + DPI_REBUILD_MAX_LAYOUT_VER = 1ULL << 5, /** query all above optional info */ DPI_ALL = -1, }; diff --git a/src/include/daos_prop.h b/src/include/daos_prop.h index 3b7216efd0e..c57399744e6 100644 --- a/src/include/daos_prop.h +++ b/src/include/daos_prop.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2015-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -142,6 +143,8 @@ enum daos_pool_props { DAOS_PROP_PO_MAX, }; +#define DAOS_PROP_PO_NUM (DAOS_PROP_PO_MAX - DAOS_PROP_PO_MIN - 1) + #define DAOS_PROP_PO_EC_CELL_SZ_MIN (1UL << 10) #define DAOS_PROP_PO_EC_CELL_SZ_MAX (1UL << 30) diff --git a/src/include/daos_srv/container.h b/src/include/daos_srv/container.h index ccd5b43e0a0..7c9ccaf27ca 100644 --- a/src/include/daos_srv/container.h +++ b/src/include/daos_srv/container.h @@ -137,6 +137,7 @@ struct ds_cont_child { d_list_t sc_dtx_batched_list; /* the pool map version of updating DAOS_PROP_CO_STATUS prop */ uint32_t sc_status_pm_ver; + int sc_ec_agg_updates; }; struct agg_param { @@ -149,6 +150,14 @@ struct agg_param { typedef int (*cont_aggregate_cb_t)(struct ds_cont_child *cont, daos_epoch_range_t *epr, uint32_t flags, struct agg_param *param); + +static inline bool +ds_cont_child_ec_aggregating(struct ds_cont_child *cont) +{ + /* either I'm running aggregation, or someone else is writing to me for aggregation */ + return cont->sc_ec_agg_updates || cont->sc_ec_agg_active; +} + void cont_aggregate_interval(struct ds_cont_child *cont, cont_aggregate_cb_t cb, struct agg_param *param); @@ -199,6 +208,9 @@ ds_cont_child_destroy(uuid_t pool_uuid, uuid_t cont_uuid); void ds_cont_child_reset_ec_agg_eph_all(struct ds_pool_child *pool_child); +void + ds_cont_child_wait_ec_agg_pause(struct ds_pool_child *pool_child, int wait_timeout); + /** initialize a csummer based on container properties. Will retrieve the * checksum related properties from IV */ diff --git a/src/include/daos_srv/evtree.h b/src/include/daos_srv/evtree.h index 8ae4f12638e..366b8d8b1e5 100644 --- a/src/include/daos_srv/evtree.h +++ b/src/include/daos_srv/evtree.h @@ -644,19 +644,19 @@ int evt_debug(daos_handle_t toh, int debug_level); enum { /** Return extents visible in the search rectangle */ - EVT_ITER_VISIBLE = (1 << 0), + EVT_ITER_VISIBLE = (1 << 0), /** Add fully or partially covered extents to EVT_ITER_VISIBLE */ - EVT_ITER_COVERED = (1 << 1) | EVT_ITER_VISIBLE, + EVT_ITER_COVERED = (1 << 1) | EVT_ITER_VISIBLE, /** Skip visible holes (Only valid with EVT_ITER_VISIBLE) */ - EVT_ITER_SKIP_HOLES = (1 << 2), + EVT_ITER_SKIP_HOLES = (1 << 2), /** * Use the embedded iterator of the open handle. * It can reduce memory consumption, but state of iterator can be * overwritten by other tree operation. */ - EVT_ITER_EMBEDDED = (1 << 3), + EVT_ITER_EMBEDDED = (1 << 3), /** Reverse iterator (ordered iterator only) */ - EVT_ITER_REVERSE = (1 << 4), + EVT_ITER_REVERSE = (1 << 4), /* If EVT_ITER_VISIBLE is set, evt_iter_probe will calculate and cache * visible extents and iterate through the cached extents. Each * rectangle will be marked as visible or covered. The partial bit will @@ -668,15 +668,18 @@ enum { * search rectangle, including punched extents, are returned. */ /** The iterator is for purge operation */ - EVT_ITER_FOR_PURGE = (1 << 5), + EVT_ITER_FOR_PURGE = (1 << 5), /** The iterator is for data migration scan */ - EVT_ITER_FOR_MIGRATION = (1 << 6), + EVT_ITER_FOR_MIGRATION = (1 << 6), /** The iterator is for data discard */ - EVT_ITER_FOR_DISCARD = (1 << 7), + EVT_ITER_FOR_DISCARD = (1 << 7), /** Skip visible data (Only valid with EVT_ITER_VISIBLE) */ - EVT_ITER_SKIP_DATA = (1 << 8), + EVT_ITER_SKIP_DATA = (1 << 8), /** Only process removals */ - EVT_ITER_REMOVALS = (1 << 9), + EVT_ITER_REMOVALS = (1 << 9), + /** Checking whether the target is aborted or not. */ + EVT_ITER_FOR_CHECK = (1 << 10), + }; D_CASSERT((int)EVT_VISIBLE == (int)EVT_ITER_VISIBLE); diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index 8d0b1450a32..285bbf223c6 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -95,9 +95,13 @@ struct ds_pool { /* pool_uuid + map version + leader term + rebuild generation define a * rebuild job. */ - uint32_t sp_rebuild_gen; - + uint32_t sp_rebuild_gen; int sp_rebuilding; + /** + * someone has already messaged this pool to for rebuild scan, + * NB: all xstreams can do lockless-write on it but it's OK + */ + int sp_rebuild_scan; int sp_discard_status; /** path to ephemeral metrics */ @@ -211,6 +215,12 @@ struct ds_pool_svc_op_val { char ov_resvd[60]; }; +static inline bool +ds_pool_is_rebuilding(struct ds_pool *pool) +{ + return (pool->sp_rebuilding > 0 || pool->sp_rebuild_scan > 0); +} + /* encode metadata RPC operation key: HLC time first, in network order, for keys sorted by time. * allocates the byte-stream, caller must free with D_FREE(). */ @@ -278,7 +288,9 @@ int int ds_pool_tgt_add_in(uuid_t pool_uuid, struct pool_target_id_list *list); int ds_pool_tgt_revert_rebuild(uuid_t pool_uuid, struct pool_target_id_list *list); -int ds_pool_tgt_finish_rebuild(uuid_t pool_uuid, struct pool_target_id_list *list); +int + ds_pool_tgt_finish_rebuild(uuid_t pool_uuid, struct pool_target_id_list *list, + uint32_t *reclaim_ver); int ds_pool_tgt_map_update(struct ds_pool *pool, struct pool_buf *buf, unsigned int map_version); diff --git a/src/include/daos_srv/ras.h b/src/include/daos_srv/ras.h index 5f22a75795f..5df47372eb9 100644 --- a/src/include/daos_srv/ras.h +++ b/src/include/daos_srv/ras.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2020-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -49,6 +50,7 @@ X(RAS_POOL_REPS_UPDATE, "pool_replicas_updated") \ X(RAS_POOL_DF_INCOMPAT, "pool_durable_format_incompatible") \ X(RAS_POOL_DEFER_DESTROY, "pool_destroy_deferred") \ + X(RAS_POOL_START_FAILED, "pool_start_failed") \ X(RAS_CONT_DF_INCOMPAT, "container_durable_format_incompatible") \ X(RAS_RDB_DF_INCOMPAT, "rdb_durable_format_incompatible") \ X(RAS_SWIM_RANK_ALIVE, "swim_rank_alive") \ diff --git a/src/include/daos_srv/rebuild.h b/src/include/daos_srv/rebuild.h index 96e2911903a..0fdfb6334aa 100644 --- a/src/include/daos_srv/rebuild.h +++ b/src/include/daos_srv/rebuild.h @@ -16,6 +16,7 @@ #define REBUILD_ENV "DAOS_REBUILD" #define REBUILD_ENV_DISABLED "no" +#define REBUILD_WAIT_EC_PAUSE_ENV "DAOS_REBUILD_WAIT_EC_PAUSE" /** * Enum values to indicate the rebuild operation that should be applied to the diff --git a/src/include/gurt/common.h b/src/include/gurt/common.h index 060cdc86c18..241dcd9b74a 100644 --- a/src/include/gurt/common.h +++ b/src/include/gurt/common.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ diff --git a/src/mgmt/srv_pool.c b/src/mgmt/srv_pool.c index 5c0a8e230cc..f9d14160b46 100644 --- a/src/mgmt/srv_pool.c +++ b/src/mgmt/srv_pool.c @@ -202,9 +202,10 @@ ds_mgmt_create_pool(uuid_t pool_uuid, const char *group, char *tgt_dev, d_rank_l D_GOTO(out, rc); rc = d_rank_list_to_str(targets, &tgt_str); - if (rc != 0) + if (rc != 0) { + D_FREE(pg_str); D_GOTO(out, rc); - + } D_ERROR(DF_UUID": targets (%s) contains ranks not in pg (%s)\n", DP_UUID(pool_uuid), tgt_str, pg_str); @@ -594,7 +595,13 @@ ds_mgmt_pool_set_prop(uuid_t pool_uuid, d_rank_list_t *svc_ranks, int rc; if (prop == NULL || prop->dpp_entries == NULL || prop->dpp_nr < 1) { - D_ERROR("invalid property list\n"); + D_ERROR("no properties in prop list\n"); + rc = -DER_INVAL; + goto out; + } + + if (!daos_prop_valid(prop, true, true)) { + D_ERROR("invalid properties\n"); rc = -DER_INVAL; goto out; } diff --git a/src/mgmt/srv_target.c b/src/mgmt/srv_target.c index 7975a2115d4..054842e5e76 100644 --- a/src/mgmt/srv_target.c +++ b/src/mgmt/srv_target.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1273,6 +1274,7 @@ tgt_destroy_cleanup(void *arg) /** make sure the rename is persistent */ (void)dir_fsync(zombie); + D_INFO(DF_UUID ": moved %s to %s\n", DP_UUID(tda->tda_id.uuid), tda->tda_path, zombie); /** * once successfully moved to the ZOMBIES directory, the target will @@ -1282,6 +1284,7 @@ tgt_destroy_cleanup(void *arg) if (tda->tda_rc == 0) { (void)subtree_destroy(zombie); (void)rmdir(zombie); + D_INFO(DF_UUID ": removed %s\n", DP_UUID(tda->tda_id.uuid), zombie); } else { D_INFO("Defer cleanup for lingering pool:"DF_UUID"\n", DP_UUID(tda->tda_id.uuid)); diff --git a/src/object/cli_obj.c b/src/object/cli_obj.c index c0de8cd4a18..6909a611245 100644 --- a/src/object/cli_obj.c +++ b/src/object/cli_obj.c @@ -3939,9 +3939,9 @@ obj_shard_comp_cb(tse_task_t *task, struct shard_auxi_args *shard_auxi, * 2) if any shard failed, store it in obj_auxi->result, the * un-retryable error with higher priority. */ + if (obj_auxi->map_ver_reply < shard_auxi->map_ver) + obj_auxi->map_ver_reply = shard_auxi->map_ver; if (ret == 0) { - if (obj_auxi->map_ver_reply < shard_auxi->map_ver) - obj_auxi->map_ver_reply = shard_auxi->map_ver; if (obj_req_is_ec_cond_fetch(obj_auxi)) { iter_arg->cond_fetch_exist = true; if (obj_auxi->result == -DER_NONEXIST) diff --git a/src/object/obj_rpc.c b/src/object/obj_rpc.c index 562c5c631bb..e8a0d202bdd 100644 --- a/src/object/obj_rpc.c +++ b/src/object/obj_rpc.c @@ -878,22 +878,11 @@ crt_proc_struct_daos_cpd_bulk(crt_proc_t proc, crt_proc_op_t proc_op, if (unlikely(rc)) return rc; - if (DECODING(proc_op)) { - D_ALLOC_PTR(dcb->dcb_bulk); - if (dcb->dcb_bulk == NULL) - return -DER_NOMEM; - } - - rc = crt_proc_crt_bulk_t(proc, proc_op, dcb->dcb_bulk); - if (unlikely(rc)) - return rc; - - if (FREEING(proc_op)) - D_FREE(dcb->dcb_bulk); + rc = crt_proc_crt_bulk_t(proc, proc_op, &dcb->dcb_bulk); /* The other fields will not be packed on-wire. */ - return 0; + return rc; } static int diff --git a/src/object/obj_rpc.h b/src/object/obj_rpc.h index d4fad534a5f..4c79eb7bef2 100644 --- a/src/object/obj_rpc.h +++ b/src/object/obj_rpc.h @@ -645,7 +645,7 @@ struct daos_cpd_bulk { struct daos_cpd_sub_head dcb_head; uint32_t dcb_size; uint32_t dcb_padding; - crt_bulk_t *dcb_bulk; + crt_bulk_t dcb_bulk; /* * The following are only used to handle the bulk for CPD RPC body temporarily, * do not pack on-wire. diff --git a/src/object/obj_tx.c b/src/object/obj_tx.c index 1e415246665..6194acd6507 100644 --- a/src/object/obj_tx.c +++ b/src/object/obj_tx.c @@ -393,6 +393,15 @@ dc_tx_alloc(daos_handle_t coh, daos_epoch_t epoch, uint64_t flags, return rc; } +static inline void +dc_tx_bulk_free(crt_bulk_t *hdl) +{ + if (hdl != NULL && *hdl != CRT_BULK_NULL) { + crt_bulk_free(*hdl); + *hdl = CRT_BULK_NULL; + } +} + static void dc_tx_cleanup_one(struct dc_tx *tx, struct daos_cpd_sub_req *dcsr) { @@ -407,11 +416,8 @@ dc_tx_cleanup_one(struct dc_tx *tx, struct daos_cpd_sub_req *dcsr) csummer = tx->tx_co->dc_csummer; if (dcu->dcu_flags & ORF_CPD_BULK) { - for (i = 0; i < dcsr->dcsr_nr; i++) { - if (dcu->dcu_bulks[i] != CRT_BULK_NULL) - crt_bulk_free(dcu->dcu_bulks[i]); - } - + for (i = 0; i < dcsr->dcsr_nr; i++) + dc_tx_bulk_free(&dcu->dcu_bulks[i]); D_FREE(dcu->dcu_bulks); } @@ -522,12 +528,7 @@ dc_tx_cleanup(struct dc_tx *tx) /* Keep 'tx_set_resend'. */ if (tx->tx_reqs.dcs_type == DCST_BULK_REQ) { - if (tx->tx_reqs_bulk.dcb_bulk != NULL) { - if (tx->tx_reqs_bulk.dcb_bulk[0] != CRT_BULK_NULL) - crt_bulk_free(tx->tx_reqs_bulk.dcb_bulk[0]); - D_FREE(tx->tx_reqs_bulk.dcb_bulk); - } - + dc_tx_bulk_free(&tx->tx_reqs_bulk.dcb_bulk); D_FREE(tx->tx_reqs_bulk.dcb_iov.iov_buf); } @@ -539,11 +540,7 @@ dc_tx_cleanup(struct dc_tx *tx) } if (tx->tx_head.dcs_type == DCST_BULK_HEAD) { - if (tx->tx_head_bulk.dcb_bulk != NULL) { - if (tx->tx_head_bulk.dcb_bulk[0] != CRT_BULK_NULL) - crt_bulk_free(tx->tx_head_bulk.dcb_bulk[0]); - D_FREE(tx->tx_head_bulk.dcb_bulk); - } + dc_tx_bulk_free(&tx->tx_head_bulk.dcb_bulk); /* Free MBS buffer. */ D_FREE(tx->tx_head_bulk.dcb_iov.iov_buf); } else { @@ -556,11 +553,7 @@ dc_tx_cleanup(struct dc_tx *tx) tx->tx_head.dcs_buf = NULL; if (tx->tx_disp.dcs_type == DCST_BULK_ENT) { - if (tx->tx_disp_bulk.dcb_bulk != NULL) { - if (tx->tx_disp_bulk.dcb_bulk[0] != CRT_BULK_NULL) - crt_bulk_free(tx->tx_disp_bulk.dcb_bulk[0]); - D_FREE(tx->tx_disp_bulk.dcb_bulk); - } + dc_tx_bulk_free(&tx->tx_disp_bulk.dcb_bulk); dcde = tx->tx_disp_bulk.dcb_iov.iov_buf; } else { dcde = tx->tx_disp.dcs_buf; @@ -576,11 +569,7 @@ dc_tx_cleanup(struct dc_tx *tx) } if (tx->tx_tgts.dcs_type == DCST_BULK_TGT) { - if (tx->tx_tgts_bulk.dcb_bulk != NULL) { - if (tx->tx_tgts_bulk.dcb_bulk[0] != CRT_BULK_NULL) - crt_bulk_free(tx->tx_tgts_bulk.dcb_bulk[0]); - D_FREE(tx->tx_tgts_bulk.dcb_bulk); - } + dc_tx_bulk_free(&tx->tx_tgts_bulk.dcb_bulk); D_FREE(tx->tx_tgts_bulk.dcb_iov.iov_buf); tx->tx_tgts.dcs_buf = NULL; } else { @@ -1823,15 +1812,23 @@ dc_tx_cpd_body_bulk(struct daos_cpd_sg *dcs, struct daos_cpd_bulk *dcb, dcb->dcb_sgl.sg_nr_out = 1; dcb->dcb_sgl.sg_iovs = &dcb->dcb_iov; - rc = obj_bulk_prep(&dcb->dcb_sgl, 1, true, CRT_BULK_RO, task, &dcb->dcb_bulk); - if (rc == 0) { - dcs->dcs_type = type; - dcs->dcs_nr = nr; - dcs->dcs_buf = dcb; - } else { + rc = crt_bulk_create(daos_task2ctx(task), &dcb->dcb_sgl, CRT_BULK_RO, &dcb->dcb_bulk); + if (rc != 0) + goto out; + + rc = crt_bulk_bind(dcb->dcb_bulk, daos_task2ctx(task)); + if (rc != 0) + goto out; + + dcs->dcs_type = type; + dcs->dcs_buf = dcb; + dcs->dcs_nr = nr; + +out: + if (rc != 0) { + dc_tx_bulk_free(&dcb->dcb_bulk); dcb->dcb_iov.iov_buf = NULL; } - return rc; } diff --git a/src/object/srv_ec_aggregate.c b/src/object/srv_ec_aggregate.c index b6fee91c619..c9f02618a5d 100644 --- a/src/object/srv_ec_aggregate.c +++ b/src/object/srv_ec_aggregate.c @@ -133,6 +133,7 @@ struct ec_agg_param { daos_handle_t ap_cont_handle; /* VOS container handle */ int (*ap_yield_func)(void *arg); /* yield function*/ void *ap_yield_arg; /* yield argument */ + struct ds_cont_child *ap_cont; uint32_t ap_credits_max; /* # of tight loops to yield */ uint32_t ap_credits; /* # of tight loops */ uint32_t ap_initialized:1; /* initialized flag */ @@ -146,8 +147,7 @@ struct ec_agg_stripe_ud { daos_recx_t *asu_recxs; /* For re-replicate */ unsigned int asu_cell_cnt; /* Count of cells */ bool asu_recalc; /* Should recalc parity */ - bool asu_write_par; /* Should write parity */ - struct daos_csummer *asu_csummer; + bool asu_write_par; /* Should write parity */ daos_iod_t asu_iod; d_iov_t asu_csum_iov; struct dcs_iod_csums *asu_iod_csums; /* iod csums */ @@ -787,12 +787,12 @@ agg_update_vos(struct ec_agg_param *agg_param, struct ec_agg_entry *entry, { daos_recx_t recx = { 0 }; daos_epoch_range_t epoch_range = { 0 }; - struct ec_agg_param *ap; - struct ec_agg_extent *ext; + struct ec_agg_param *ap; uint32_t len = ec_age2cs(entry); uint32_t pidx = ec_age2pidx(entry); struct daos_csummer *csummer; struct dcs_iod_csums *iod_csums = NULL; + int err; int rc = 0; ap = container_of(entry, struct ec_agg_param, ap_agg_entry); @@ -838,21 +838,16 @@ agg_update_vos(struct ec_agg_param *agg_param, struct ec_agg_entry *entry, } } - d_list_for_each_entry(ext, &entry->ae_cur_stripe.as_dextents, ae_link) { - int err; - - epoch_range.epr_lo = epoch_range.epr_hi = ext->ae_epoch; - err = vos_obj_array_remove(ap->ap_cont_handle, - entry->ae_oid, - &epoch_range, - &entry->ae_dkey, - &entry->ae_akey, - &ext->ae_recx); - if (err) - D_ERROR("array_remove fails: "DF_RC"\n", DP_RC(err)); - if (!rc && err) - rc = err; - } + recx.rx_idx = entry->ae_cur_stripe.as_stripenum * ec_age2ss(entry); + recx.rx_nr = ec_age2ss(entry); + epoch_range.epr_lo = ap->ap_epr.epr_lo; + epoch_range.epr_hi = entry->ae_cur_stripe.as_hi_epoch; + err = vos_obj_array_remove(ap->ap_cont_handle, entry->ae_oid, &epoch_range, &entry->ae_dkey, + &entry->ae_akey, &recx); + if (err) + D_ERROR("array_remove fails: " DF_RC "\n", DP_RC(err)); + if (!rc && err) + rc = err; out: return rc; } @@ -1225,6 +1220,7 @@ agg_process_partial_stripe(struct ec_agg_entry *entry) entry->ae_cur_stripe.as_stripenum, &full_cell_cnt); + has_old_replicas = true; /* XXX force full recalculation */ if (full_cell_cnt >= k / 2 || cell_cnt == k || has_old_replicas) { stripe_ud.asu_recalc = true; cell_cnt = full_cell_cnt; @@ -1283,6 +1279,62 @@ agg_process_partial_stripe(struct ec_agg_entry *entry) return rc; } +int +agg_peer_check_avail(struct ec_agg_param *agg_param, struct ec_agg_entry *entry) +{ + struct pool_target *targets = NULL; + struct daos_shard_loc *peer_loc; + uint32_t failed_tgts_cnt = 0; + uint32_t p = ec_age2p(entry); + uint32_t pidx = ec_age2pidx(entry); + int peer; + int i; + int rc; + + if (ds_pool_is_rebuilding(agg_param->ap_cont->sc_pool->spc_pool)) { + /* We currently pause EC aggregation for rebuild, so just cancel the + * aggregation for the current stripe. It means the following peer status + * check may not be checked at all, but let's keep the code because it could + * be useful in the future. + */ + D_ERROR(DF_UOID " pauses EC aggregation for rebuild\n", DP_UOID(entry->ae_oid)); + return -DER_OP_CANCELED; + } + + rc = pool_map_find_failed_tgts(agg_param->ap_pool_info.api_pool->sp_map, &targets, + &failed_tgts_cnt); + if (rc) { + D_ERROR(DF_UOID " pool_map_find_failed_tgts failed: " DF_RC "\n", + DP_UOID(entry->ae_oid), DP_RC(rc)); + return rc; + } + + if (targets == NULL) + return 0; + + for (peer = 0; peer < p; peer++) { + if (peer == pidx) + continue; + + peer_loc = &entry->ae_peer_pshards[peer]; + for (i = 0; i < failed_tgts_cnt; i++) { + if (peer_loc->sd_rank == DAOS_TGT_IGNORE || + (targets[i].ta_comp.co_rank == peer_loc->sd_rank && + targets[i].ta_comp.co_index == peer_loc->sd_tgt_idx)) { + D_DEBUG(DB_EPC, + DF_UOID " peer parity tgt failed rank %d, " + "tgt_idx %d.\n", + DP_UOID(entry->ae_oid), peer_loc->sd_rank, + peer_loc->sd_tgt_idx); + D_GOTO(out, rc = -DER_OP_CANCELED); + } + } + } +out: + D_FREE(targets); + return rc; +} + /* Sends the generated parity and the stripe number to the peer * parity target. Handler writes the parity and deletes the replicas * for the stripe. @@ -1300,7 +1352,7 @@ agg_peer_update_ult(void *arg) crt_endpoint_t tgt_ep = { 0 }; unsigned char *buf = NULL; struct obj_ec_agg_in *ec_agg_in = NULL; - struct obj_ec_agg_out *ec_agg_out; + struct obj_ec_agg_out *ec_agg_out = NULL; struct ec_agg_param *agg_param; crt_bulk_t bulk_hdl = NULL; struct daos_csummer *csummer = NULL; @@ -1312,34 +1364,42 @@ agg_peer_update_ult(void *arg) uint32_t peer, peer_shard; struct dc_object *obj = NULL; crt_rpc_t *rpc = NULL; - int rc = 0; - uint64_t enqueue_id; + int peer_rc = 0, rc = 0; + int peer_updated = 0; uint32_t max_delay = 0; if (unlikely(DAOS_FAIL_CHECK(DAOS_FORCE_EC_AGG_PEER_FAIL))) D_GOTO(out, rc = -DER_TIMEDOUT); agg_param = container_of(entry, struct ec_agg_param, ap_agg_entry); + rc = agg_peer_check_avail(agg_param, entry); + if (rc != 0) + D_GOTO(out, rc); + csummer = ec_agg_param2csummer(agg_param); iod.iod_type = DAOS_IOD_ARRAY; iod.iod_name = entry->ae_akey; iod.iod_size = entry->ae_rsize; obj = obj_hdl2ptr(entry->ae_obj_hdl); for (peer = 0; peer < p; peer++) { + uint64_t enqueue_id = 0; + bool overloaded; + if (peer == pidx) continue; D_ASSERT(entry->ae_peer_pshards[peer].sd_rank != DAOS_TGT_IGNORE); tgt_ep.ep_rank = entry->ae_peer_pshards[peer].sd_rank; - tgt_ep.ep_tag = entry->ae_peer_pshards[peer].sd_tgt_idx; - enqueue_id = 0; + tgt_ep.ep_tag = entry->ae_peer_pshards[peer].sd_tgt_idx; retry: + overloaded = false; rc = obj_req_create(dss_get_module_info()->dmi_ctx, &tgt_ep, DAOS_OBJ_RPC_EC_AGGREGATE, &rpc); if (rc) { D_ERROR(DF_UOID" pidx %d to peer %d, rank %d tag %d obj_req_create " DF_RC"\n", DP_UOID(entry->ae_oid), pidx, peer, tgt_ep.ep_rank, tgt_ep.ep_tag, DP_RC(rc)); - goto out; + if (peer_updated == 0) + goto next; } ec_agg_in = crt_req_get(rpc); uuid_copy(ec_agg_in->ea_pool_uuid, @@ -1380,26 +1440,21 @@ agg_peer_update_ult(void *arg) rc = crt_bulk_create(dss_get_module_info()->dmi_ctx, &sgl, CRT_BULK_RW, &bulk_hdl); if (rc) { - D_ERROR(DF_UOID" pidx %d to peer %d, " - "crt_bulk_create "DF_RC"\n", - DP_UOID(entry->ae_oid), pidx, peer, - DP_RC(rc)); - goto out_rpc; + D_ERROR(DF_UOID " pidx %d to peer %d, crt_bulk_create " DF_RC "\n", + DP_UOID(entry->ae_oid), pidx, peer, DP_RC(rc)); + goto next; } ec_agg_in->ea_bulk = bulk_hdl; if (csummer != NULL) { - rc = daos_csummer_calc_iods(csummer, &sgl, &iod, - NULL, 1, false, - NULL, 0, - &iod_csums); + rc = daos_csummer_calc_iods(csummer, &sgl, &iod, NULL, 1, false, + NULL, 0, &iod_csums); if (rc) { - D_ERROR("daos_csummer_calc_iods " - "failed: "DF_RC"\n", DP_RC(rc)); - goto out; + D_ERROR("daos_csummer_calc_iods failed: " DF_RC "\n", + DP_RC(rc)); + goto next; } D_ASSERT(iod_csums != NULL); - ec_agg_in->ea_iod_csums.ca_arrays = iod_csums; ec_agg_in->ea_iod_csums.ca_count = 1; } @@ -1408,46 +1463,47 @@ agg_peer_update_ult(void *arg) rc = dss_rpc_send(rpc); if (rc) { - D_ERROR(DF_UOID" pidx %d to peer %d, dss_rpc_send " - DF_RC"\n", DP_UOID(entry->ae_oid), pidx, peer, - DP_RC(rc)); - goto out_bulk; + D_ERROR(DF_UOID " pidx %d to parity[%d], dss_rpc_send " DF_RC "\n", + DP_UOID(entry->ae_oid), pidx, peer, DP_RC(rc)); + } else { + crt_req_get_timeout(rpc, &max_delay); + ec_agg_out = crt_reply_get(rpc); + rc = ec_agg_out->ea_status; + if (rc == -DER_OVERLOAD_RETRY) { + enqueue_id = ec_agg_out->ea_comm_out.req_out_enqueue_id; + overloaded = true; + } + D_CDEBUG(rc == 0, DB_TRACE, DLOG_ERR, + "update parity[%d] to %d:%d, status = " DF_RC "\n", peer, + tgt_ep.ep_rank, tgt_ep.ep_tag, DP_RC(rc)); + peer_updated += rc == 0; } - crt_req_get_timeout(rpc, &max_delay); - ec_agg_out = crt_reply_get(rpc); - rc = ec_agg_out->ea_status; - if (bulk_hdl) { +next: + if (bulk_hdl) crt_bulk_free(bulk_hdl); - bulk_hdl = NULL; - } - D_DEBUG(DB_TRACE, "send DAOS_OBJ_RPC_EC_AGGREGATE to %d:%d, peer %d, rc %d\n", - tgt_ep.ep_rank, tgt_ep.ep_tag, peer, rc); - if (csummer != NULL && iod_csums != NULL) + if (rpc) + crt_req_decref(rpc); + if (iod_csums != NULL) daos_csummer_free_ic(csummer, &iod_csums); - crt_req_decref(rpc); rpc = NULL; - if (rc) { - D_ERROR(DF_UOID" pidx %d to peer %d, ea_status " - DF_RC"\n", DP_UOID(entry->ae_oid), pidx, peer, - DP_RC(rc)); - if (rc == -DER_OVERLOAD_RETRY) { - enqueue_id = ec_agg_out->ea_comm_out.req_out_enqueue_id; - dss_sleep(daos_rpc_rand_delay(max_delay) << 10); - goto retry; - } - break; + bulk_hdl = NULL; + iod_csums = NULL; + if (overloaded) { + dss_sleep(daos_rpc_rand_delay(max_delay) << 10); + goto retry; } + peer_rc = peer_rc ?: rc; + if (peer_updated == 0 && peer_rc != 0) + break; /* otherwise continue because no DTX no rollback */ } - -out_bulk: - if (bulk_hdl) - crt_bulk_free(bulk_hdl); -out_rpc: - if (rpc) - crt_req_decref(rpc); out: if (obj) obj_decref(obj); + /* NB: before switching to DTX, any successful parity write is deemed as success + * of all parity shards. + * NB: it's OK if the only remote parity peer failed. + */ + rc = peer_updated > 0 ? 0 : peer_rc; ABT_eventual_set(stripe_ud->asu_eventual, (void *)&rc, sizeof(rc)); } @@ -1457,28 +1513,11 @@ agg_peer_update_ult(void *arg) static int agg_peer_update(struct ec_agg_entry *entry, bool write_parity) { - struct ec_agg_stripe_ud stripe_ud = { 0 }; - struct ec_agg_param *agg_param; - struct pool_target *targets; + struct ec_agg_stripe_ud stripe_ud = {0}; int *status; - struct daos_shard_loc *peer_loc; - uint32_t failed_tgts_cnt = 0; - uint32_t p = ec_age2p(entry); - uint32_t pidx = ec_age2pidx(entry); - uint32_t peer; - int i, tid, rc = 0; - - D_ASSERT(!write_parity || - entry->ae_sgl.sg_iovs[AGG_IOV_PARITY].iov_buf); - agg_param = container_of(entry, struct ec_agg_param, ap_agg_entry); + int tid, rc = 0; - /* If rebuild started, abort it before sending RPC to save conflict window with rebuild - * (see obj_inflight_io_check()). - */ - if (agg_param->ap_pool_info.api_pool->sp_rebuilding > 0) { - D_DEBUG(DB_EPC, DF_UOID" abort as rebuild started\n", DP_UOID(entry->ae_oid)); - return -1; - } + D_ASSERT(!write_parity || entry->ae_sgl.sg_iovs[AGG_IOV_PARITY].iov_buf); rc = agg_get_obj_handle(entry, false); if (rc) { @@ -1486,56 +1525,28 @@ agg_peer_update(struct ec_agg_entry *entry, bool write_parity) return rc; } - rc = pool_map_find_failed_tgts(agg_param->ap_pool_info.api_pool->sp_map, - &targets, &failed_tgts_cnt); - if (rc) { - D_ERROR(DF_UOID" pool_map_find_failed_tgts failed: "DF_RC"\n", - DP_UOID(entry->ae_oid), DP_RC(rc)); - return rc; - } - - if (targets != NULL) { - for (peer = 0; peer < p; peer++) { - if (peer == pidx) - continue; - peer_loc = &entry->ae_peer_pshards[peer]; - for (i = 0; i < failed_tgts_cnt; i++) { - if (peer_loc->sd_rank == DAOS_TGT_IGNORE || - (targets[i].ta_comp.co_rank == peer_loc->sd_rank && - targets[i].ta_comp.co_index == peer_loc->sd_tgt_idx)) { - D_DEBUG(DB_EPC, DF_UOID" peer parity tgt failed rank %d, " - "tgt_idx %d.\n", DP_UOID(entry->ae_oid), - peer_loc->sd_rank, peer_loc->sd_tgt_idx); - D_GOTO(out, rc = -1); - } - } - } - } - stripe_ud.asu_write_par = write_parity; stripe_ud.asu_agg_entry = entry; rc = ABT_eventual_create(sizeof(*status), &stripe_ud.asu_eventual); if (rc != ABT_SUCCESS) { rc = dss_abterr2der(rc); - goto out; + return rc; } tid = dss_get_module_info()->dmi_tgt_id; rc = dss_ult_create(agg_peer_update_ult, &stripe_ud, DSS_XS_IOFW, tid, 0, NULL); if (rc) - goto ev_out; + goto out; rc = ABT_eventual_wait(stripe_ud.asu_eventual, (void **)&status); if (rc != ABT_SUCCESS) { rc = dss_abterr2der(rc); - goto ev_out; + goto out; } if (*status != 0) rc = *status; -ev_out: - ABT_eventual_free(&stripe_ud.asu_eventual); out: - D_FREE(targets); + ABT_eventual_free(&stripe_ud.asu_eventual); return rc; } @@ -1547,8 +1558,7 @@ agg_process_holes_ult(void *arg) struct ec_agg_stripe_ud *stripe_ud = (struct ec_agg_stripe_ud *)arg; daos_iod_t *iod = &stripe_ud->asu_iod; struct ec_agg_entry *entry = stripe_ud->asu_agg_entry; - struct ec_agg_extent *agg_extent; - struct pool_target *targets = NULL; + struct ec_agg_extent *agg_extent; struct ec_agg_param *agg_param; struct obj_ec_rep_in *ec_rep_in = NULL; struct obj_ec_rep_out *ec_rep_out = NULL; @@ -1563,13 +1573,12 @@ agg_process_holes_ult(void *arg) k * len; uint64_t last_ext_end = 0; uint64_t ext_cnt = 0; - uint64_t ext_tot_len = 0; - uint32_t failed_tgts_cnt = 0; + uint64_t ext_tot_len = 0; uint32_t pidx = ec_age2pidx(entry); uint32_t peer; - int i, rc = 0; - uint32_t max_delay = 0; - uint64_t enqueue_id; + int rc = 0, peer_rc = 0; + int peer_updated = 0; + uint32_t max_delay = 0; stripe_ud->asu_valid_hole = false; /* Process extent list to find what to re-replicate -- build recx array @@ -1581,13 +1590,10 @@ agg_process_holes_ult(void *arg) if (agg_extent->ae_hole) stripe_ud->asu_valid_hole = true; if (agg_extent->ae_recx.rx_idx - ss > last_ext_end) { - stripe_ud->asu_recxs[ext_cnt].rx_idx = - ss + last_ext_end; + stripe_ud->asu_recxs[ext_cnt].rx_idx = ss + last_ext_end; stripe_ud->asu_recxs[ext_cnt].rx_nr = - agg_extent->ae_recx.rx_idx - ss - - last_ext_end; - ext_tot_len += - stripe_ud->asu_recxs[ext_cnt++].rx_nr; + agg_extent->ae_recx.rx_idx - ss - last_ext_end; + ext_tot_len += stripe_ud->asu_recxs[ext_cnt++].rx_nr; } last_ext_end = agg_extent->ae_recx.rx_idx + agg_extent->ae_recx.rx_nr - ss; @@ -1598,6 +1604,11 @@ agg_process_holes_ult(void *arg) if (!stripe_ud->asu_valid_hole) goto out; + agg_param = container_of(entry, struct ec_agg_param, ap_agg_entry); + rc = agg_peer_check_avail(agg_param, entry); + if (rc != 0) + goto out; + obj = obj_hdl2ptr(entry->ae_obj_hdl); if (last_ext_end < k * len) { stripe_ud->asu_recxs[ext_cnt].rx_idx = ss + last_ext_end; @@ -1614,7 +1625,6 @@ agg_process_holes_ult(void *arg) entry->ae_sgl.sg_iovs[AGG_IOV_DATA].iov_len = ext_tot_len * entry->ae_rsize; D_ASSERT(entry->ae_sgl.sg_iovs[AGG_IOV_DATA].iov_len <= k * cell_b); - agg_param = container_of(entry, struct ec_agg_param, ap_agg_entry); /* Pull data via dsc_obj_fetch */ if (ext_cnt) { struct daos_csummer *csummer; @@ -1624,6 +1634,7 @@ agg_process_holes_ult(void *arg) NULL, DIOF_FOR_EC_AGG, NULL, NULL); if (rc) { D_ERROR("dsc_obj_fetch failed: "DF_RC"\n", DP_RC(rc)); + D_GOTO(out, rc); goto out; } @@ -1632,6 +1643,7 @@ agg_process_holes_ult(void *arg) NULL, -1, &stripe_ud->asu_iod_csums); if (rc != 0) { D_ERROR("setting up iods csums failed: "DF_RC"\n", DP_RC(rc)); + D_GOTO(out, rc); goto out; } } @@ -1645,39 +1657,22 @@ agg_process_holes_ult(void *arg) if (rc) { D_ERROR(DF_UOID" crt_bulk_create returned: "DF_RC"\n", DP_UOID(entry->ae_oid), DP_RC(rc)); + D_GOTO(out, rc); goto out; } } - rc = pool_map_find_failed_tgts(agg_param->ap_pool_info.api_pool->sp_map, - &targets, &failed_tgts_cnt); - if (rc) { - D_ERROR(DF_UOID" pool_map_find_failed_tgts failed: " - DF_RC"\n", DP_UOID(entry->ae_oid), DP_RC(rc)); - goto out; - } - /* Invoke peer re-replicate */ for (peer = 0; peer < p; peer++) { + uint64_t enqueue_id = 0; uint32_t peer_shard; + bool overloaded; if (pidx == peer) continue; - for (i = 0; targets && i < failed_tgts_cnt; i++) { - if (entry->ae_peer_pshards[peer].sd_rank == DAOS_TGT_IGNORE || - (targets[i].ta_comp.co_rank == entry->ae_peer_pshards[peer].sd_rank && - targets[i].ta_comp.co_index == - entry->ae_peer_pshards[peer].sd_tgt_idx)) { - D_ERROR(DF_UOID" peer %d parity tgt failed\n", - DP_UOID(entry->ae_oid), peer); - rc = -1; - goto out; - } - } - - enqueue_id = 0; retry: + overloaded = false; D_ASSERT(entry->ae_peer_pshards[peer].sd_rank != DAOS_TGT_IGNORE); tgt_ep.ep_rank = entry->ae_peer_pshards[peer].sd_rank; tgt_ep.ep_tag = entry->ae_peer_pshards[peer].sd_tgt_idx; @@ -1686,7 +1681,11 @@ agg_process_holes_ult(void *arg) if (rc) { D_ERROR(DF_UOID" obj_req_create failed: "DF_RC"\n", DP_UOID(entry->ae_oid), DP_RC(rc)); - goto out; + peer_rc = peer_rc ?: rc; + if (peer_updated == 0) + break; + else + continue; } ec_rep_in = crt_req_get(rpc); uuid_copy(ec_rep_in->er_pool_uuid, @@ -1711,38 +1710,43 @@ agg_process_holes_ult(void *arg) ec_rep_in->er_map_ver = agg_param->ap_pool_info.api_pool->sp_map_version; ec_rep_in->er_bulk = bulk_hdl; ec_rep_in->er_comm_in.req_in_enqueue_id = enqueue_id; - crt_req_get_timeout(rpc, &max_delay); rc = dss_rpc_send(rpc); if (rc) { D_ERROR(DF_UOID" peer %d dss_rpc_send failed "DF_RC"\n", DP_UOID(entry->ae_oid), peer, DP_RC(rc)); - goto out; - } - ec_rep_out = crt_reply_get(rpc); - rc = ec_rep_out->er_status; - crt_req_decref(rpc); - rpc = NULL; - if (rc) { - D_ERROR(DF_UOID" peer %d er_status failed "DF_RC"\n", - DP_UOID(entry->ae_oid), peer, DP_RC(rc)); + } else { + crt_req_get_timeout(rpc, &max_delay); + ec_rep_out = crt_reply_get(rpc); + rc = ec_rep_out->er_status; if (rc == -DER_OVERLOAD_RETRY) { enqueue_id = ec_rep_out->er_comm_out.req_out_enqueue_id; - dss_sleep(daos_rpc_rand_delay(max_delay) << 10); - goto retry; + overloaded = true; } - break; + D_CDEBUG(rc == 0, DB_TRACE, DLOG_ERR, + DF_UOID " parity[%d] er_status = " DF_RC "\n", + DP_UOID(entry->ae_oid), peer, DP_RC(rc)); + peer_updated += rc == 0; + } + crt_req_decref(rpc); + rpc = NULL; + if (overloaded) { + dss_sleep(daos_rpc_rand_delay(max_delay) << 10); + goto retry; } + peer_rc = peer_rc ?: rc; + if (peer_updated == 0 && peer_rc != 0) + break; /* otherwise continue because no DTX no rollback */ } - out: - D_FREE(targets); - if (rpc) - crt_req_decref(rpc); if (bulk_hdl) crt_bulk_free(bulk_hdl); if (obj) obj_decref(obj); entry->ae_sgl.sg_nr = AGG_IOV_CNT; + /* NB: before switching to DTX, any successful parity write is deemed as success + * of all parity shards. + */ + rc = peer_updated > 0 ? 0 : peer_rc; ABT_eventual_set(stripe_ud->asu_eventual, (void *)&rc, sizeof(rc)); } @@ -1761,11 +1765,6 @@ agg_process_holes(struct ec_agg_entry *entry) /* If rebuild started, abort it before sending RPC to save conflict window with rebuild * (see obj_inflight_io_check()). */ - if (agg_param->ap_pool_info.api_pool->sp_rebuilding > 0) { - D_DEBUG(DB_EPC, DF_UOID" abort as rebuild started\n", DP_UOID(entry->ae_oid)); - return -1; - } - D_ALLOC_ARRAY(stripe_ud.asu_recxs, entry->ae_cur_stripe.as_extent_cnt + 1); if (stripe_ud.asu_recxs == NULL) { @@ -1789,6 +1788,7 @@ agg_process_holes(struct ec_agg_entry *entry) goto out; } tid = dss_get_module_info()->dmi_tgt_id; + rc = dss_ult_create(agg_process_holes_ult, &stripe_ud, DSS_XS_IOFW, tid, 0, NULL); if (rc) @@ -1843,11 +1843,9 @@ agg_process_holes(struct ec_agg_entry *entry) ABT_eventual_free(&stripe_ud.asu_eventual); out: - D_FREE(stripe_ud.asu_recxs); - daos_csummer_free_ic(stripe_ud.asu_csummer, &stripe_ud.asu_iod_csums); + daos_csummer_free_ic(ec_agg_param2csummer(agg_param), &stripe_ud.asu_iod_csums); D_FREE(stripe_ud.asu_csum_iov.iov_buf); - daos_csummer_destroy(&stripe_ud.asu_csummer); - + D_FREE(stripe_ud.asu_recxs); return rc; } @@ -1863,6 +1861,15 @@ agg_process_stripe(struct ec_agg_param *agg_param, struct ec_agg_entry *entry) bool write_parity = true; int rc = 0; + /* avoid race between EC aggregation and rebuild scanner */ + agg_param->ap_cont->sc_ec_agg_updates++; + if (ds_pool_is_rebuilding(agg_param->ap_cont->sc_pool->spc_pool)) { + D_DEBUG(DB_EPC, DF_UOID " abort as rebuild started\n", DP_UOID(entry->ae_oid)); + update_vos = false; + rc = -1; + goto out; + } + if (DAOS_FAIL_CHECK(DAOS_FORCE_EC_AGG_FAIL)) D_GOTO(out, rc = -DER_DATA_LOSS); @@ -1940,12 +1947,13 @@ agg_process_stripe(struct ec_agg_param *agg_param, struct ec_agg_entry *entry) if (ec_age_with_hole(entry)) { entry->ae_process_partial = 0; rc = agg_process_holes(entry); - if (rc != 0 || !entry->ae_process_partial) - goto clear_exts; + if (rc != 0 || !entry->ae_process_partial) { + update_vos = false; + goto out; + } } rc = agg_process_partial_stripe(entry); - out: if (update_vos && rc == 0) { if (ec_age2p(entry) > 1) { @@ -1963,8 +1971,7 @@ agg_process_stripe(struct ec_agg_param *agg_param, struct ec_agg_entry *entry) write_parity); } } - -clear_exts: + agg_param->ap_cont->sc_ec_agg_updates--; agg_clear_extents(entry); return rc; } @@ -2281,7 +2288,7 @@ ec_aggregate_yield(struct ec_agg_param *agg_param) { int rc; - if (agg_param->ap_pool_info.api_pool->sp_rebuilding > 0) { + if (ds_pool_is_rebuilding(agg_param->ap_pool_info.api_pool)) { D_INFO(DF_UUID": abort ec aggregation, sp_rebuilding %d\n", DP_UUID(agg_param->ap_pool_info.api_pool->sp_uuid), agg_param->ap_pool_info.api_pool->sp_rebuilding); @@ -2487,7 +2494,7 @@ agg_iterate_pre_cb(daos_handle_t ih, vos_iter_entry_t *entry, /* If rebuild started, abort it to save conflict window with rebuild * (see obj_inflight_io_check()). */ - if (agg_param->ap_pool_info.api_pool->sp_rebuilding > 0) { + if (ds_pool_is_rebuilding(agg_param->ap_pool_info.api_pool)) { D_INFO(DF_CONT" abort as rebuild started, sp_rebuilding %d\n", DP_CONT(agg_param->ap_pool_info.api_pool_uuid, agg_param->ap_pool_info.api_cont_uuid), @@ -2648,6 +2655,7 @@ ec_agg_param_init(struct ds_cont_child *cont, struct agg_param *param) uuid_copy(info->api_cont_uuid, cont->sc_uuid); info->api_pool = cont->sc_pool->spc_pool; + agg_param->ap_cont = cont; agg_param->ap_cont_handle = cont->sc_hdl; agg_param->ap_yield_func = agg_rate_ctl; agg_param->ap_yield_arg = param; @@ -2788,12 +2796,7 @@ cont_ec_aggregate_cb(struct ds_cont_child *cont, daos_epoch_range_t *epr, agg_clear_extents(&ec_agg_param->ap_agg_entry); agg_reset_entry(&ec_agg_param->ap_agg_entry, NULL, NULL); - if (daos_handle_is_valid(ec_agg_param->ap_agg_entry.ae_obj_hdl)) { - dsc_obj_close(ec_agg_param->ap_agg_entry.ae_obj_hdl); - ec_agg_param->ap_agg_entry.ae_obj_hdl = DAOS_HDL_INVAL; - } - - if (rc == -DER_BUSY && cont->sc_pool->spc_pool->sp_rebuilding == 0) { + if (rc == -DER_BUSY && !ds_pool_is_rebuilding(cont->sc_pool->spc_pool)) { /** Hit an object conflict VOS aggregation or discard. Rather than exiting, let's * yield and try again. */ @@ -2812,7 +2815,7 @@ cont_ec_aggregate_cb(struct ds_cont_child *cont, daos_epoch_range_t *epr, /* clear the flag before next turn's cont_aggregate_runnable(), to save conflict * window with rebuild (see obj_inflight_io_check()). */ - if (cont->sc_pool->spc_pool->sp_rebuilding > 0) + if (ds_pool_is_rebuilding(cont->sc_pool->spc_pool)) cont->sc_ec_agg_active = 0; if (rc == 0) { diff --git a/src/object/srv_obj.c b/src/object/srv_obj.c index 620e63aef65..96666370d94 100644 --- a/src/object/srv_obj.c +++ b/src/object/srv_obj.c @@ -2450,7 +2450,8 @@ obj_inflight_io_check(struct ds_cont_child *child, uint32_t opc, uint32_t rpc_map_ver, uint32_t flags) { if (opc == DAOS_OBJ_RPC_ENUMERATE && flags & ORF_FOR_MIGRATION) { - if (child->sc_ec_agg_active) { + /* EC aggregation is still inflight, rebuild should wait until it's paused */ + if (ds_cont_child_ec_aggregating(child)) { D_ERROR(DF_CONT" ec aggregate still active, rebuilding %d\n", DP_CONT(child->sc_pool->spc_uuid, child->sc_uuid), child->sc_pool->spc_pool->sp_rebuilding); @@ -2566,6 +2567,8 @@ ds_obj_ec_rep_handler(crt_rpc_t *rpc) } D_ASSERT(ioc.ioc_coc != NULL); + ioc.ioc_coc->sc_ec_agg_updates++; + dkey = (daos_key_t *)&oer->er_dkey; iod = (daos_iod_t *)&oer->er_iod; if (iod->iod_nr == 0) /* nothing to replicate, directly remove parity */ @@ -2576,7 +2579,7 @@ ds_obj_ec_rep_handler(crt_rpc_t *rpc) if (rc) { D_ERROR(DF_UOID" Update begin failed: "DF_RC"\n", DP_UOID(oer->er_oid), DP_RC(rc)); - goto out; + goto out_agg; } biod = vos_ioh2desc(ioh); rc = bio_iod_prep(biod, BIO_CHK_TYPE_IO, rpc->cr_ctx, CRT_BULK_RW); @@ -2600,13 +2603,15 @@ ds_obj_ec_rep_handler(crt_rpc_t *rpc) if (rc) { D_ERROR(DF_UOID " vos_update_end failed: " DF_RC "\n", DP_UOID(oer->er_oid), DP_RC(rc)); - goto out; + goto out_agg; } remove_parity: recx.rx_nr = obj_ioc2ec_cs(&ioc); recx.rx_idx = (oer->er_stripenum * recx.rx_nr) | PARITY_INDICATOR; rc = vos_obj_array_remove(ioc.ioc_coc->sc_hdl, oer->er_oid, &oer->er_epoch_range, dkey, &iod->iod_name, &recx); +out_agg: + ioc.ioc_coc->sc_ec_agg_updates--; out: obj_rw_reply(rpc, rc, 0, false, &ioc); obj_ioc_end(&ioc, rc); @@ -2646,6 +2651,8 @@ ds_obj_ec_agg_handler(crt_rpc_t *rpc) } D_ASSERT(ioc.ioc_coc != NULL); + ioc.ioc_coc->sc_ec_agg_updates++; + dkey = (daos_key_t *)&oea->ea_dkey; if (parity_bulk != CRT_BULK_NULL) { rc = vos_update_begin(ioc.ioc_coc->sc_hdl, oea->ea_oid, oea->ea_epoch_range.epr_hi, @@ -2653,7 +2660,7 @@ ds_obj_ec_agg_handler(crt_rpc_t *rpc) if (rc) { D_ERROR(DF_UOID" Update begin failed: "DF_RC"\n", DP_UOID(oea->ea_oid), DP_RC(rc)); - goto out; + goto out_agg; } biod = vos_ioh2desc(ioh); rc = bio_iod_prep(biod, BIO_CHK_TYPE_IO, rpc->cr_ctx, @@ -2686,7 +2693,7 @@ ds_obj_ec_agg_handler(crt_rpc_t *rpc) } else { D_ERROR(DF_UOID " vos_update_end failed: " DF_RC "\n", DP_UOID(oea->ea_oid), DP_RC(rc)); - D_GOTO(out, rc); + D_GOTO(out_agg, rc); } } } @@ -2703,6 +2710,8 @@ ds_obj_ec_agg_handler(crt_rpc_t *rpc) if (rc1) D_ERROR(DF_UOID ": array_remove failed: " DF_RC "\n", DP_UOID(oea->ea_oid), DP_RC(rc1)); +out_agg: + ioc.ioc_coc->sc_ec_agg_updates--; out: obj_rw_reply(rpc, rc, 0, false, &ioc); obj_ioc_end(&ioc, rc); @@ -3390,8 +3399,14 @@ obj_local_enum(struct obj_io_context *ioc, crt_rpc_t *rpc, goto failed; } - if (oei->oei_flags & ORF_FOR_MIGRATION) + if (oei->oei_flags & ORF_FOR_MIGRATION) { + /* just in case ds_pool::sp_rebuilding is not set, pause my local EC aggregation + * by setting this flag. + * NB: it's a lockess write to shared data structure and it's harmless. + */ + ioc->ioc_coc->sc_pool->spc_pool->sp_rebuild_scan = 1; flags = DTX_FOR_MIGRATION; + } rc = dtx_begin(ioc->ioc_vos_coh, &oei->oei_dti, &epoch, 0, oei->oei_map_ver, &oei->oei_oid, NULL, 0, flags, @@ -5276,11 +5291,7 @@ ds_obj_cpd_body_bulk(crt_rpc_t *rpc, struct obj_io_context *ioc, bool leader, struct daos_cpd_bulk **dcbs = NULL; struct daos_cpd_bulk *dcb = NULL; crt_bulk_t *bulks = NULL; - d_sg_list_t **sgls = NULL; - struct daos_cpd_sub_head *dcsh; - struct daos_cpd_disp_ent *dcde; - struct daos_cpd_req_idx *dcri; - void *end; + d_sg_list_t **sgls = NULL; uint32_t total = 0; uint32_t count = 0; int rc = 0; @@ -5363,21 +5374,31 @@ ds_obj_cpd_body_bulk(crt_rpc_t *rpc, struct obj_io_context *ioc, bool leader, D_GOTO(out, rc = -DER_NOMEM); for (i = 0; i < count; i++) { - bulks[i] = *dcbs[i]->dcb_bulk; + bulks[i] = dcbs[i]->dcb_bulk; sgls[i] = &dcbs[i]->dcb_sgl; } - rc = obj_bulk_transfer(rpc, CRT_BULK_GET, ORF_BULK_BIND, bulks, NULL, NULL, DAOS_HDL_INVAL, - sgls, count, count, NULL, check_conn(NULL, leader)); + rc = obj_bulk_transfer(rpc, CRT_BULK_GET, true, bulks, NULL, NULL, DAOS_HDL_INVAL, sgls, + count, count, NULL, check_conn(NULL, leader)); if (rc != 0) goto out; for (i = 0; i < count; i++) { + D_ASSERTF(dcbs[i]->dcb_size == dcbs[i]->dcb_iov.iov_len, "Bad bulk: %u vs %u\n", + dcbs[i]->dcb_size, (uint32_t)dcbs[i]->dcb_iov.iov_len); + switch (dcbs[i]->dcb_type) { - case DCST_BULK_HEAD: - dcsh = &dcbs[i]->dcb_head; + case DCST_BULK_HEAD: { + struct daos_cpd_sub_head *dcsh = &dcbs[i]->dcb_head; + dcsh->dcsh_mbs = dcbs[i]->dcb_iov.iov_buf; + D_ASSERTF(dcsh->dcsh_mbs->dm_data_size == + dcbs[i]->dcb_iov.iov_len - sizeof(*dcsh->dcsh_mbs), + "Invalid bulk MBS data for CPD RPC on %s: %u vs %u\n", + leader ? "leader" : "follower", dcsh->dcsh_mbs->dm_data_size, + (uint32_t)(dcbs[i]->dcb_iov.iov_len - sizeof(*dcsh->dcsh_mbs))); break; + } case DCST_BULK_REQ: rc = crt_proc_create(dss_get_module_info()->dmi_ctx, dcbs[i]->dcb_iov.iov_buf, dcbs[i]->dcb_iov.iov_len, @@ -5397,17 +5418,36 @@ ds_obj_cpd_body_bulk(crt_rpc_t *rpc, struct obj_io_context *ioc, bool leader, goto out; } break; - case DCST_BULK_ENT: + case DCST_BULK_ENT: { + struct daos_cpd_disp_ent *dcde; + struct daos_cpd_req_idx *dcri; + void *end; + dcde = dcbs[i]->dcb_iov.iov_buf; dcri = dcbs[i]->dcb_iov.iov_buf + sizeof(*dcde) * dcbs[i]->dcb_item_nr; - end = dcbs[i]->dcb_iov.iov_buf + dcbs[i]->dcb_iov.iov_len; + end = dcbs[i]->dcb_iov.iov_buf + dcbs[i]->dcb_iov.iov_len; for (j = 0; j < dcbs[i]->dcb_item_nr; j++) { dcde[j].dcde_reqs = dcri; dcri += dcde[j].dcde_read_cnt + dcde[j].dcde_write_cnt; - D_ASSERT((void *)dcri <= end); + D_ASSERTF((void *)dcri <= end, + "Invalid bulk ENT data for CPD RPC on %s: pos %u, rd %u, " + "wr %u, nr %u, size %u, base %p, cur %p, end %p\n", + leader ? "leader" : "follower", j, dcde[j].dcde_read_cnt, + dcde[j].dcde_write_cnt, dcbs[i]->dcb_item_nr, + dcbs[i]->dcb_size, dcde, dcri, end); } break; + } + case DCST_BULK_TGT: { + struct daos_shard_tgt *tgts = dcbs[i]->dcb_iov.iov_buf; + + D_ASSERTF(dss_self_rank() == tgts[0].st_rank && + dss_get_module_info()->dmi_tgt_id == tgts[0].st_tgt_idx, + "Invalid bulk tgt data on %u: rank %u, tgt_idx %u\n", + dss_self_rank(), tgts[0].st_rank, tgts[0].st_tgt_idx); + break; + } default: break; } diff --git a/src/object/srv_obj_migrate.c b/src/object/srv_obj_migrate.c index 2a91c7f1722..1d5715260eb 100644 --- a/src/object/srv_obj_migrate.c +++ b/src/object/srv_obj_migrate.c @@ -2865,6 +2865,7 @@ migrate_one_epoch_object(daos_epoch_range_t *epr, struct migrate_pool_tls *tls, uint32_t minimum_nr; uint32_t enum_flags; uint32_t num; + int waited = 0; int rc = 0; D_DEBUG(DB_REBUILD, "migrate obj "DF_UOID" for shard %u eph " @@ -3028,7 +3029,10 @@ migrate_one_epoch_object(daos_epoch_range_t *epr, struct migrate_pool_tls *tls, /* -DER_UPDATE_AGAIN means the remote target does not parse EC * aggregation yet, so let's retry. */ - D_DEBUG(DB_REBUILD, DF_UOID "retry with %d \n", DP_UOID(arg->oid), rc); + waited++; + dss_sleep(5000); + D_DEBUG(DB_REBUILD, DF_UOID "retry %d secs with %d \n", DP_UOID(arg->oid), + waited * 5, rc); rc = 0; continue; } else if (rc) { diff --git a/src/pool/cli.c b/src/pool/cli.c index ad504861574..17a7f2a63ea 100644 --- a/src/pool/cli.c +++ b/src/pool/cli.c @@ -910,6 +910,7 @@ pool_connect_cp(tse_task_t *task, void *data) d_iov_t *credp; bool free_tpriv = true; int rc = task->dt_result; + struct daos_rebuild_status *rs; rc = pool_rsvc_client_complete_rpc(tpriv->pool, &arg->rpc->cr_ep, rc, &pco->pco_op, task); if (rc < 0) { @@ -941,9 +942,14 @@ pool_connect_cp(tse_task_t *task, void *data) D_GOTO(out, rc); } + rs = &pco->pco_rebuild_st; rc = process_query_reply(tpriv->pool, map_buf, pco->pco_op.po_map_version, - pco->pco_op.po_hint.sh_rank, &pco->pco_space, &pco->pco_rebuild_st, - NULL /* tgts */, info, NULL, NULL, true); + pco->pco_op.po_hint.sh_rank, &pco->pco_space, rs, NULL /* tgts */, + info, NULL, NULL, true); + + tpriv->pool->dp_max_supported_layout_ver = rs->rs_max_supported_layout_ver + ? rs->rs_max_supported_layout_ver + : DAOS_POOL_OBJ_VERSION_1; if (rc != 0) { if (rc == -DER_AGAIN) { rc = tse_task_reinit(task); @@ -1085,8 +1091,9 @@ dc_pool_connect_internal(tse_task_t *task, daos_pool_info_t *info, const char *l D_GOTO(out_cred, rc); /** fill in request buffer */ - pool_connect_in_set_data(rpc, pool->dp_capas, pool_query_bits(info, NULL), bulk, - DAOS_POOL_GLOBAL_VERSION); + pool_connect_in_set_data(rpc, pool->dp_capas, + pool_query_bits(info, NULL) | DAOS_PO_QUERY_REBULD_MAX_LAYOUT_VER, + bulk, DAOS_POOL_GLOBAL_VERSION); /** Prepare "con_args" for pool_connect_cp(). */ con_args.pca_info = info; @@ -1348,7 +1355,8 @@ dc_pool_disconnect(tse_task_t *task) struct dc_pool_glob { /* magic number, DC_POOL_GLOB_MAGIC */ uint32_t dpg_magic; - uint32_t dpg_padding; + uint16_t dpg_max_supported_layout_ver; + uint16_t dpg_padding; /* pool UUID, pool handle UUID, and capas */ uuid_t dpg_pool; uuid_t dpg_pool_hdl; @@ -1403,6 +1411,7 @@ swap_pool_glob(struct dc_pool_glob *pool_glob) D_SWAP32S(&pool_glob->dpg_magic); /* skip pool_glob->dpg_padding */ + D_SWAP16S(&pool_glob->dpg_max_supported_layout_ver); /* skip pool_glob->dpg_pool (uuid_t) */ /* skip pool_glob->dpg_pool_hdl (uuid_t) */ D_SWAP64S(&pool_glob->dpg_capas); @@ -1470,6 +1479,7 @@ dc_pool_l2g(daos_handle_t poh, d_iov_t *glob) /* init pool global handle */ pool_glob = (struct dc_pool_glob *)glob->iov_buf; pool_glob->dpg_magic = DC_POOL_GLOB_MAGIC; + pool_glob->dpg_max_supported_layout_ver = pool->dp_max_supported_layout_ver; uuid_copy(pool_glob->dpg_pool, pool->dp_pool); uuid_copy(pool_glob->dpg_pool_hdl, pool->dp_pool_hdl); pool_glob->dpg_capas = pool->dp_capas; @@ -1540,6 +1550,7 @@ dc_pool_g2l(struct dc_pool_glob *pool_glob, size_t len, daos_handle_t *poh) if (pool == NULL) D_GOTO(out, rc = -DER_NOMEM); + pool->dp_max_supported_layout_ver = pool_glob->dpg_max_supported_layout_ver; uuid_copy(pool->dp_pool, pool_glob->dpg_pool); uuid_copy(pool->dp_pool_hdl, pool_glob->dpg_pool_hdl); pool->dp_capas = pool_glob->dpg_capas; diff --git a/src/pool/srv.c b/src/pool/srv.c index df93ecf3a5f..5d933c21684 100644 --- a/src/pool/srv.c +++ b/src/pool/srv.c @@ -118,7 +118,7 @@ setup(void) bool start = true; if (!engine_in_check()) { - d_getenv_bool("DAOS_START_POOL_SVC", &start); + d_getenv_bool("DAOS_START_POOLS", &start); if (start) return ds_pool_start_all(); } diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 96b3729a2cd..996f680f8e1 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -386,6 +386,11 @@ pool_prop_default_copy(daos_prop_t *prop_def, daos_prop_t *prop) if (prop == NULL || prop->dpp_nr == 0 || prop->dpp_entries == NULL) return 0; + if (!daos_prop_valid(prop, true, true)) { + D_ERROR("invalid pool property\n"); + return -DER_INVAL; + } + for (i = 0; i < prop->dpp_nr; i++) { entry = &prop->dpp_entries[i]; entry_def = daos_prop_entry_get(prop_def, entry->dpe_type); @@ -1528,7 +1533,7 @@ handle_event(struct pool_svc *svc, struct pool_svc_event_set *event_set) if (!pool_disable_exclude) { rc = pool_svc_exclude_ranks(svc, event_set); if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to exclude ranks", DP_UUID(svc->ps_uuid)); + DL_INFO(rc, DF_UUID ": exclude ranks", DP_UUID(svc->ps_uuid)); return rc; } } @@ -2796,9 +2801,17 @@ start_one(uuid_t uuid, void *varg) rc = ds_pool_start(uuid, aft_chk, immutable); if (rc != 0) { + uuid_t uuid_tmp; + DL_ERROR(rc, DF_UUID ": failed to start pool, aft_chk %s, immutable %s", DP_UUID(uuid), aft_chk ? "yes" : "no", immutable ? "yes" : "no"); ds_pool_failed_add(uuid, rc); + uuid_copy(uuid_tmp, uuid); + ds_notify_ras_eventf(RAS_POOL_START_FAILED, RAS_TYPE_INFO, RAS_SEV_ERROR, + NULL /* hwid */, NULL /* rank */, NULL /* inc */, + NULL /* jobid */, &uuid_tmp, NULL /* cont */, NULL /* objid */, + NULL /* ctlop */, NULL /* data */, + "failed to start pool: " DF_RC, DP_RC(rc)); } return 0; @@ -2809,11 +2822,10 @@ pool_start_all(void *arg) { int rc; - /* Scan the storage and start all pool services. */ + /* Scan the storage and start all pools. */ rc = ds_mgmt_tgt_pool_iterate(start_one, NULL /* arg */); if (rc != 0) - D_ERROR("failed to scan all pool services: "DF_RC"\n", - DP_RC(rc)); + DL_ERROR(rc, "failed to scan pools"); } bool @@ -4038,6 +4050,8 @@ ds_pool_connect_handler(crt_rpc_t *rpc, int handler_version) if (rc != 0) D_GOTO(out_svc, rc); } + if (query_bits & DAOS_PO_QUERY_REBULD_MAX_LAYOUT_VER) + out->pco_rebuild_st.rs_max_supported_layout_ver = DAOS_POOL_OBJ_VERSION; rc = rdb_tx_begin(svc->ps_rsvc.s_db, svc->ps_rsvc.s_term, &tx); if (rc != 0) @@ -5615,6 +5629,15 @@ ds_pool_prop_set_handler(crt_rpc_t *rpc) "%d.\n", DP_UUID(in->psi_op.pi_uuid), rc); daos_prop_free(prop); } + /* If self_heal.exclude is set, resume event handling. */ + if (rc == 0 && !dup_op) { + struct daos_prop_entry *entry; + + entry = daos_prop_entry_get(prop_in, DAOS_PROP_PO_SELF_HEAL); + if (entry != NULL && daos_prop_is_set(entry) && + entry->dpe_val & DAOS_SELF_HEAL_AUTO_EXCLUDE) + resume_event_handling(svc); + } out_svc: ds_rsvc_set_hint(&svc->ps_rsvc, &out->pso_op.po_hint); pool_svc_put_leader(svc); @@ -7084,6 +7107,29 @@ pool_svc_update_map_internal(struct pool_svc *svc, unsigned int opc, bool exclud goto out; ABT_rwlock_wrlock(svc->ps_lock); + if (opc == MAP_EXCLUDE && src == MUS_SWIM) { + daos_prop_t *prop; + struct daos_prop_entry *entry; + + /* Check if self_heal.exclude is enabled. */ + rc = pool_prop_read(&tx, svc, DAOS_PO_QUERY_PROP_SELF_HEAL, &prop); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to read self_heal property", + DP_UUID(svc->ps_uuid)); + goto out_lock; + } + entry = daos_prop_entry_get(prop, DAOS_PROP_PO_SELF_HEAL); + D_ASSERT(entry != NULL); + if (!daos_prop_is_set(entry) || !(entry->dpe_val & DAOS_SELF_HEAL_AUTO_EXCLUDE)) { + D_INFO(DF_UUID ": not excluding for SWIM: self_heal.exclude not enabled\n", + DP_UUID(svc->ps_uuid)); + rc = -DER_NO_PERM; + daos_prop_free(prop); + goto out_lock; + } + daos_prop_free(prop); + } + /* Create a temporary pool map based on the last committed version. */ rc = read_map(&tx, &svc->ps_root, &map); if (rc != 0) @@ -7402,10 +7448,11 @@ ds_pool_tgt_add_in(uuid_t pool_uuid, struct pool_target_id_list *list) } int -ds_pool_tgt_finish_rebuild(uuid_t pool_uuid, struct pool_target_id_list *list) +ds_pool_tgt_finish_rebuild(uuid_t pool_uuid, struct pool_target_id_list *list, + uint32_t *reclaim_ver) { - return pool_update_map_internal(pool_uuid, MAP_FINISH_REBUILD, false, list, - NULL, NULL, NULL, NULL, NULL, NULL); + return pool_update_map_internal(pool_uuid, MAP_FINISH_REBUILD, false, list, NULL, NULL, + NULL, NULL, reclaim_ver, NULL); } int diff --git a/src/pool/srv_pool_map.c b/src/pool/srv_pool_map.c index 0807a87e048..32f0710102c 100644 --- a/src/pool/srv_pool_map.c +++ b/src/pool/srv_pool_map.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2021-2024 Intel Corporation. + * (C) Copyright 2025 Google LLC * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent @@ -72,13 +73,13 @@ update_one_tgt(uuid_t pool_uuid, struct pool_map *map, struct pool_target *targe switch (target->ta_comp.co_status) { case PO_COMP_ST_DOWN: case PO_COMP_ST_DOWNOUT: + case PO_COMP_ST_DRAIN: /* Nothing to do, already excluded */ D_INFO(DF_MAP ": Skip exclude down " DF_TARGET "\n", DP_MAP(pool_uuid, map), DP_TARGET(target)); break; case PO_COMP_ST_UP: case PO_COMP_ST_UPIN: - case PO_COMP_ST_DRAIN: D_DEBUG(DB_MD, DF_MAP ": change " DF_TARGET " to DOWN\n", DP_MAP(pool_uuid, map), DP_TARGET(target)); target->ta_comp.co_status = PO_COMP_ST_DOWN; diff --git a/src/pool/srv_target.c b/src/pool/srv_target.c index 46c7fadaeb4..8a48c0c6759 100644 --- a/src/pool/srv_target.c +++ b/src/pool/srv_target.c @@ -463,7 +463,6 @@ pool_child_recreate(struct ds_pool_child *child) } - static int pool_child_start(struct ds_pool_child *child, bool recreate) { @@ -493,14 +492,14 @@ pool_child_start(struct ds_pool_child *child, bool recreate) D_FREE(path); if (rc != 0) { - if (rc != -DER_NONEXIST) { + if (!engine_in_check() || rc != -DER_NONEXIST) { DL_CDEBUG(rc == -DER_NVME_IO, DB_MGMT, DLOG_ERR, rc, DF_UUID": Open VOS pool failed.", DP_UUID(child->spc_uuid)); goto out; } - D_WARN("Lost pool "DF_UUIDF" shard %u on rank %u.\n", - DP_UUID(child->spc_uuid), info->dmi_tgt_id, dss_self_rank()); + D_WARN(DF_UUID ": Lost pool shard %u on rank %u.\n", DP_UUID(child->spc_uuid), + info->dmi_tgt_id, dss_self_rank()); /* * Ignore the failure to allow subsequent logic (such as DAOS check) * to handle the trouble. diff --git a/src/rebuild/README.md b/src/rebuild/README.md index 47827b62bfd..33f6c32d7ca 100644 --- a/src/rebuild/README.md +++ b/src/rebuild/README.md @@ -204,8 +204,10 @@ struct daos_rebuild_status { int32_t rs_done; }; - /* padding of rebuild status */ - int32_t rs_padding32; + /** Maximum supported layout version */ + uint16_t rs_max_supported_layout_ver; + /** padding of rebuild status */ + int16_t rs_padding16; /* Failure on which rank */ int32_t rs_fail_rank; diff --git a/src/rebuild/rebuild_internal.h b/src/rebuild/rebuild_internal.h index 0510a32f0f8..4eb7f8ef2b5 100644 --- a/src/rebuild/rebuild_internal.h +++ b/src/rebuild/rebuild_internal.h @@ -217,6 +217,7 @@ struct rebuild_global { /* Per target structure to track the rebuild status */ extern struct rebuild_global rebuild_gst; +extern unsigned int rebuild_wait_ec_pause; struct rebuild_task { d_list_t dst_list; diff --git a/src/rebuild/scan.c b/src/rebuild/scan.c index 41a1dac2fe3..f7208d62287 100644 --- a/src/rebuild/scan.c +++ b/src/rebuild/scan.c @@ -965,18 +965,24 @@ rebuild_scanner(void *data) { struct rebuild_scan_arg arg = {0}; struct rebuild_tgt_pool_tracker *rpt = data; - struct ds_pool_child *child; - struct rebuild_pool_tls *tls; - vos_iter_param_t param = {0}; - struct vos_iter_anchors anchor = {0}; - ABT_thread ult_send = ABT_THREAD_NULL; - struct umem_attr uma; - int rc = 0; + struct ds_pool_child *child = NULL; + struct rebuild_pool_tls *tls; + vos_iter_param_t param = { 0 }; + struct vos_iter_anchors anchor = { 0 }; + ABT_thread ult_send = ABT_THREAD_NULL; + struct umem_attr uma; + int rc = 0; tls = rebuild_pool_tls_lookup(rpt->rt_pool_uuid, rpt->rt_rebuild_ver, rpt->rt_rebuild_gen); if (tls == NULL) return 0; + child = ds_pool_child_lookup(rpt->rt_pool_uuid); + if (child == NULL) + D_GOTO(out, rc = -DER_NONEXIST); + + ds_cont_child_wait_ec_agg_pause(child, rebuild_wait_ec_pause); + /* There maybe orphan DTX entries after DTX resync, let's cleanup before rebuild scan. */ rc = dtx_cleanup_orphan(rpt->rt_pool_uuid, rpt->rt_pool->sp_dtx_resync_version); if (rc != 0) @@ -1014,10 +1020,6 @@ rebuild_scanner(void *data) } } - child = ds_pool_child_lookup(rpt->rt_pool_uuid); - if (child == NULL) - D_GOTO(out, rc = -DER_NONEXIST); - param.ip_hdl = child->spc_hdl; param.ip_flags = VOS_IT_FOR_MIGRATION; arg.rpt = rpt; @@ -1025,11 +1027,11 @@ rebuild_scanner(void *data) rc = vos_iterate(¶m, VOS_ITER_COUUID, false, &anchor, rebuild_container_scan_cb, NULL, &arg, NULL); if (rc < 0) - D_GOTO(put, rc); + D_GOTO(out, rc); rc = 0; /* rc might be 1 if rebuild is aborted */ -put: - ds_pool_child_put(child); out: + if (child) + ds_pool_child_put(child); tls->rebuild_pool_scan_done = 1; if (ult_send != ABT_THREAD_NULL) ABT_thread_free(&ult_send); diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c index 817dda33ac4..c0ebf709453 100644 --- a/src/rebuild/srv.c +++ b/src/rebuild/srv.c @@ -25,6 +25,7 @@ #define RBLD_CHECK_INTV 2000 /* milliseconds interval to check*/ struct rebuild_global rebuild_gst; +unsigned int rebuild_wait_ec_pause = 0; struct pool_map * rebuild_pool_map_get(struct ds_pool *pool) @@ -1472,7 +1473,8 @@ retry_rebuild_task(struct rebuild_task *task, int error, daos_rebuild_opc_t *opc static int rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, - struct rebuild_global_pool_tracker *rgt, int ret) + struct rebuild_global_pool_tracker *rgt, uint32_t obj_reclaim_ver, + int ret) { int rc = 0; int rc1; @@ -1561,9 +1563,12 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, D_DEBUG(DB_REBUILD, DF_UUID" opc %u/%u, error %d, schedule RECLAIM.\n", DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, task->dst_map_ver, rgt->rgt_status.rs_errno); - rc = ds_rebuild_schedule(pool, task->dst_map_ver, rgt->rgt_reclaim_epoch, - task->dst_new_layout_version, - &task->dst_tgts, RB_OP_RECLAIM, 5); + + /* NB: if pool map remains unchanged, ds_pool_tgt_finish_rebuild sets it to 0 */ + obj_reclaim_ver = obj_reclaim_ver > 0 ? obj_reclaim_ver : task->dst_map_ver; + rc = ds_rebuild_schedule(pool, obj_reclaim_ver, rgt->rgt_reclaim_epoch, + task->dst_new_layout_version, &task->dst_tgts, + RB_OP_RECLAIM, 5); if (rc != 0) D_ERROR("reschedule reclaim, "DF_UUID" failed: "DF_RC"\n", DP_UUID(task->dst_pool_uuid), DP_RC(rc)); @@ -1596,6 +1601,7 @@ rebuild_task_ult(void *arg) struct rebuild_global_pool_tracker *rgt = NULL; d_rank_t myrank; uint64_t cur_ts = 0; + uint32_t obj_reclaim_ver = 0; int rc; cur_ts = daos_gettime_coarse(); @@ -1724,7 +1730,8 @@ rebuild_task_ult(void *arg) goto iv_stop; if (task->dst_rebuild_op == RB_OP_REBUILD) - rc = ds_pool_tgt_finish_rebuild(pool->sp_uuid, &task->dst_tgts); + rc = ds_pool_tgt_finish_rebuild(pool->sp_uuid, &task->dst_tgts, + &obj_reclaim_ver); D_INFO("finish rebuild %d of " DF_UUID " "DF_RC"\n", task->dst_tgts.pti_ids[0].pti_id, DP_UUID(task->dst_pool_uuid), DP_RC(rc)); } @@ -1748,7 +1755,7 @@ rebuild_task_ult(void *arg) } try_reschedule: - rebuild_task_complete_schedule(task, pool, rgt, rc); + rebuild_task_complete_schedule(task, pool, rgt, obj_reclaim_ver, rc); output: rc = rebuild_notify_ras_end(&task->dst_pool_uuid, task->dst_map_ver, RB_OP_STR(task->dst_rebuild_op), @@ -2282,7 +2289,6 @@ rebuild_fini_one(void *arg) } ds_pool_child_put(dpc); - return 0; } @@ -2297,6 +2303,7 @@ rebuild_tgt_fini(struct rebuild_tgt_pool_tracker *rpt) D_ASSERT(rpt->rt_pool->sp_rebuilding > 0); rpt->rt_pool->sp_rebuilding--; + rpt->rt_pool->sp_rebuild_scan = 0; ABT_mutex_lock(rpt->rt_lock); ABT_cond_signal(rpt->rt_global_dtx_wait_cond); @@ -2737,6 +2744,10 @@ init(void) { int rc; + rc = d_getenv_uint(REBUILD_WAIT_EC_PAUSE_ENV, &rebuild_wait_ec_pause); + if (rc == 0) + D_DEBUG(DB_REBUILD, "Set REBUILD_WAIT_EC_PAUSE to %u\n", rebuild_wait_ec_pause); + D_INIT_LIST_HEAD(&rebuild_gst.rg_tgt_tracker_list); D_INIT_LIST_HEAD(&rebuild_gst.rg_global_tracker_list); D_INIT_LIST_HEAD(&rebuild_gst.rg_completed_list); diff --git a/src/tests/ftest/cart/rpc/swim_notification.yaml b/src/tests/ftest/cart/rpc/swim_notification.yaml index 89e3af14238..83cbac5799d 100644 --- a/src/tests/ftest/cart/rpc/swim_notification.yaml +++ b/src/tests/ftest/cart/rpc/swim_notification.yaml @@ -4,7 +4,7 @@ ENV: default: #!filter-only : /run/tests/proto_np - - D_LOG_MASK: "WARN" + - D_LOG_MASK: "INFO" - D_INTERFACE: "eth0" - CRT_TEST_CONT: "1" - test_servers_CRT_CTX_NUM: "16" @@ -29,11 +29,11 @@ tests: !mux test_clients_env: "" test_clients_ppn: "1" test_clients_arg: - - "--name client_group --attach_to tg_srv_grp_swim_test --init_only --holdtime 20" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank2=alive' --skip_shutdown --skip_check_in" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 2 --shut_only --holdtime 30" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank2=dead' --skip_shutdown --skip_check_in" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank1=alive' --skip_shutdown --skip_check_in" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank0=alive' --skip_shutdown --skip_check_in" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --disable_swim --skip_shutdown --skip_check_in" - - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --shut_only" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1,2 --init_only --holdtime 30" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank2=alive' --skip_shutdown --skip_check_in --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 2 --shut_only --holdtime 30 --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank2=dead' --skip_shutdown --skip_check_in --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank1=alive' --skip_shutdown --skip_check_in --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --verify_swim_status 'rank0=alive' --skip_shutdown --skip_check_in --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --disable_swim --skip_shutdown --skip_check_in --skip_wait" + - "--name client_group --attach_to tg_srv_grp_swim_test --rank 0,1 --shut_only --skip_wait" diff --git a/src/tests/ftest/cart/test_group_np_cli.c b/src/tests/ftest/cart/test_group_np_cli.c index f796fc1a152..0ca71e895c9 100644 --- a/src/tests/ftest/cart/test_group_np_cli.c +++ b/src/tests/ftest/cart/test_group_np_cli.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -23,21 +24,20 @@ static void send_rpc_swim_check(crt_endpoint_t server_ep, crt_rpc_t *rpc_req) { - struct test_swim_status_in *rpc_req_input; + struct test_swim_status_in *input; int rc = crt_req_create(test_g.t_crt_ctx[0], &server_ep, TEST_OPC_SWIM_STATUS, &rpc_req); D_ASSERTF(rc == 0 && rpc_req != NULL, "crt_req_create() failed. " - "rc: %d, rpc_req: %p\n", rc, rpc_req); + "rc: %d, rpc_req: %p\n", + rc, rpc_req); - rpc_req_input = crt_req_get(rpc_req); - D_ASSERTF(rpc_req_input != NULL, "crt_req_get() failed." - " rpc_req_input: %p\n", rpc_req_input); + input = crt_req_get(rpc_req); /* Set rank and expected swim status based on CLI options */ - rpc_req_input->rank = test_g.t_verify_swim_status.rank; - rpc_req_input->exp_status = test_g.t_verify_swim_status.swim_status; + input->rank = test_g.t_verify_swim_status.rank; + input->exp_status = test_g.t_verify_swim_status.swim_status; /* RPC is expected to finish in 10 seconds */ rc = crt_req_set_timeout(rpc_req, 10); @@ -52,20 +52,19 @@ send_rpc_swim_check(crt_endpoint_t server_ep, crt_rpc_t *rpc_req) static void send_rpc_disable_swim(crt_endpoint_t server_ep, crt_rpc_t *rpc_req) { - struct test_disable_swim_in *rpc_req_input; + struct test_disable_swim_in *input; int rc = crt_req_create(test_g.t_crt_ctx[0], &server_ep, TEST_OPC_DISABLE_SWIM, &rpc_req); D_ASSERTF(rc == 0 && rpc_req != NULL, "crt_req_create() failed. " - "rc: %d, rpc_req: %p\n", rc, rpc_req); + "rc: %d, rpc_req: %p\n", + rc, rpc_req); - rpc_req_input = crt_req_get(rpc_req); - D_ASSERTF(rpc_req_input != NULL, "crt_req_get() failed." - " rpc_req_input: %p\n", rpc_req_input); + input = crt_req_get(rpc_req); /* Set rank and expected swim status based on CLI options */ - rpc_req_input->rank = server_ep.ep_rank; + input->rank = server_ep.ep_rank; /* RPC is expected to finish in 10 seconds */ rc = crt_req_set_timeout(rpc_req, 10); @@ -94,43 +93,43 @@ test_run(void) if (test_g.t_skip_init) { DBG_PRINT("Skipping init stage.\n"); + goto skip_init; + } - } else { - if (test_g.t_save_cfg) { - rc = crt_group_config_path_set(test_g.t_cfg_path); - D_ASSERTF(rc == 0, - "crt_group_config_path_set failed %d\n", rc); - } + if (test_g.t_save_cfg) { + DBG_PRINT("Setting group config path: %p\n", test_g.t_cfg_path); + rc = crt_group_config_path_set(test_g.t_cfg_path); + D_ASSERTF(rc == 0, "crt_group_config_path_set failed %d\n", rc); + } - rc = crtu_cli_start_basic(test_g.t_local_group_name, - test_g.t_remote_group_name, - &grp, &rank_list, &test_g.t_crt_ctx[0], - &test_g.t_tid[0], test_g.t_srv_ctx_num, - test_g.t_use_cfg, NULL, test_g.t_use_daos_agent_env); - D_ASSERTF(rc == 0, "crtu_cli_start_basic() failed\n"); - - rc = sem_init(&test_g.t_token_to_proceed, 0, 0); - D_ASSERTF(rc == 0, "sem_init() failed.\n"); - - /* register RPCs */ - rc = crt_proto_register(&my_proto_fmt_test_group1); - D_ASSERTF(rc == 0, "crt_proto_register() failed. rc: %d\n", - rc); - - /* Process the --rank option, e.g., --rank 1,2-4 */ - if (test_g.cg_num_ranks > 0) { - _cg_ranks = (uint32_t *)test_g.cg_ranks; - _cg_num_ranks = test_g.cg_num_ranks; - - /* free up rank list from crtu_cli_start_basic */ - if (rank_list != NULL) { - /* avoid checkpatch warning */ - d_rank_list_free(rank_list); - } - rank_list = uint32_array_to_rank_list(_cg_ranks, _cg_num_ranks); - D_ASSERTF(rank_list != NULL, "failed to convert array to rank list\n"); + rc = crtu_cli_start_basic(test_g.t_local_group_name, test_g.t_remote_group_name, &grp, + &rank_list, &test_g.t_crt_ctx[0], &test_g.t_tid[0], + test_g.t_srv_ctx_num, test_g.t_use_cfg, NULL, + test_g.t_use_daos_agent_env); + D_ASSERTF(rc == 0, "crtu_cli_start_basic() failed\n"); + + rc = sem_init(&test_g.t_token_to_proceed, 0, 0); + D_ASSERTF(rc == 0, "sem_init() failed.\n"); + + /* register RPCs */ + rc = crt_proto_register(&my_proto_fmt_test_group1); + D_ASSERTF(rc == 0, "crt_proto_register() failed. rc: %d\n", rc); + + /* Process the --rank option, e.g., --rank 1,2-4 */ + if (test_g.cg_num_ranks > 0) { + _cg_ranks = (uint32_t *)test_g.cg_ranks; + _cg_num_ranks = test_g.cg_num_ranks; + + /* free up rank list from crtu_cli_start_basic */ + if (rank_list != NULL) { + /* avoid checkpatch warning */ + d_rank_list_free(rank_list); } + rank_list = uint32_array_to_rank_list(_cg_ranks, _cg_num_ranks); + D_ASSERTF(rank_list != NULL, "failed to convert array to rank list\n"); + } + if (!test_g.t_skip_wait) { rc = crtu_wait_for_ranks(test_g.t_crt_ctx[0], grp, rank_list, @@ -141,6 +140,8 @@ test_run(void) D_ASSERTF(rc == 0, "wait_for_ranks() failed; rc=%d\n", rc); } +skip_init: + if (test_g.t_init_only) { DBG_PRINT("Init only. Returning now.\n"); D_GOTO(clean_up, rc = 0); diff --git a/src/tests/ftest/cart/test_group_np_common.h b/src/tests/ftest/cart/test_group_np_common.h index 8da4c047c7f..86786ba5ab2 100644 --- a/src/tests/ftest/cart/test_group_np_common.h +++ b/src/tests/ftest/cart/test_group_np_common.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -33,6 +34,7 @@ struct test_t { int t_shut_only; int t_init_only; int t_skip_init; + int t_skip_wait; int t_skip_shutdown; int t_skip_check_in; bool t_save_cfg; @@ -189,6 +191,7 @@ test_swim_status_handler(crt_rpc_t *rpc_req) int rank; int rc_dead; int rc_alive; + bool result_ok = false; /* CaRT internally already allocated the input/output buffer */ e_req = crt_req_get(rpc_req); @@ -198,51 +201,43 @@ test_swim_status_handler(crt_rpc_t *rpc_req) /* compile and run regex's */ regcomp(®ex_dead, dead_regex, REG_EXTENDED); - rc_dead = regexec(®ex_dead, - swim_seq_by_rank[rank], - 0, NULL, 0); + rc_dead = regexec(®ex_dead, swim_seq_by_rank[rank], 0, NULL, 0); + regcomp(®ex_alive, alive_regex, REG_EXTENDED); - rc_alive = regexec(®ex_alive, - swim_seq_by_rank[rank], - 0, NULL, 0); + rc_alive = regexec(®ex_alive, swim_seq_by_rank[rank], 0, NULL, 0); regfree(®ex_alive); regfree(®ex_dead); - DBG_PRINT("tier1 test_server recv'd swim_status, opc: %#x.\n", - rpc_req->cr_opc); - DBG_PRINT("tier1 swim_status input - rank: %d, exp_status: %d.\n", - rank, e_req->exp_status); - - if (e_req->exp_status == CRT_EVT_ALIVE) - D_ASSERTF(rc_alive == 0, - "Swim status alive sequence (%s) " - "does not match '%s' for rank %d.\n", - swim_seq_by_rank[rank], alive_regex, rank); - else if (e_req->exp_status == CRT_EVT_DEAD) - D_ASSERTF(rc_dead == 0, - "Swim status dead sequence (%s) " - "does not match '%s' for rank %d..\n", - swim_seq_by_rank[rank], dead_regex, rank); - - DBG_PRINT("Rank [%d] SWIM state sequence (%s) for " - "status [%d] is as expected.\n", - rank, swim_seq_by_rank[rank], - e_req->exp_status); + if (e_req->exp_status == CRT_EVT_ALIVE) { + if (rc_alive != 0) { + D_ERROR("Swim status alive sequence (%s) " + "does not match '%s' for rank %d.\n", + swim_seq_by_rank[rank], alive_regex, rank); + result_ok = false; + } else { + result_ok = true; + } + } else if (e_req->exp_status == CRT_EVT_DEAD) { + if (rc_dead != 0) { + D_ERROR("Swim status dead sequence (%s) " + "does not match '%s' for rank %d..\n", + swim_seq_by_rank[rank], dead_regex, rank); + result_ok = false; + } else { + result_ok = true; + } + } e_reply = crt_reply_get(rpc_req); + D_ASSERTF(e_reply != NULL, "crt_reply_get() failed"); + e_reply->bool_val = result_ok; - /* If we got past the previous assert, then we've succeeded */ - e_reply->bool_val = true; - D_ASSERTF(e_reply != NULL, "crt_reply_get() failed. e_reply: %p\n", - e_reply); + DBG_PRINT("rank:%d exp_status: %d matches_expected: %d\n", rank, e_req->exp_status, + result_ok); rc = crt_reply_send(rpc_req); D_ASSERTF(rc == 0, "crt_reply_send() failed. rc: %d\n", rc); - - DBG_PRINT("tier1 test_srver sent swim_status reply," - "e_reply->bool_val: %d.\n", - e_reply->bool_val); } static void @@ -256,15 +251,13 @@ test_ping_delay_handler(crt_rpc_t *rpc_req) p_req = crt_req_get(rpc_req); D_ASSERTF(p_req != NULL, "crt_req_get() failed. p_req: %p\n", p_req); - DBG_PRINT("tier1 test_server recv'd ping delay, opc: %#x.\n", - rpc_req->cr_opc); - DBG_PRINT("tier1 delayed ping input - age: %d, name: %s, days: %d, " - "delay: %u.\n", p_req->age, p_req->name, p_req->days, - p_req->delay); + DBG_PRINT("test_server recv'd ping delay, opc: %#x.\n", rpc_req->cr_opc); + DBG_PRINT("delayed ping input - age: %d, name: %s, days: %d, " + "delay: %u.\n", + p_req->age, p_req->name, p_req->days, p_req->delay); p_reply = crt_reply_get(rpc_req); - D_ASSERTF(p_reply != NULL, "crt_reply_get() failed. p_reply: %p\n", - p_reply); + D_ASSERTF(p_reply != NULL, "crt_reply_get() failed\n"); p_reply->ret = 0; p_reply->room_no = test_g.t_roomno++; @@ -745,29 +738,28 @@ test_parse_args(int argc, char **argv) int option_index = 0; int rc = 0; int ss; - struct option long_options[] = { - {"name", required_argument, 0, 'n'}, - {"attach_to", required_argument, 0, 'a'}, - {"holdtime", required_argument, 0, 'h'}, - {"hold", no_argument, &test_g.t_hold, 1}, - {"srv_ctx_num", required_argument, 0, 'c'}, - {"shut_only", no_argument, &test_g.t_shut_only, 1}, - {"init_only", no_argument, &test_g.t_init_only, 1}, - {"skip_init", no_argument, &test_g.t_skip_init, 1}, - {"skip_shutdown", no_argument, &test_g.t_skip_shutdown, 1}, - {"skip_check_in", no_argument, &test_g.t_skip_check_in, 1}, - {"rank", required_argument, 0, 'r'}, - {"cfg_path", required_argument, 0, 's'}, - {"use_cfg", required_argument, 0, 'u'}, - {"register_swim_callback", required_argument, 0, 'w'}, - {"verify_swim_status", required_argument, 0, 'v'}, - {"disable_swim", no_argument, &test_g.t_disable_swim, 1}, - {"get_swim_status", no_argument, 0, 'g'}, - {"shutdown_delay", required_argument, 0, 'd'}, - {"write_completion_file", no_argument, - &test_g.t_write_completion_file, 1}, - {0, 0, 0, 0} - }; + struct option long_options[] = { + {"name", required_argument, 0, 'n'}, + {"attach_to", required_argument, 0, 'a'}, + {"holdtime", required_argument, 0, 'h'}, + {"hold", no_argument, &test_g.t_hold, 1}, + {"srv_ctx_num", required_argument, 0, 'c'}, + {"shut_only", no_argument, &test_g.t_shut_only, 1}, + {"init_only", no_argument, &test_g.t_init_only, 1}, + {"skip_init", no_argument, &test_g.t_skip_init, 1}, + {"skip_wait", no_argument, &test_g.t_skip_wait, 1}, + {"skip_shutdown", no_argument, &test_g.t_skip_shutdown, 1}, + {"skip_check_in", no_argument, &test_g.t_skip_check_in, 1}, + {"rank", required_argument, 0, 'r'}, + {"cfg_path", required_argument, 0, 's'}, + {"use_cfg", required_argument, 0, 'u'}, + {"register_swim_callback", required_argument, 0, 'w'}, + {"verify_swim_status", required_argument, 0, 'v'}, + {"disable_swim", no_argument, &test_g.t_disable_swim, 1}, + {"get_swim_status", no_argument, 0, 'g'}, + {"shutdown_delay", required_argument, 0, 'd'}, + {"write_completion_file", no_argument, &test_g.t_write_completion_file, 1}, + {0, 0, 0, 0}}; test_g.cg_num_ranks = 0; test_g.t_use_cfg = true; diff --git a/src/tests/ftest/deployment/agent_failure.py b/src/tests/ftest/deployment/agent_failure.py index eaa97cdde2b..d423d3454bc 100644 --- a/src/tests/ftest/deployment/agent_failure.py +++ b/src/tests/ftest/deployment/agent_failure.py @@ -123,7 +123,7 @@ def test_agent_failure(self): # 5. Verify journalctl shows the log that the agent is stopped. results = get_journalctl( hosts=self.hostlist_clients, since=since, until=until, - journalctl_type="daos_agent") + journalctl_type="daos_agent", run_user=self.test_env.agent_user) self.log.info("journalctl results = %s", results) if "shutting down" not in results[0]["data"]: msg = "Agent shut down message not found in journalctl! Output = {}".format( @@ -210,7 +210,8 @@ def test_agent_failure_isolation(self): since = journalctl_time() self.log.info("Stopping agent on %s", agent_host_kill) pattern = self.agent_managers[0].manager.job.command_regex - detected, running = stop_processes(self.log, hosts=agent_host_kill, pattern=pattern) + detected, running = stop_processes(self.log, hosts=agent_host_kill, pattern=pattern, + user=self.agent_managers[0].manager.job.run_user) if not detected: msg = "No daos_agent process killed on {}!".format(agent_host_kill) errors.append(msg) @@ -241,7 +242,7 @@ def test_agent_failure_isolation(self): # stopped. results = get_journalctl( hosts=NodeSet(agent_host_kill), since=since, until=until, - journalctl_type="daos_agent") + journalctl_type="daos_agent", run_user=self.test_env.agent_user) self.log.info("journalctl results (kill) = %s", results) if "shutting down" not in results[0]["data"]: msg = ("Agent shut down message not found in journalctl on killed client! " @@ -252,7 +253,7 @@ def test_agent_failure_isolation(self): # in the previous step doesn't show that the agent is stopped. results = get_journalctl( hosts=NodeSet(agent_host_keep), since=since, until=until, - journalctl_type="daos_agent") + journalctl_type="daos_agent", run_user=self.test_env.agent_user) self.log.info("journalctl results (keep) = %s", results) if "shutting down" in results[0]["data"]: msg = ("Agent shut down message found in journalctl on keep client! " diff --git a/src/tests/ftest/dfuse/sparse_file.py b/src/tests/ftest/dfuse/sparse_file.py index 484b787c38c..f1d4f4ad02f 100644 --- a/src/tests/ftest/dfuse/sparse_file.py +++ b/src/tests/ftest/dfuse/sparse_file.py @@ -1,5 +1,6 @@ """ (C) Copyright 2020-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -68,10 +69,11 @@ def test_sparsefile(self): # create and open a connection on remote node to open file on that # remote node + key_path = os.path.expanduser(os.path.join('~', '.ssh', 'id_rsa')) + self.log.info("=does %s exist: %s", key_path, os.path.exists(key_path)) ssh = paramiko.SSHClient() - ssh.load_system_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(self.hostlist_clients[0], username=getuser()) + ssh.connect(hostname=self.hostlist_clients[0], username=getuser(), key_filename=key_path) sftp = ssh.open_sftp() # open remote file diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index f3daa8464d7..a279f7027d4 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """ (C) Copyright 2018-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -14,6 +15,7 @@ from collections import OrderedDict from tempfile import TemporaryDirectory +import distro from ClusterShell.NodeSet import NodeSet from process_core_files import get_core_file_pattern # pylint: disable=import-error,no-name-in-module @@ -176,6 +178,7 @@ def _configure(self, overwrite_config=False): # Add details about the run self.details["avocado version"] = str(self.avocado) + self.details["distro"] = distro.linux_distribution(full_distribution_name=True) self.details["launch host"] = str(self.local_host) def _create_log_dir(self): @@ -272,7 +275,7 @@ def _run(self, args): all_hosts = args.test_servers | args.test_clients | self.local_host self.details["installed packages"] = find_packages( logger, all_hosts, - "'^(daos|libfabric|mercury|ior|openmpi|mpifileutils|mlnx-ofed-basic)-'") + "'^(daos|libfabric|mercury|ior|openmpi|mpich|mpifileutils|mlnx-ofed-basic)-'") # Setup the test environment test_env = TestEnvironment() diff --git a/src/tests/ftest/pool/create_all_hw.yaml b/src/tests/ftest/pool/create_all_hw.yaml index a8070c93b5d..b47802955d6 100644 --- a/src/tests/ftest/pool/create_all_hw.yaml +++ b/src/tests/ftest/pool/create_all_hw.yaml @@ -1,7 +1,6 @@ setup: start_agents_once: False start_servers_once: False - server_config_namespace: /run/server_configurations/*/server_config/* hosts: test_servers: 3 @@ -39,148 +38,44 @@ test_two_pools_hw: scm: 2147483648 # 2GiB nvme: 0 -# Mux the number of targets per engine (1-4) with both engines using the same number of targets -server_configurations: !mux - server_config_1_target: - server_config: - name: daos_server - engines_per_host: 2 - engines: +server_config: + name: daos_server + engines_per_host: 2 + engines: + 0: + pinned_numa_node: 0 + fabric_iface: ib0 + fabric_iface_port: 31317 + log_file: daos_server0.log + storage: 0: - targets: 1 - pinned_numa_node: 0 - nr_xs_helpers: 1 - fabric_iface: ib0 - fabric_iface_port: 31317 - log_file: daos_server0.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem0"] - scm_mount: /mnt/daos0 - 1: - class: nvme - bdev_list: ["aaaa:aa:aa.a"] + class: dcpm + scm_list: ["/dev/pmem0"] + scm_mount: /mnt/daos0 1: - targets: 1 - pinned_numa_node: 1 - nr_xs_helpers: 1 - fabric_iface: ib1 - fabric_iface_port: 31417 - log_file: daos_server1.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem1"] - scm_mount: /mnt/daos1 - 1: - class: nvme - bdev_list: ["bbbb:bb:bb.b"] - server_config_2_target: - server_config: - name: daos_server - engines_per_host: 2 - engines: + class: nvme + bdev_list: ["aaaa:aa:aa.a"] + 1: + pinned_numa_node: 1 + fabric_iface: ib1 + fabric_iface_port: 31417 + log_file: daos_server1.log + storage: 0: - targets: 2 - pinned_numa_node: 0 - nr_xs_helpers: 1 - fabric_iface: ib0 - fabric_iface_port: 31317 - log_file: daos_server0.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem0"] - scm_mount: /mnt/daos0 - 1: - class: nvme - bdev_list: ["aaaa:aa:aa.a"] + class: dcpm + scm_list: ["/dev/pmem1"] + scm_mount: /mnt/daos1 1: - targets: 2 - pinned_numa_node: 1 - nr_xs_helpers: 1 - fabric_iface: ib1 - fabric_iface_port: 31417 - log_file: daos_server1.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem1"] - scm_mount: /mnt/daos1 - 1: - class: nvme - bdev_list: ["bbbb:bb:bb.b"] - server_config_3_target: - server_config: - name: daos_server - engines_per_host: 2 - engines: - 0: - targets: 3 - pinned_numa_node: 0 - nr_xs_helpers: 1 - fabric_iface: ib0 - fabric_iface_port: 31317 - log_file: daos_server0.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem0"] - scm_mount: /mnt/daos0 - 1: - class: nvme - bdev_list: ["aaaa:aa:aa.a"] - 1: - targets: 3 - pinned_numa_node: 1 - nr_xs_helpers: 1 - fabric_iface: ib1 - fabric_iface_port: 31417 - log_file: daos_server1.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem1"] - scm_mount: /mnt/daos1 - 1: - class: nvme - bdev_list: ["bbbb:bb:bb.b"] - server_config_5_target: - server_config: - name: daos_server - engines_per_host: 2 - engines: - 0: - targets: 5 - pinned_numa_node: 0 - nr_xs_helpers: 1 - fabric_iface: ib0 - fabric_iface_port: 31317 - log_file: daos_server0.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem0"] - scm_mount: /mnt/daos0 - 1: - class: nvme - bdev_list: ["aaaa:aa:aa.a"] - 1: - targets: 5 - pinned_numa_node: 1 - nr_xs_helpers: 1 - fabric_iface: ib1 - fabric_iface_port: 31417 - log_file: daos_server1.log - storage: - 0: - class: dcpm - scm_list: ["/dev/pmem1"] - scm_mount: /mnt/daos1 - 1: - class: nvme - bdev_list: ["bbbb:bb:bb.b"] - -pool: - control_method: dmg + class: nvme + bdev_list: ["bbbb:bb:bb.b"] + engines_common: + targets_mux: !mux + one_target: + targets: 1 + two_targets: + targets: 2 + three_targets: + targets: 3 + five_targets: + targets: 5 + nr_xs_helpers: 1 diff --git a/src/tests/ftest/rebuild/basic.py b/src/tests/ftest/rebuild/basic.py index 787e0f922e5..d1a19247067 100644 --- a/src/tests/ftest/rebuild/basic.py +++ b/src/tests/ftest/rebuild/basic.py @@ -1,5 +1,6 @@ """ (C) Copyright 2018-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -15,20 +16,23 @@ class RbldBasic(TestWithServers): :avocado: recursive """ - def run_rebuild_test(self, pool_quantity): - """Run the rebuild test for the specified number of pools. + def test_rebuild_basic(self): + """ + Test Description: + The most basic rebuild test. + + Use Cases: + Multiple pool rebuild, single client, various record/object counts - Args: - pool_quantity (int): number of pools to test + :avocado: tags=all,daily_regression + :avocado: tags=vm + :avocado: tags=rebuild,pool,daos_cmd + :avocado: tags=RbldBasic,test_rebuild_basic """ # Get the test parameters - pools = [] - containers = [] - for _ in range(pool_quantity): - pools.append(self.get_pool(create=False)) - containers.append(self.get_container(pools[-1], create=False)) - rank = self.params.get("rank", "/run/testparams/*") - obj_class = self.params.get("object_class", "/run/testparams/*") + rank = self.random.choice(list(self.server_managers[0].ranks.keys())) + datasets = self.params.get("datasets", "/run/testparams/*") + num_pools = self.params.get("num_pools", "/run/testparams/*") # Collect server configuration information server_count = len(self.hostlist_servers) @@ -39,26 +43,33 @@ def run_rebuild_test(self, pool_quantity): "Running with %s servers, %s engines per server, and %s targets per engine", server_count, engine_count, target_count) - # Create the pools and confirm their status + self.log_step(f"Create {num_pools} pools") + pools = [] status = True - for pool in pools: - pool.create() + for _ in range(num_pools): + pool = self.get_pool() + pools.append(pool) status &= pool.check_pool_info( pi_nnodes=server_count * engine_count, pi_ntargets=server_count * engine_count * target_count, - pi_ndisabled=0 - ) + pi_ndisabled=0) status &= pool.check_rebuild_status(rs_state=1, rs_obj_nr=0, rs_rec_nr=0, rs_errno=0) self.assertTrue(status, "Error confirming pool info before rebuild") - # Create containers in each pool and fill them with data - rs_obj_nr = [] - rs_rec_nr = [] - for container in containers: - container.create() - container.write_objects(rank, obj_class) + self.log_step(f"Create {len(datasets)} containers and fill with data") + containers = [] + for idx, dataset in enumerate(datasets): + # Round-robin create containers across pools + container = self.get_container(pools[idx % len(pools)]) + containers.append(container) + (oclass, data_size, object_qty, record_qty) = dataset + container.update_params( + data_size=data_size, object_qty=object_qty, record_qty=record_qty) + container.write_objects(rank, oclass) # Determine how many objects will need to be rebuilt + rs_obj_nr = [] + rs_rec_nr = [] for container in containers: target_rank_lists = container.get_target_rank_lists(" prior to rebuild") rebuild_qty = container.get_target_rank_count(rank, target_rank_lists) @@ -72,24 +83,21 @@ def run_rebuild_test(self, pool_quantity): rs_rec_nr[-1], container.object_qty.value * container.record_qty.value, container, rank) - # Manually exclude the specified rank - for index, pool in enumerate(pools): - if index == 0: - self.server_managers[0].stop_ranks([rank], self.d_log, True) - else: - # Use the direct dmg pool exclude command to avoid updating the pool version again - pool.exclude([rank]) + self.log_step(f"Exclude random rank {rank}") + self.server_managers[0].stop_ranks([rank], self.d_log, True) + for pool in pools: + pool.exclude([rank]) - # Wait for recovery to start for first pool. - pools[0].wait_for_rebuild_to_start() + self.log_step("Wait for rebuild to start") + pools[0].wait_for_rebuild_to_start(interval=3) - # Wait for recovery to complete + self.log_step("Wait for rebuild to end") for pool in pools: - pool.wait_for_rebuild_to_end() + pool.wait_for_rebuild_to_end(interval=3) - # Check the pool information after the rebuild + self.log_step("Verify pool info after rebuild") status = True - for index, pool in enumerate(pools): + for pool in pools: status &= pool.check_pool_info( pi_nnodes=server_count * engine_count, pi_ntargets=server_count * engine_count * target_count, @@ -99,41 +107,10 @@ def run_rebuild_test(self, pool_quantity): rs_state=2, rs_errno=0) self.assertTrue(status, "Error confirming pool info after rebuild") - # Verify the data after rebuild + self.log_step("Verify container data after rebuild") for container in containers: container.set_prop(prop="status", value="healthy") if container.object_qty.value != 0: self.assertTrue(container.read_objects(), "Data verification error after rebuild") - self.log.info("Test Passed") - - def test_simple_rebuild(self): - """JIRA ID: DAOS-XXXX Rebuild-001. - - Test Description: - The most basic rebuild test. - - Use Cases: - single pool rebuild, single client, various record/object counts - - :avocado: tags=all,daily_regression - :avocado: tags=vm - :avocado: tags=rebuild,pool,daos_cmd - :avocado: tags=RbldBasic,test_simple_rebuild - """ - self.run_rebuild_test(1) - - def test_multipool_rebuild(self): - """JIRA ID: DAOS-XXXX (Rebuild-002). - - Test Description: - Expand on the basic test by rebuilding 2 pools at once. - Use Cases: - multi-pool rebuild, single client, various object and record counts - - :avocado: tags=all,daily_regression - :avocado: tags=vm - :avocado: tags=rebuild,pool - :avocado: tags=RbldBasic,test_multipool_rebuild - """ - self.run_rebuild_test(self.params.get("pool_quantity", "/run/testparams/*")) + self.log_step("Test Passed") diff --git a/src/tests/ftest/rebuild/basic.yaml b/src/tests/ftest/rebuild/basic.yaml index 31d203f3036..80fbb15d9ce 100644 --- a/src/tests/ftest/rebuild/basic.yaml +++ b/src/tests/ftest/rebuild/basic.yaml @@ -1,8 +1,9 @@ -# change host names to your reserved nodes, the -# required quantity is indicated by the placeholders hosts: test_servers: 6 + test_clients: 1 + timeout: 600 + server_config: name: daos_server crt_timeout: 10 @@ -19,42 +20,28 @@ server_config: class: ram scm_mount: /mnt/daos system_ram_reserved: 1 + pool: size: 1G pool_query_timeout: 30 properties: rd_fac:2 + container: akey_size: 5 dkey_size: 5 debug: true properties: rd_fac:2 - data: !mux - small_data: - data_size: 32 - large_data: - data_size: 250 - objects: !mux - zero_objects: - # Don't run duplicate variants when there are zero objects - !filter-only : "/run/container/data/small_data" # yamllint disable-line rule:colons - !filter-only : "/run/container/records/one_record" # yamllint disable-line rule:colons - object_qty: 0 - one_object: - object_qty: 1 - twenty_objects: - object_qty: 20 - records: !mux - one_record: - record_qty: 1 - ten_records: - record_qty: 10 + testparams: - ranks: !mux - rank1: - rank: 1 - rank2: - rank: 2 - rank3: - rank: 3 - pool_quantity: 2 - object_class: OC_RP_3G1 + num_pools: 2 + datasets: + # oclass, data_size, object_qty, record_qty + - [OC_RP_3G1, 32, 0, 1] + - [OC_RP_3G1, 32, 1, 1] + - [OC_RP_3G1, 32, 1, 10] + - [OC_RP_3G1, 32, 20, 1] + - [OC_RP_3G1, 32, 20, 10] + - [OC_RP_3G1, 250, 1, 1] + - [OC_RP_3G1, 250, 1, 10] + - [OC_RP_3G1, 250, 20, 1] + - [OC_RP_3G1, 250, 20, 10] diff --git a/src/tests/ftest/recovery/cat_recov_core.yaml b/src/tests/ftest/recovery/cat_recov_core.yaml index 6cba4c6caf3..23200a8b403 100644 --- a/src/tests/ftest/recovery/cat_recov_core.yaml +++ b/src/tests/ftest/recovery/cat_recov_core.yaml @@ -1,6 +1,6 @@ hosts: test_servers: 4 -timeout: 5400 +timeout: 5700 server_config: name: daos_server engines_per_host: 2 diff --git a/src/tests/ftest/recovery/ms_membership.py b/src/tests/ftest/recovery/ms_membership.py index f7c7c8250a5..f490f125989 100644 --- a/src/tests/ftest/recovery/ms_membership.py +++ b/src/tests/ftest/recovery/ms_membership.py @@ -1,5 +1,6 @@ """ (C) Copyright 2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -19,7 +20,7 @@ def test_checker_on_admin_excluded(self): """Test checker can only be run when the system status is AdminExcluded. 1. Call dmg check enable. - 2. Stop rank 1. + 2. Stop 1 random rank 3. Call dmg system query and check that status of at least one rank is not "checkerstarted". We verify that the new checkerstarted state is properly changed. 4. Call dmg check start. It should show error because the stopped rank is not at @@ -38,45 +39,42 @@ def test_checker_on_admin_excluded(self): """ dmg_command = self.get_dmg_command() - # 1. Call dmg check enable. + self.log_step("Call dmg check enable.") dmg_command.check_enable() - # 2. Stop rank 1. - dmg_command.system_stop(ranks="1") + self.log_step("Stop 1 random rank") + rank_to_stop = self.random.choice(list(self.server_managers[0].ranks.keys())) + dmg_command.system_stop(ranks=rank_to_stop) - # 3. Call dmg system query and check that status of at least one rank is not - # "checkerstarted". We verify that the new checkerstarted state is properly - # changed. + self.log_step("Verify the the stopped rank is stopped and all others are checkerstarted") query_out = dmg_command.system_query() - not_checkerstarted_found = False for member in query_out["response"]["members"]: - if member["state"] != "checkerstarted": - not_checkerstarted_found = True - break - if not not_checkerstarted_found: - # All rank's status is "checkerstarted". - self.fail("All rank's status is checkerstarted!") - - # 4. Call dmg check start. It should show error because the stopped rank is not at - # CheckerStarted state. + if member["rank"] == rank_to_stop: + if member["state"] != "stopped": + self.fail(f"Stopped rank has invalid state: {member['state']}") + elif member["state"] != "checkerstarted": + self.fail("Expected non-stopped ranks to be in state checkstarted") + + self.log_step("Verify dmg check start fails") try: dmg_command.check_start() + self.fail("dmg check start did not fail as expected") except CommandFailure as error: self.log.info("dmg check start is expected to fail. Error: %s", error) - # 5. Call dmg check query. It should show error because the stopped rank is not at - # CheckerStarted state. + self.log_step("Verify dmg check query fails") try: dmg_command.check_query() + # To be fixed by DAOS-18001 + # self.fail("dmg check query did not fail as expected") except CommandFailure as error: self.log.info("dmg check query is expected to fail. Error: %s", error) - # 6. Call dmg check disable. It should work. + self.log_step("Verify dmg check disable works") dmg_command.check_disable() - # 7. Restart the stopped rank for cleanup. - self.log.info("Restart stopped rank for cleanup.") - dmg_command.system_start(ranks="1") + self.log_step("Restart stopped rank for cleanup.") + dmg_command.system_start(ranks=rank_to_stop) def test_enable_disable_admin_excluded(self): """Test dmg system exclude and clear-exclude. @@ -84,15 +82,14 @@ def test_enable_disable_admin_excluded(self): Test admin can enable and disable the rank state to AdminExcluded when the rank is down. - 1. Stop rank 1. - 2. Set rank 1 to AdminExcluded by calling dmg system exclude --ranks=1 and verify + 1. Stop all ranks + 2. Set 1 random rank to AdminExcluded by calling dmg system exclude --ranks=1 and verify the state has been changed. 3. Verify that the checker can be run with AdminExcluded state by calling enable, start, query, and disable. Verify that none of the commands returns error. - 4. Disable AdminExcluded of rank 1 by calling dmg system clear-exclude --ranks=1 + 4. Disable AdminExcluded of the excluded rank by calling dmg system clear-exclude and verify the state has been changed. - 5. Servers haven't been started, so update the expected state of rank 0 for - cleanup. + 5. Start the ranks again Jira ID: DAOS-11704 @@ -104,15 +101,16 @@ def test_enable_disable_admin_excluded(self): errors = [] dmg_command = self.get_dmg_command() - # 1. Stop rank 1. - dmg_command.system_stop(ranks="1") + self.log_step("Stop all ranks") + dmg_command.system_stop(force=True) - # 2. Set rank 1 to AdminExcluded and verify the state has been changed. - self.server_managers[-1].system_exclude(ranks=[1]) + self.log_step("Exclude 1 random rank") + rank_to_exclude = self.random.choice(list(self.server_managers[0].ranks.keys())) + self.server_managers[-1].system_exclude(ranks=[rank_to_exclude]) - # 3. Verify that the checker can be run with AdminExcluded state. + self.log_step("Verify that the checker can be run with AdminExcluded state.") try: - dmg_command.check_enable() + dmg_command.check_enable(stop=False) dmg_command.check_start() dmg_command.check_query() # We need to start after calling dmg system clear-exclude, otherwise the start @@ -122,11 +120,14 @@ def test_enable_disable_admin_excluded(self): msg = f"dmg check command failed! {error}" errors.append(msg) - # 4. Disable AdminExcluded of rank 1 and verify the state has been changed. - self.server_managers[-1].system_clear_exclude(ranks=[1]) + self.log_step( + "Disable AdminExcluded of the stopped rank and verify the state has been changed.") + self.server_managers[-1].system_clear_exclude(ranks=[rank_to_exclude]) - # 5. Servers haven't been started, so update the expected state of rank 0 for - # cleanup. - self.server_managers[-1].update_expected_states(0, ['stopped']) + self.log_step("Verify all ranks are able to restart") + dmg_command.system_start() + all_ranks = list(self.server_managers[0].ranks.keys()) + self.server_managers[-1].update_expected_states(all_ranks, ['joined']) + self.server_managers[-1].check_rank_state(all_ranks, ['joined']) report_errors(test=self, errors=errors) diff --git a/src/tests/ftest/recovery/ms_membership.yaml b/src/tests/ftest/recovery/ms_membership.yaml index b698773d1bf..e4157dae333 100644 --- a/src/tests/ftest/recovery/ms_membership.yaml +++ b/src/tests/ftest/recovery/ms_membership.yaml @@ -1,13 +1,23 @@ hosts: - test_servers: 2 + test_servers: 4 timeout: 180 server_config: - name: daos_server - engines_per_host: 1 + engines_per_host: 2 engines: 0: - targets: 1 + pinned_numa_node: 0 + nr_xs_helpers: 1 + fabric_iface: ib0 + fabric_iface_port: 31317 + log_file: daos_server0.log + storage: auto + 1: + pinned_numa_node: 1 + nr_xs_helpers: 1 + fabric_iface: ib1 + fabric_iface_port: 31417 + log_file: daos_server1.log storage: auto system_ram_reserved: 1 diff --git a/src/tests/ftest/server/storage_tiers.py b/src/tests/ftest/server/storage_tiers.py index e627b8d62c2..0dd71d79b65 100644 --- a/src/tests/ftest/server/storage_tiers.py +++ b/src/tests/ftest/server/storage_tiers.py @@ -1,5 +1,6 @@ """ (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -25,9 +26,10 @@ def test_tiers(self): Test Description: Verify storage tiers are correctly obtained from the test yaml file. + Does not test or rely on actual hardware being present. :avocado: tags=all,pr - :avocado: tags=hw,medium + :avocado: tags=vm :avocado: tags=server,storage,storage_tiers :avocado: tags=StorageTiers,test_tiers """ @@ -58,9 +60,9 @@ def test_tiers(self): "class": storage_class, "bdev_list": self.params.get("bdev_list", namespace), } - storage_roles = self.params.get("roles", namespace, None) + storage_roles = self.params.get("bdev_roles", namespace, None) if storage_roles: - data["roles"] = storage_roles + data["bdev_roles"] = storage_roles storage.append(data) expected.append(storage) self.log.info("expected:\n%s", yaml.dump(expected, default_flow_style=False)) diff --git a/src/tests/ftest/server/storage_tiers.yaml b/src/tests/ftest/server/storage_tiers.yaml index 225bbae47e1..b9bc500e30f 100644 --- a/src/tests/ftest/server/storage_tiers.yaml +++ b/src/tests/ftest/server/storage_tiers.yaml @@ -5,7 +5,9 @@ hosts: setup: start_servers: false start_agents: false - server_config_namespace: /run/server_configurations/*/server_config/* + server_config_namespace: /run/configurations/*/server_config/* + +check_server_storage: false timeout: 1800 @@ -33,23 +35,23 @@ storage_1: &storage_ram storage_2: &storage_nvme_all class: nvme - roles: [wal, index, data] + bdev_roles: [wal, index, data] storage_3: &storage_nvme_wal_index class: nvme - roles: [wal, index] + bdev_roles: [wal, index] storage_4: &storage_nvme_wal class: nvme - roles: [wal] + bdev_roles: [wal] storage_5: &storage_nvme_index class: nvme - roles: [index] + bdev_roles: [index] storage_6: &storage_nvme_data class: nvme - roles: [data] + bdev_roles: [data] configurations: !mux 1_tier_dcpm: diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index a900769f4d6..7c0d3ccf08d 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -1,10 +1,12 @@ """ (C) Copyright 2022-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ # pylint: disable=too-many-lines import glob +import json import os import re import sys @@ -17,6 +19,7 @@ from util.environment_utils import TestEnvironment from util.host_utils import get_local_host from util.run_utils import find_command, run_local, run_remote, stop_processes +from util.storage_utils import find_pci_address from util.systemctl_utils import stop_service from util.user_utils import get_chown_command from util.yaml_utils import get_test_category @@ -154,6 +157,73 @@ def cleanup_processes(logger, test, result): return not any_found +def check_server_storage(logger, test, test_result, stage): + """Verify that the expected server storage devices exist. + + Args: + logger (Logger): logger for the messages produced by this method + test (TestInfo): the test information + test_result (TestResult): the test result used to update the status of the test + stage (str): current launch test execution stage + + Returns: + bool: True if all expected devices found on all server hosts; False otherwise + """ + status = True + commands = { + "scm_list": + r"ndctl list -c -v | grep pmem | sed -e 's/.*:\"\(.*\)\"/\/dev\/\1/' | grep -Eo '{}'", + "bdev_list": + r"lspci -D | grep -E '{}'" + } + logger.debug("-" * 80) + logger.debug(f"Verifying server storage during the {stage.lower()} stage for \'{test}\'") + if "check_server_storage" in test.yaml_info and test.yaml_info["check_server_storage"] is False: + logger.debug(" - Test requested skip via 'check_server_storage' yaml entry") + return status + for key, command in commands.items(): + if key not in test.yaml_info or test.yaml_info[key] is None: + # No need to check storage w/o a scm/bdev entry + logger.debug(f" - No {key} storage defined for this test") + continue + + # Convert single engine storage entries into lists + test_yaml_info = test.yaml_info[key] + if not isinstance(test_yaml_info, (list, tuple)): + test_yaml_info = [test_yaml_info] + + if "daos" in test_yaml_info: + # No need to check ram class storage (using /mnt/daos* scm) + logger.debug(f" - {key} storage using ram class") + continue + + result = run_remote( + logger, test.host_info.servers.hosts, command.format('|'.join(test_yaml_info))) + if not result.passed: + message = f"Failure detected attempting to verify {key} storage for '{test}'" + test_result.fail_test(logger, stage, message) + status = False + continue + + item_set = set(test_yaml_info) + logger.debug(f" - Expecting the following {key} storage: {item_set}") + for data in result.output: + if key == "bdev_list": + result_set = set(find_pci_address("\n".join(data.stdout))) + else: + result_set = set(data.stdout) + match = bool(item_set & result_set == item_set) + if not match: + logger.error( + f"Mismatch detected for {key}: expected:{item_set} != detected:{result_set}") + message = f"Mismatch detected verifying {key} storage for '{test}'" + test_result.fail_test(logger, stage, message) + status = False + else: + logger.debug(f"Storage verified: expected:{item_set} == detected:{result_set}") + return status + + def archive_files(logger, summary, hosts, source, pattern, destination, depth, threshold, timeout, test_result, test=None): # pylint: disable=too-many-arguments @@ -565,6 +635,46 @@ def create_steps_log(logger, job_results_dir, test_result): return 0 +def record_variant_details(logger, job_results_dir, test_result, details): + """Record variant test results to the details.json file. + + Args: + logger (Logger): logger for the messages produced by this method + job_results_dir (str): path to the avocado job results + test_result (TestResult): the test result used to update the status of the test + details (dict): dictionary to update with test results + + Returns: + int: status code: 16384 = problem parsing the results.json; 0 = success + """ + logger.debug("=" * 80) + logger.info("Recording variant details/results") + if "test_variants" not in details: + details["test_variants"] = [] + + test_logs_lnk = os.path.join(job_results_dir, "latest") + test_logs_dir = os.path.realpath(test_logs_lnk) + results_json = os.path.join(test_logs_dir, 'results.json') + try: + with open(results_json, "r", encoding="utf-8") as results: + data = json.loads(results.read()) + for index, test in enumerate(data["tests"]): + if len(details["test_variants"]) == index: + # Add an entry for this test variant the first time its run in the loop + details["test_variants"].append({ + "variant": test["id"].split(";")[0], + "status/loop": [], + "time/loop": []}) + # Append status and run times for each loop + details["test_variants"][index]["status/loop"].append(test["status"]) + details["test_variants"][index]["time/loop"].append(f"{round(test['time'], 2)}s") + except Exception: # pylint: disable=broad-except + message = f"Error parsing {results_json}" + test_result.fail_test(logger, "Process", message, sys.exc_info()) + return 16384 + return 0 + + def rename_avocado_test_dir(logger, test, job_results_dir, test_result, jenkins_xml, total_repeats): """Append the test name to its avocado job-results directory name. @@ -827,7 +937,7 @@ def replace_xml(logger, xml_file, pattern, replacement, xml_data, test_result): def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, archive, rename, - jenkins_xml, core_files, threshold, total_repeats): + jenkins_xml, core_files, threshold, total_repeats, details): # pylint: disable=too-many-arguments """Process the test results. @@ -851,6 +961,7 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a core_files (dict): location and pattern defining where core files may be written threshold (str): optional upper size limit for test log files total_repeats (int): total number of times the test will be repeated + details (dict): dictionary to update with test results Returns: int: status code: 0 = success, >0 = failure @@ -869,6 +980,10 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a if not cleanup_processes(logger, test, test_result): return_code |= 4096 + # Check storage devices for servers + if not check_server_storage(logger, test, test_result, "Process"): + return_code |= 512 + # Mark the test execution as failed if a results.xml file is not found test_logs_dir = os.path.realpath(os.path.join(job_results_dir, "latest")) results_xml = os.path.join(test_logs_dir, "results.xml") @@ -952,6 +1067,9 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a # Generate a steps.log file return_code |= create_steps_log(logger, job_results_dir, test_result) + # Add test variant results to the details + return_code |= record_variant_details(logger, job_results_dir, test_result, details) + # Optionally rename the test results directory for this test if rename: return_code |= rename_avocado_test_dir( diff --git a/src/tests/ftest/util/command_utils_base.py b/src/tests/ftest/util/command_utils_base.py index 837e61b339f..0c19e4ea322 100644 --- a/src/tests/ftest/util/command_utils_base.py +++ b/src/tests/ftest/util/command_utils_base.py @@ -1,5 +1,6 @@ """ (C) Copyright 2020-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -159,6 +160,9 @@ def get_yaml_value(self, name, test, path): name (str): name of the value in the yaml file test (Test): avocado Test object to use to read the yaml file path (str): yaml path where the name is to be found + + Returns: + bool: True if a new value was assigned; False if the default value was assigned """ if self._yaml_key is not None: # Use the yaml key name instead of the variable name @@ -167,6 +171,7 @@ def get_yaml_value(self, name, test, path): self.value = test.config.get(name, path, self._default) else: self.value = test.params.get(name, path, self._default) + return self.value != self._default def update(self, value, name=None, append=False): """Update the value of the parameter. @@ -333,10 +338,14 @@ def get_yaml_value(self, name, test, path): name (str): name of the value in the yaml file test (Test): avocado Test object to use to read the yaml file path (str): yaml path where the name is to be found + + Returns: + bool: True if a new value was assigned; False if the default value was assigned """ - super().get_yaml_value(name, test, path) + new_value = super().get_yaml_value(name, test, path) self._add_directory() self.log.debug(" Added the directory: %s => %s", name, self.value) + return new_value def update(self, value, name=None, append=False): """Update the value of the parameter. diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index e36d750500e..bed802347f3 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -1,5 +1,6 @@ """ (C) Copyright 2018-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -109,6 +110,7 @@ class TestEnvironment(): 'bullseye_file': 'COVFILE', 'daos_prefix': 'DAOS_TEST_PREFIX', 'agent_user': 'DAOS_TEST_AGENT_USER', + 'systemd_path': 'DAOS_TEST_SYSTEMD_PATH', 'systemd_library_path': 'DAOS_TEST_SYSTEMD_LIBRARY_PATH', 'control_config': 'DAOS_TEST_CONTROL_CONFIG', 'agent_config': 'DAOS_TEST_AGENT_CONFIG', @@ -120,7 +122,7 @@ def __init__(self): self.set_defaults(None) def set_defaults(self, logger, servers=None, clients=None, provider=None, insecure_mode=None, - agent_user=None, log_dir=None, systemd_lib_path=None): + agent_user=None, log_dir=None, systemd_path=None, systemd_library_path=None): """Set the default test environment variable values with optional inputs. Args: @@ -135,7 +137,8 @@ def set_defaults(self, logger, servers=None, clients=None, provider=None, insecu agent_user (str, optional): user account to use when running the daos_agent. Defaults to None. log_dir (str, optional): test log directory base path. Defaults to None. - systemd_lib_path (str, optional): systemd library path. Defaults to None. + systemd_path (str, optional): systemd path. Defaults to None. + systemd_library_path (str, optional): systemd library path. Defaults to None. Raises: TestEnvironmentException: if there are any issues setting environment variable default @@ -154,8 +157,10 @@ def set_defaults(self, logger, servers=None, clients=None, provider=None, insecu self.insecure_mode = insecure_mode if agent_user is not None: self.agent_user = agent_user - if systemd_lib_path is not None: - self.systemd_library_path = systemd_lib_path + if systemd_path is not None: + self.systemd_path = systemd_path + if systemd_library_path is not None: + self.systemd_library_path = systemd_library_path # Set defaults for any unset values if self.log_dir is None: @@ -520,6 +525,24 @@ def agent_user(self, value): """ self.__set_value('agent_user', value) + @property + def systemd_path(self): + """Get the systemd PATH. + + Returns: + str: the systemd PATH + """ + return os.environ.get(self.__ENV_VAR_MAP['systemd_path']) + + @systemd_path.setter + def systemd_path(self, value): + """Set the systemd PATH. + + Args: + value (str): the systemd PATH + """ + self.__set_value('systemd_path', value) + @property def systemd_library_path(self): """Get the systemd LD_LIBRARY_PATH. @@ -607,7 +630,8 @@ def config_file_directories(self): def set_test_environment(logger, test_env=None, servers=None, clients=None, provider=None, insecure_mode=False, details=None, agent_user=None, log_dir=None, - systemd_lib_path=None): + systemd_path=None, systemd_library_path=None): + # pylint: disable=too-many-arguments """Set up the test environment. Args: @@ -625,7 +649,8 @@ def set_test_environment(logger, test_env=None, servers=None, clients=None, prov agent_user (str, optional): user account to use when running the daos_agent. Defaults to None. log_dir (str, optional): test log directory base path. Defaults to None. - systemd_lib_path (str, optional): systemd library path. Defaults to None. + systemd_path (str, optional): systemd library path. Defaults to None. + systemd_library_path (str, optional): systemd library path. Defaults to None. Raises: TestEnvironmentException: if there is a problem setting up the test environment @@ -638,7 +663,7 @@ def set_test_environment(logger, test_env=None, servers=None, clients=None, prov # Get the default fabric interface, provider, and daos prefix test_env.set_defaults( logger, servers, clients, provider, insecure_mode, agent_user, log_dir, - systemd_lib_path) + systemd_path, systemd_library_path) logger.info("Testing with interface: %s", test_env.interface) logger.info("Testing with provider: %s", test_env.provider) logger.info("Testing with daos_prefix: %s", test_env.daos_prefix) diff --git a/src/tests/ftest/util/general_utils.py b/src/tests/ftest/util/general_utils.py index 941697f8af0..9a4f82de62d 100644 --- a/src/tests/ftest/util/general_utils.py +++ b/src/tests/ftest/util/general_utils.py @@ -1011,7 +1011,8 @@ def percent_change(val1, val2): return math.nan -def get_journalctl_command(since, until=None, system=False, units=None, identifiers=None): +def get_journalctl_command(since, until=None, system=False, units=None, identifiers=None, + run_user="root"): """Get the journalctl command to capture all unit/identifier activity from since to until. Args: @@ -1023,12 +1024,13 @@ def get_journalctl_command(since, until=None, system=False, units=None, identifi None. identifiers (str/list, optional): show messages for the specified syslog identifier(s). Defaults to None. + run_user (str, optional): user to run as. Defaults to root Returns: str: journalctl command to capture all unit activity """ - command = ["sudo", os.path.join(os.sep, "usr", "bin", "journalctl")] + command = [os.path.join(os.sep, "usr", "bin", "journalctl")] if system: command.append("--system") for key, values in {"unit": units or [], "identifier": identifiers or []}.items(): @@ -1037,10 +1039,10 @@ def get_journalctl_command(since, until=None, system=False, units=None, identifi command.append("--since=\"{}\"".format(since)) if until: command.append("--until=\"{}\"".format(until)) - return " ".join(command) + return command_as_user(" ".join(command), run_user) -def get_journalctl(hosts, since, until, journalctl_type): +def get_journalctl(hosts, since, until, journalctl_type, run_user="root"): """Run the journalctl on the hosts. Args: @@ -1048,6 +1050,7 @@ def get_journalctl(hosts, since, until, journalctl_type): since (str): Start time to search the log. until (str): End time to search the log. journalctl_type (str): String to search in the log. -t param for journalctl. + run_user (str, optional): user to run as. Defaults to root Returns: list: a list of dictionaries containing the following key/value pairs: @@ -1055,7 +1058,9 @@ def get_journalctl(hosts, since, until, journalctl_type): "data": data requested for the group of hosts """ - command = get_journalctl_command(since, until, True, identifiers=journalctl_type) + system = run_user != getuser() + command = get_journalctl_command( + since, until, system, identifiers=journalctl_type, run_user=run_user) err = "Error gathering system log events" return get_host_data(hosts=hosts, command=command, text="journalctl", error=err) diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index 0f7284c50ef..02f90083f03 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -1,5 +1,6 @@ """ (C) Copyright 2022-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -14,7 +15,7 @@ from ClusterShell.NodeSet import NodeSet from slurm_setup import SlurmSetup, SlurmSetupException # pylint: disable=import-error,no-name-in-module -from util.collection_utils import TEST_RESULTS_DIRS, collect_test_result +from util.collection_utils import TEST_RESULTS_DIRS, check_server_storage, collect_test_result from util.data_utils import dict_extract_values, list_flatten, list_unique from util.environment_utils import TestEnvironment from util.host_utils import HostException, HostInfo, get_local_host, get_node_set @@ -129,12 +130,12 @@ def setup_systemctl(logger, servers, clients, test_env): __add_systemctl_override( logger, servers, "daos_server.service", "root", os.path.join(test_env.daos_prefix, "bin", "daos_server"), test_env.server_config, - None, None)) + test_env.systemd_path, test_env.systemd_library_path)) systemctl_configs.update( __add_systemctl_override( logger, clients, "daos_agent.service", test_env.agent_user, os.path.join(test_env.daos_prefix, "bin", "daos_agent"), test_env.agent_config, - None, None)) + test_env.systemd_path, test_env.systemd_library_path)) return systemctl_configs @@ -182,7 +183,8 @@ def summarize_run(logger, mode, status): 1024: "ERROR: Failed to rename logs and results after one or more tests!", 2048: "ERROR: Core stack trace files detected!", 4096: "ERROR: Unexpected processes or mounts found running!", - 8192: "ERROR: Failed to create steps.log!" + 8192: "ERROR: Failed to create steps.log!", + 16384: "ERROR: Failed to parse test variant status!" } for bit_code, error_message in bit_error_map.items(): if status & bit_code == bit_code: @@ -245,6 +247,9 @@ class TestInfo(): "client_partition", "client_reservation", "client_users", + "scm_list", + "bdev_list", + "check_server_storage", ] def __init__(self, test_file, order, yaml_extension=None): @@ -290,10 +295,13 @@ def set_yaml_info(self, logger, include_local_host=False): """ self.yaml_info = {"include_local_host": include_local_host} yaml_data = get_yaml_data(self.yaml_file) + for extra_yaml in self.extra_yaml: + yaml_data.update(get_yaml_data(extra_yaml)) info = {} for key in self.YAML_INFO_KEYS: # Get the unique values with lists flattened - values = list_unique(list_flatten(dict_extract_values(yaml_data, [key], (str, list)))) + values = list_unique(list_flatten( + dict_extract_values(yaml_data, [key], (str, list, bool)))) if values: # Use single value if list only contains 1 element info[key] = values if len(values) > 1 else values[0] @@ -378,6 +386,7 @@ def __init__(self, avocado, launch_result, total_tests, total_repeats, tag_filte def prepare(self, logger, test_log_file, test, repeat, user_create, slurm_setup, control_host, partition_hosts, clear_mounts): + # pylint: disable=too-many-return-statements """Prepare the test for execution. Args: @@ -404,28 +413,30 @@ def prepare(self, logger, test_log_file, test, repeat, user_create, slurm_setup, self.test_result.start() # Setup the test host information, including creating any required slurm partitions - status = self._setup_host_information( - logger, test, slurm_setup, control_host, partition_hosts) - if status: - return status + if not self._setup_host_information( + logger, test, slurm_setup, control_host, partition_hosts): + return 128 # Setup (remove/create/list) the common test directory on each test host - status = self._setup_test_directory(logger, test) - if status: - return status + if not self._setup_test_directory(logger, test): + return 128 # Setup additional test users - status = self._user_setup(logger, test, user_create) - if status: - return status + if not self._user_setup(logger, test, user_create): + return 128 # Remove existing mount points on each test host - status = self._clear_mount_points(logger, test, clear_mounts) - if status: - return status + if not self._clear_mount_points(logger, test, clear_mounts): + return 128 + + # Check storage devices for servers + if not check_server_storage(logger, test, self.test_result, "Prepare"): + return 128 # Generate certificate files for the test - return self._generate_certs(logger) + if not self._generate_certs(logger): + return 128 + return 0 def execute(self, logger, test, repeat, number, sparse, fail_fast): """Run the specified test. @@ -474,8 +485,8 @@ def execute(self, logger, test, repeat, number, sparse, fail_fast): logger.info("Total test time: %ss", end_time - start_time) return return_code - def process(self, logger, job_results_dir, test, repeat, stop_daos, archive, rename, - jenkins_xml, core_files, threshold): + def process(self, logger, job_results_dir, test, loop, stop_daos, archive, rename, + jenkins_xml, core_files, threshold, details): # pylint: disable=too-many-arguments """Process the test results. @@ -489,13 +500,14 @@ def process(self, logger, job_results_dir, test, repeat, stop_daos, archive, ren Args: logger (Logger): logger for the messages produced by this method test (TestInfo): the test information - repeat (int): the test repetition number + loop (int): the test repetition number (1s based) stop_daos (bool): whether or not to stop daos servers/clients after the test archive (bool): whether or not to collect remote files generated by the test rename (bool): whether or not to rename the default avocado job-results directory names jenkins_xml (bool): whether or not to update the results.xml to use Jenkins-style names core_files (dict): location and pattern defining where core files may be written threshold (str): optional upper size limit for test log files + details (dict): dictionary to update with test results Returns: int: status code: 0 = success, >0 = failure @@ -506,10 +518,10 @@ def process(self, logger, job_results_dir, test, repeat, stop_daos, archive, ren logger.debug("=" * 80) logger.info( "Processing the %s test after the run on repeat %s/%s", - test, repeat, self.total_repeats) + test, loop, self.total_repeats) status = collect_test_result( logger, test, self.test_result, job_results_dir, stop_daos, archive, rename, - jenkins_xml, core_files, threshold, self.total_repeats) + jenkins_xml, core_files, threshold, self.total_repeats, details) # Mark the execution of the test as passed if nothing went wrong if self.test_result.status is None: @@ -531,7 +543,7 @@ def _setup_host_information(self, logger, test, slurm_setup, control_host, parti partition_hosts (NodeSet): slurm partition hosts Returns: - int: status code: 0 = success, 128 = failure + bool: True if successful; False otherwise """ logger.debug("-" * 80) logger.debug("Setting up host information for %s", test) @@ -544,14 +556,14 @@ def _setup_host_information(self, logger, test, slurm_setup, control_host, parti if not exists and not slurm_setup: message = f"Error missing {partition} partition" self.test_result.fail_test(logger, "Prepare", message, None) - return 128 + return False if slurm_setup and exists: logger.info( "Removing existing %s partition to ensure correct configuration", partition) if not delete_partition(logger, control_host, partition).passed: message = f"Error removing existing {partition} partition" self.test_result.fail_test(logger, "Prepare", message, None) - return 128 + return False if slurm_setup: hosts = partition_hosts.difference(test.yaml_info["test_servers"]) logger.debug( @@ -560,12 +572,12 @@ def _setup_host_information(self, logger, test, slurm_setup, control_host, parti if not hosts: message = "Error no partition hosts exist after removing the test servers" self.test_result.fail_test(logger, "Prepare", message, None) - return 128 + return False logger.info("Creating the '%s' partition with the '%s' hosts", partition, hosts) if not create_partition(logger, control_host, partition, hosts).passed: message = f"Error adding the {partition} partition" self.test_result.fail_test(logger, "Prepare", message, None) - return 128 + return False # Define the hosts for this test try: @@ -573,7 +585,7 @@ def _setup_host_information(self, logger, test, slurm_setup, control_host, parti except LaunchException: message = "Error setting up host information" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 + return False # Log the test information msg_format = "%3s %-40s %-60s %-20s %-20s" @@ -584,7 +596,7 @@ def _setup_host_information(self, logger, test, slurm_setup, control_host, parti logger.debug( msg_format, test.name.order, test.test_file, test.yaml_file, test.host_info.servers.hosts, test.host_info.clients.hosts) - return 0 + return True def _setup_test_directory(self, logger, test): """Set up the common test directory on all hosts. @@ -594,7 +606,7 @@ def _setup_test_directory(self, logger, test): test (TestInfo): the test information Returns: - int: status code: 0 = success, 128 = failure + bool: True if successful; False otherwise """ logger.debug("-" * 80) test_env = TestEnvironment() @@ -616,8 +628,8 @@ def _setup_test_directory(self, logger, test): if not run_remote(logger, hosts, command).passed: message = "Error setting up the common test directory on all hosts" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 - return 0 + return False + return True def _user_setup(self, logger, test, create=False): """Set up test users on client nodes. @@ -628,7 +640,7 @@ def _user_setup(self, logger, test, create=False): create (bool, optional): whether to create extra test users defined by the test Returns: - int: status code: 0 = success, 128 = failure + bool: True if successful; False otherwise """ users = test.get_yaml_client_users() clients = test.host_info.clients.hosts @@ -649,16 +661,16 @@ def _user_setup(self, logger, test, create=False): group_gid[group] = self._query_create_group(logger, clients, group, create) except LaunchException as error: self.test_result.fail_test(logger, "Prepare", str(error), sys.exc_info()) - return 128 + return False gid = group_gid.get(group, None) try: self._query_create_user(logger, clients, user, gid, create) except LaunchException as error: self.test_result.fail_test(logger, "Prepare", str(error), sys.exc_info()) - return 128 + return False - return 0 + return True @staticmethod def _query_create_group(logger, hosts, group, create=False): @@ -742,10 +754,10 @@ def _clear_mount_points(self, logger, test, clear_mounts): clear_mounts (list): mount points to remove before the test Returns: - int: status code: 0 = success, 128 = failure + bool: True if successful; False otherwise """ if not clear_mounts: - return 0 + return True logger.debug("-" * 80) hosts = test.host_info.all_hosts @@ -765,20 +777,20 @@ def _clear_mount_points(self, logger, test, clear_mounts): if not self._remove_super_blocks(logger, mount_hosts, mount_point): message = "Error removing superblocks for existing mount points" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 + return False if not self._remove_shared_memory_segments(logger, hosts): message = "Error removing shared memory segments for existing mount points" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 + return False for mount_point, mount_hosts in mount_point_hosts.items(): if not self._remove_mount_point(logger, mount_hosts, mount_point): message = "Error removing existing mount points" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 + return False - return 0 + return True def _remove_super_blocks(self, logger, hosts, mount_point): """Remove the super blocks from the specified mount point. @@ -847,10 +859,11 @@ def _remove_mount_point(self, logger, hosts, mount_point): def _generate_certs(self, logger): """Generate the certificates for the test. - Returns: + Args: logger (Logger): logger for the messages produced by this method - int: status code: 0 = success, 128 = failure + Returns: + bool: True if successful; False otherwise """ logger.debug("-" * 80) logger.debug("Generating certificates") @@ -862,12 +875,12 @@ def _generate_certs(self, logger): if not run_local(logger, f"/usr/bin/rm -rf {certs_dir}").passed: message = "Error removing old certificates" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 + return False if not run_local(logger, f"{command} {test_env.log_dir}").passed: message = "Error generating certificates" self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) - return 128 - return 0 + return False + return True def _collect_crash_files(self, logger): """Move any avocado crash files into job-results/latest/crashes. @@ -1260,12 +1273,14 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop if not code_coverage.setup(logger, result.tests[0]): return_code |= 128 + self._details["tests"] = [] + # Run each test for as many repetitions as requested for loop in range(1, repeat + 1): logger.info("-" * 80) logger.info("Starting test repetition %s/%s", loop, repeat) - for index, test in enumerate(self.tests): + for sequence, test in enumerate(self.tests): # Define a log for the execution of this test for this repetition test_log_file = test.get_log_file(logdir, loop, repeat) logger.info("-" * 80) @@ -1273,6 +1288,17 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop test_file_handler = get_file_handler(test_log_file, LOG_FILE_FORMAT, logging.DEBUG) logger.addHandler(test_file_handler) + if len(self._details["tests"]) == sequence: + # Add an entry for this test the first time its run in the loop + self._details["tests"].append({ + "sequence": sequence + 1, + "test_file": str(test), + "results": test_log_file, + "clients": str(test.yaml_info["test_clients"]), + "servers": str(test.yaml_info["test_servers"]), + "bdev_list": test.yaml_info["bdev_list"]}) + self._details["tests"][sequence]["repetitions"] = loop + # Prepare the hosts to run the tests step_status = runner.prepare( logger, test_log_file, test, loop, user_create, slurm_setup, self._control, @@ -1283,12 +1309,12 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop continue # Run the test with avocado - return_code |= runner.execute(logger, test, loop, index + 1, sparse, fail_fast) + return_code |= runner.execute(logger, test, loop, sequence + 1, sparse, fail_fast) # Archive the test results return_code |= runner.process( logger, job_results_dir, test, loop, stop_daos, archive, rename, - jenkins_log, core_files, threshold) + jenkins_log, core_files, threshold, self._details["tests"][sequence]) # Display disk usage after the test is complete display_disk_space(logger, logdir) diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index a6129bd1d41..3a0510127c2 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -541,7 +541,7 @@ def find_command(source, pattern, depth, other=None): def stop_processes(log, hosts, pattern, verbose=True, timeout=60, exclude=None, force=False, - full_command=False): + full_command=False, user="root"): """Stop the processes on each hosts that match the pattern. Args: @@ -602,9 +602,9 @@ def stop_processes(log, hosts, pattern, verbose=True, timeout=60, exclude=None, log.debug( "Killing%s any processes on %s that match %s and then waiting %s seconds", step[0], result.passed_hosts, pattern_match, step[1]) - kill_command = f"sudo /usr/bin/pkill{step[0]} {pattern}" + kill_command = command_as_user(f"/usr/bin/pkill{step[0]} {pattern}", user) if full_command: - kill_command = f"sudo /usr/bin/pkill{step[0]} --full -x {pattern}" + kill_command = command_as_user(f"/usr/bin/pkill{step[0]} --full -x {pattern}", user) run_remote(log, result.passed_hosts, kill_command, verbose, timeout) time.sleep(step[1]) result = run_remote(log, result.passed_hosts, search_command, verbose, timeout) diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index 032a8183659..996f380b004 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -436,6 +436,7 @@ class EngineYamlParameters(YamlParameters): "common": [ "D_LOG_FILE_APPEND_PID=1", "DAOS_POOL_RF=4", + "DAOS_REBUILD_WAIT_EC_PAUSE=1", "CRT_EVENT_DELAY=1", # pylint: disable-next=fixme # FIXME disable space cache since some tests need to verify instant pool space @@ -463,6 +464,8 @@ def __init__(self, base_namespace, index, provider=None, max_storage_tiers=MAX_S Defaults to MAX_STORAGE_TIERS. """ namespace = [os.sep] + base_namespace.split(os.sep)[1:-1] + ["engines", str(index), "*"] + common_ns = [os.sep] + base_namespace.split(os.sep)[1:-1] + ["engines_common", "*"] + self.common_namespace = os.path.join(*common_ns) self._base_namespace = base_namespace self._index = index self._provider = provider or os.environ.get("D_PROVIDER", "ofi+tcp") @@ -518,7 +521,11 @@ def get_params(self, test): Args: test (Test): avocado Test object """ - super().get_params(test) + # Get the values for the engine yaml parameters + for name in self.get_param_names(): + if not getattr(self, name).get_yaml_value(name, test, self.namespace): + # If a new value was not assigned, check the engine's common namespace for a value + getattr(self, name).get_yaml_value(name, test, self.common_namespace) # Override the log file file name with the test log file name if hasattr(test, "server_log") and test.server_log is not None: diff --git a/src/tests/ftest/util/test_utils_pool.py b/src/tests/ftest/util/test_utils_pool.py index 99bf439b523..925c1a6ae4b 100644 --- a/src/tests/ftest/util/test_utils_pool.py +++ b/src/tests/ftest/util/test_utils_pool.py @@ -958,7 +958,7 @@ def check_free_space(self, expected_scm=None, expected_nvme=None, timeout=30, in return False def check_rebuild_status(self, rs_version=None, rs_seconds=None, - rs_errno=None, rs_state=None, rs_padding32=None, + rs_errno=None, rs_state=None, rs_padding16=None, rs_fail_rank=None, rs_toberb_obj_nr=None, rs_obj_nr=None, rs_rec_nr=None, rs_size=None): # pylint: disable=unused-argument @@ -975,7 +975,7 @@ def check_rebuild_status(self, rs_version=None, rs_seconds=None, rs_seconds (int, optional): rebuild seconds. Defaults to None. rs_errno (int, optional): rebuild error number. Defaults to None. rs_state (int, optional): rebuild state flag. Defaults to None. - rs_padding32 (int, optional): padding. Defaults to None. + rs_padding16 (int, optional): padding. Defaults to None. rs_fail_rank (int, optional): rebuild fail target. Defaults to None. rs_toberb_obj_nr (int, optional): number of objects to be rebuilt. Defaults to None. diff --git a/src/tests/suite/daos_container.c b/src/tests/suite/daos_container.c index 243a829c317..8f61d42c63c 100644 --- a/src/tests/suite/daos_container.c +++ b/src/tests/suite/daos_container.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -2698,7 +2699,8 @@ co_rf_simple(void **state) assert_rc_equal(rc, 0); /* Hang the rebuild */ - test_set_engine_fail_loc(arg, CRT_NO_RANK, DAOS_REBUILD_TGT_REBUILD_HANG | DAOS_FAIL_ALWAYS); + test_set_engine_fail_loc(arg, CRT_NO_RANK, + DAOS_REBUILD_TGT_REBUILD_HANG | DAOS_FAIL_ALWAYS); /* IO testing */ io_oid = daos_test_oid_gen(arg->coh, OC_RP_4G1, 0, 0, arg->myrank); rc = daos_obj_open(arg->coh, io_oid, DAOS_OO_RW, &io_oh, NULL); @@ -2716,11 +2718,7 @@ co_rf_simple(void **state) iod.iod_size = 1; iod.iod_nr = 1; iod.iod_recxs = &recx; - iod.iod_type = DAOS_IOD_ARRAY; - print_message("obj update should success before RF broken\n"); - rc = daos_obj_update(io_oh, DAOS_TX_NONE, 0, &dkey, 1, &iod, &sgl, - NULL); - assert_rc_equal(rc, 0); + iod.iod_type = DAOS_IOD_ARRAY; if (arg->myrank == 0) { arg->no_rebuild = 1; @@ -3073,7 +3071,7 @@ co_redun_lvl(void **state) goto out; rc = ranks_on_same_node(poolmap, 7, ranks); assert_rc_equal(rc, 0); - for (i = 5; i > 0; i++) { + for (i = 5; i > 0; i--) { if (i != ranks[0] && i != ranks[1]) { ranks[2] = i; break; @@ -3148,16 +3146,12 @@ co_redun_lvl(void **state) assert_rc_equal(rc, -DER_INVAL); } - test_set_engine_fail_loc(arg, CRT_NO_RANK, DAOS_REBUILD_TGT_REBUILD_HANG | DAOS_FAIL_ALWAYS); - print_message("obj update should success before RF broken\n"); + test_set_engine_fail_loc(arg, CRT_NO_RANK, + DAOS_REBUILD_TGT_REBUILD_HANG | DAOS_FAIL_ALWAYS); io_oid = daos_test_oid_gen(arg->coh, OC_EC_2P2G1, 0, 0, arg->myrank); rc = daos_obj_open(arg->coh, io_oid, DAOS_OO_RW, &io_oh, NULL); assert_rc_equal(rc, 0); - rc = daos_obj_update(io_oh, DAOS_TX_NONE, 0, &dkey, 1, &iod, &sgl, - NULL); - assert_rc_equal(rc, 0); - /* exclude one more rank on another NODE dom */ if (arg->myrank == 0) rebuild_single_pool_rank(arg, ranks[2], false); diff --git a/src/tests/suite/daos_rebuild.c b/src/tests/suite/daos_rebuild.c index 20a751504ac..a30e6f1c643 100644 --- a/src/tests/suite/daos_rebuild.c +++ b/src/tests/suite/daos_rebuild.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1473,7 +1474,7 @@ rebuild_delay_and_reintegrate(void **state) if (!test_runable(arg, 6)) return; - rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "delay_rebuild"); + rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "exclude\\;delay_rebuild"); assert_rc_equal(rc, 0); for (i = 0; i < OBJ_NR; i++) { oids[i] = daos_test_oid_gen(arg->coh, DAOS_OC_R3S_SPEC_RANK, 0, 0, arg->myrank); @@ -1507,7 +1508,7 @@ rebuild_delay_and_extend(void **state) if (!test_runable(arg, 6)) return; - rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "delay_rebuild"); + rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "exclude\\;delay_rebuild"); assert_rc_equal(rc, 0); for (i = 0; i < OBJ_NR; i++) { oids[i] = daos_test_oid_gen(arg->coh, DAOS_OC_R3S_SPEC_RANK, 0, 0, arg->myrank); diff --git a/src/tests/suite/daos_rebuild_ec.c b/src/tests/suite/daos_rebuild_ec.c index 77df81730bd..b640a3df3de 100644 --- a/src/tests/suite/daos_rebuild_ec.c +++ b/src/tests/suite/daos_rebuild_ec.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1366,7 +1367,7 @@ rebuild_ec_combined_ops(void **state) if (!test_runable(arg, 6)) return; - rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "delay_rebuild"); + rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal", "exclude\\;delay_rebuild"); assert_int_equal(rc, 0); oid = daos_test_oid_gen(arg->coh, OC_EC_4P2GX, 0, 0, arg->myrank); ioreq_init(&req, arg->coh, oid, DAOS_IOD_ARRAY, arg); diff --git a/src/utils/daos_autotest.c b/src/utils/daos_autotest.c index a8b1b1a39b2..5d9f591b7fa 100644 --- a/src/utils/daos_autotest.c +++ b/src/utils/daos_autotest.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2020-2022 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -440,7 +441,9 @@ kv_put(daos_handle_t oh, daos_size_t size) char *val; daos_event_t *evp; int rc, usage_ratio1, usage_ratio2; - int eq_rc; + int eq_rc; + int num_events; + int completions = 0; clock_t last_query = start, current; deadline_count = 1; @@ -485,12 +488,14 @@ kv_put(daos_handle_t oh, daos_size_t size) * Max request in flight reached, wait for one i/o to * complete to reuse the slot */ - rc = daos_eq_poll(eq, 1, DAOS_EQ_WAIT, 1, &evp); - if (rc < 0) - break; - if (rc == 0) { - rc = -DER_IO; - break; + while (1) { + rc = daos_eq_poll(eq, 1, DAOS_EQ_NOWAIT, 1, &evp); + if (rc > 0) + break; + if (rc < 0) { + rc = -DER_IO; + break; + } } /** Check if completed operation failed */ @@ -540,19 +545,20 @@ kv_put(daos_handle_t oh, daos_size_t size) break; deadline_count++; - } /** Wait for completion of all in-flight requests */ - do { - eq_rc = daos_eq_poll(eq, 1, DAOS_EQ_WAIT, 1, &evp); - if (rc == 0 && eq_rc == 1) { - rc = evp->ev_error; + num_events = daos_eq_query(eq, DAOS_EQR_ALL, 0, NULL); + while (1) { + eq_rc = daos_eq_poll(eq, 1, DAOS_EQ_NOWAIT, 1, &evp); + if (eq_rc > 0) + completions += eq_rc; + if (eq_rc < 0) { + rc = eq_rc; + break; } - } while (eq_rc == 1); - - if (rc == 0 && eq_rc < 0) { - rc = eq_rc; + if (completions >= num_events) + break; } D_FREE(val); @@ -581,6 +587,8 @@ kv_get(daos_handle_t oh, daos_size_t size) uint64_t res = 0; int rc; int eq_rc; + int completions = 0; + int num_events; total_nr = deadline_count; setup_progress(); @@ -620,12 +628,14 @@ kv_get(daos_handle_t oh, daos_size_t size) * Max request in flight reached, wait for one i/o to * complete to reuse the slot */ - rc = daos_eq_poll(eq, 1, DAOS_EQ_WAIT, 1, &evp); - if (rc < 0) - break; - if (rc == 0) { - rc = -DER_IO; - break; + while (1) { + rc = daos_eq_poll(eq, 1, DAOS_EQ_NOWAIT, 1, &evp); + if (rc > 0) + break; + if (rc < 0) { + rc = -DER_IO; + break; + } } /** Check if completed operation failed */ @@ -665,11 +675,12 @@ kv_get(daos_handle_t oh, daos_size_t size) } /** Wait for completion of all in-flight requests */ - do { - eq_rc = daos_eq_poll(eq, 1, DAOS_EQ_WAIT, 1, &evp); - if (rc == 0 && eq_rc == 1) { - rc = evp->ev_error; - if (rc == 0) { + num_events = daos_eq_query(eq, DAOS_EQR_ALL, 0, NULL); + while (1) { + eq_rc = daos_eq_poll(eq, 1, DAOS_EQ_NOWAIT, 1, &evp); + if (eq_rc > 0) { + completions += eq_rc; + if (!evp->ev_error) { int slot = evp - ev_array; if (val_sz[slot] != size) { @@ -679,10 +690,12 @@ kv_get(daos_handle_t oh, daos_size_t size) } } } - } while (eq_rc == 1); - - if (rc == 0 && eq_rc < 0) { - rc = eq_rc; + if (eq_rc < 0) { + rc = eq_rc; + break; + } + if (completions >= num_events) + break; } D_FREE(val); diff --git a/src/utils/daos_hdlr.c b/src/utils/daos_hdlr.c index 3d63aedb655..68f217cd12c 100644 --- a/src/utils/daos_hdlr.c +++ b/src/utils/daos_hdlr.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1490,8 +1491,8 @@ dm_connect(struct cmd_args_s *ap, rc = dm_cont_get_all_props(ap, ca->src_coh, &props, false, false, false); else - rc = dm_cont_get_all_props(ap, ca->src_coh, &props, - true, false, true); + rc = dm_cont_get_all_props(ap, ca->src_coh, &props, true, false, + false); if (rc != 0) { DH_PERROR_DER(ap, rc, "Failed to get container properties"); D_GOTO(err, rc); diff --git a/src/vos/evt_iter.c b/src/vos/evt_iter.c index fe5ad6b4f48..c251fb3b45c 100644 --- a/src/vos/evt_iter.c +++ b/src/vos/evt_iter.c @@ -214,6 +214,8 @@ evt_iter_intent(struct evt_iterator *iter) return DAOS_INTENT_DISCARD; if (iter->it_options & EVT_ITER_FOR_MIGRATION) return DAOS_INTENT_MIGRATION; + if (iter->it_options & EVT_ITER_FOR_CHECK) + return DAOS_INTENT_CHECK; return DAOS_INTENT_DEFAULT; } diff --git a/src/vos/sys_db.c b/src/vos/sys_db.c index d1f6d4bce98..cd1f10179f2 100644 --- a/src/vos/sys_db.c +++ b/src/vos/sys_db.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2020-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -278,8 +279,7 @@ db_trav_cb(daos_handle_t ih, vos_iter_entry_t *entry, vos_iter_type_t type, { struct db_trav_args *ta = data; - return ta->ta_cb(ta->ta_db, ta->ta_table, &entry->ie_key, - ta->ta_cb_args); + return ta->ta_cb(ta->ta_db, ta->ta_table, &entry->ie_key, ta->ta_cb_args, acts); } static int diff --git a/src/vos/vos_obj.c b/src/vos/vos_obj.c index 364d3d9145c..fff7e059eed 100644 --- a/src/vos/vos_obj.c +++ b/src/vos/vos_obj.c @@ -1526,6 +1526,8 @@ recx_get_flags(struct vos_obj_iter *oiter, bool embed) options |= EVT_ITER_FOR_DISCARD; if (oiter->it_flags & VOS_IT_FOR_MIGRATION) options |= EVT_ITER_FOR_MIGRATION; + if (oiter->it_flags & VOS_IT_FOR_CHECK) + options |= EVT_ITER_FOR_CHECK; return options; } diff --git a/utils/githooks/prepare-commit-msg.d/10-submodule-update-check.py b/utils/githooks/prepare-commit-msg.d/10-submodule-update-check.py index 34a30c55c6e..561ae503c1c 100755 --- a/utils/githooks/prepare-commit-msg.d/10-submodule-update-check.py +++ b/utils/githooks/prepare-commit-msg.d/10-submodule-update-check.py @@ -7,8 +7,6 @@ import subprocess # nosec import sys -modified_re = re.compile(r'^(?:M|A)(\s+)(?P.*)') - def rebasing(): """Determines if the current operation is a rebase""" @@ -20,39 +18,53 @@ def rebasing(): return stdout.split('\n', maxsplit=1)[0].startswith("* (no branch, rebasing") -def submodule_check(modname, msg_file): - """Determines if a sub-module has been updated""" - modified = False +def git_submodules(): + """Get a list of submodules""" + lines = subprocess.check_output(['git', 'submodule', 'status']).decode().rstrip().split('\n') + return [ + line[1:].split(' ')[1] + for line in lines + if line + ] + +def git_modified_files(): + """Get a list of modified files""" + modified_re = re.compile(r'^(?:M|A)(\s+)(?P.*)') + files = [] with subprocess.Popen(['git', 'status', '--porcelain'], stdout=subprocess.PIPE) as proc: out, _ = proc.communicate() for line in out.decode().splitlines(): match = modified_re.match(line) if match: - modified = modified | (match.group('name') == modname) + files.append(match.group('name')) + return files - if not rebasing() and modified: - with open(msg_file, 'r', encoding='utf-8') as file: - lines = file.readlines() - message = f'# WARNING *** This patch modifies the {modname} reference. ' \ - 'Are you sure this is intended? *** WARNING' +def main(msg_file): + """main""" + # Ignore if this is a rebase + if rebasing(): + return + + modified_files = git_modified_files() + submodules = git_submodules() - if lines[0] != message: - lines = [message, "\n", "\n"] + lines + messages = [] + for modified_submodule in set(modified_files).intersection(submodules): + messages.append( + f'# WARNING *** This patch modifies the {modified_submodule} reference. ' + 'Are you sure this is intended? *** WARNING\n\n') - with open(msg_file, 'w', encoding='utf-8') as file: - file.writelines(lines) + if messages: + with open(msg_file, 'r', encoding='utf-8') as file: + lines = file.readlines() + lines = messages + lines -def main(msg_file): - """main""" - for line in subprocess.check_output(['git', 'submodule', - 'status']).decode().rstrip().split('\n'): - if line: - submodule_check(line[1:].split(' ')[1], msg_file) + with open(msg_file, 'w', encoding='utf-8') as file: + file.writelines(lines) if __name__ == '__main__': - main(sys.argv[1]) diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 07b9908cb2d..783ddce317e 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -10,9 +10,9 @@ %else %global daos_build_args client test %endif -%global mercury_version 2.4 -%global libfabric_version 1.15.1-1 -%global argobots_version 1.2 +%global mercury_version 2.4.0-7 +%global libfabric_version 1.20 +%global argobots_version 1.2-3 %global __python %{__python3} %if (0%{?rhel} >= 8) @@ -23,7 +23,7 @@ Name: daos Version: 2.6.4 -Release: 3%{?relval}%{?dist} +Release: 7%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -77,10 +77,10 @@ BuildRequires: spdk-devel >= 22.01.2 %endif %if (0%{?rhel} >= 8) BuildRequires: isa-l-devel -BuildRequires: libisa-l_crypto-devel +BuildRequires: libisa-l_crypto-devel >= 2.24.0-1 %else -BuildRequires: libisal-devel -BuildRequires: libisal_crypto-devel +BuildRequires: libisal-devel >= 2.31.1-7 +BuildRequires: libisal_crypto-devel >= 2.24.0-3 %endif BuildRequires: openssl-devel BuildRequires: libevent-devel @@ -152,11 +152,11 @@ Requires: ndctl # needed to set PMem configuration goals in BIOS through control-plane %if (0%{?suse_version} >= 1500) Requires: ipmctl >= 03.00.00.0423 -Requires: libpmemobj1 >= 2.1.0-1.suse1500 +Requires: libpmemobj1 >= 2.1.0-3.suse1500 Requires: libfabric1 >= %{libfabric_version} %else Requires: ipmctl >= 03.00.00.0468 -Requires: libpmemobj >= 2.1.0-1%{?dist} +Requires: libpmemobj >= 2.1.0-6%{?dist} %endif Requires: libfabric >= %{libfabric_version} Requires: mercury >= %{mercury_version} @@ -629,6 +629,24 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Fri Oct 31 2025 Mohamad Chaarawi 2.6.4-7 +- Revert bump of libfabric to 1.20 + +* Thu Oct 23 2025 Phillip Henderson 2.6.4-6 +- Fourth release candidate for 2.6.4 + +* Tue Oct 21 2025 Ryon Jensen 2.6.4-5 +- Bump libfabric version >= 1.22.0-4 +- Bump mercury version >= 2.4.0-7 +- Bump argobots version >= 1.2-3 +- Bump libisal version >= 2.31.1-7 +- Bump libisal_crypto version >= 2.24.0-3 +- Bump pmemobj1 >= 2.1.0-3 +- Bump pmemobj >= 2.1.0-6 + +* Fri Oct 17 2025 Phillip Henderson 2.6.4-4 +- Third release candidate for 2.6.4 + * Wed Sep 10 2025 Jeff Olivier 2.6.4-3 - Ensure daos-server installs spdk