From 38eedcde54fcef702c4bd51614aec71b0bd42ff8 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Tue, 21 Jul 2026 11:37:26 -0500 Subject: [PATCH] DAOS-19299 mercury: add patch to fix handling of canceled events This would have caused RPCs to remain in flight until timeout was reached Skip-stack-optimization: true Signed-off-by: Jerome Soumagne --- .../mercury/0007_canceled_events.patch | 76 +++++++++++++++++++ utils/build.config | 2 +- utils/rpms/mercury.changelog | 3 + utils/rpms/package_info.sh | 2 +- 4 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 deps/patches/mercury/0007_canceled_events.patch diff --git a/deps/patches/mercury/0007_canceled_events.patch b/deps/patches/mercury/0007_canceled_events.patch new file mode 100644 index 00000000000..29c79320b06 --- /dev/null +++ b/deps/patches/mercury/0007_canceled_events.patch @@ -0,0 +1,76 @@ +From 061e86519f587308d44e424e02e6b6b208ddafe4 Mon Sep 17 00:00:00 2001 +From: Jerome Soumagne +Date: Mon, 20 Jul 2026 17:14:52 -0500 +Subject: [PATCH] HG core: fix handling of canceled event on send unexpected + +In rare cases, the underlying provider could return a canceled event +without explicit user cancelation, which would lead to a lingering +recv not canceled (since CANCELED is handled separately from other +error codes). This could have in turn led to a hang until explicit +cancelation was made. +--- + src/mercury_core.c | 48 ++++++++++++++++++++++++++-------------------- + 1 file changed, 27 insertions(+), 21 deletions(-) + +diff --git a/src/mercury_core.c b/src/mercury_core.c +index 06404586..eb7d2803 100644 +--- a/src/mercury_core.c ++++ b/src/mercury_core.c +@@ -4433,30 +4433,36 @@ hg_core_send_input_cb(const struct na_cb_info *callback_info) + + if (callback_info->ret == NA_SUCCESS) { + /* Nothing */ +- } else if (callback_info->ret == NA_CANCELED) { +- HG_CHECK_SUBSYS_WARNING(rpc, +- hg_atomic_get32(&hg_core_handle->status) & HG_CORE_OP_COMPLETED, +- "Operation was completed"); +- HG_LOG_SUBSYS_DEBUG( +- rpc, "NA_CANCELED event on handle %p", (void *) hg_core_handle); +- +- hg_atomic_cas32(&hg_core_handle->ret_status, (int32_t) HG_SUCCESS, +- (int32_t) HG_CANCELED); +- } else { /* All other errors */ +- char addr_buf[HG_CORE_ADDR_MAX_SIZE]; +- size_t addr_buf_size = sizeof(addr_buf); ++ } else { + int32_t status; + +- /* Mark handle as errored */ +- status = hg_atomic_or32(&hg_core_handle->status, HG_CORE_OP_ERRORED); ++ if (callback_info->ret == NA_CANCELED) { ++ status = hg_atomic_get32(&hg_core_handle->status); + +- /* Keep first non-success ret status */ +- hg_atomic_cas32(&hg_core_handle->ret_status, (int32_t) HG_SUCCESS, +- (int32_t) callback_info->ret); +- HG_LOG_SUBSYS_ERROR(rpc, "NA callback returned error (%s, dest=\"%s\")", +- NA_Error_to_string(callback_info->ret), +- hg_core_na_addr_to_string(hg_core_handle->na_class, +- hg_core_handle->na_addr, addr_buf, &addr_buf_size)); ++ HG_CHECK_SUBSYS_WARNING( ++ rpc, status & HG_CORE_OP_COMPLETED, "Operation was completed"); ++ HG_LOG_SUBSYS_DEBUG( ++ rpc, "NA_CANCELED event on handle %p", (void *) hg_core_handle); ++ ++ hg_atomic_cas32(&hg_core_handle->ret_status, (int32_t) HG_SUCCESS, ++ (int32_t) HG_CANCELED); ++ } else { /* All other errors */ ++ char addr_buf[HG_CORE_ADDR_MAX_SIZE]; ++ size_t addr_buf_size = sizeof(addr_buf); ++ ++ /* Mark handle as errored */ ++ status = ++ hg_atomic_or32(&hg_core_handle->status, HG_CORE_OP_ERRORED); ++ ++ /* Keep first non-success ret status */ ++ hg_atomic_cas32(&hg_core_handle->ret_status, (int32_t) HG_SUCCESS, ++ (int32_t) callback_info->ret); ++ HG_LOG_SUBSYS_ERROR(rpc, ++ "NA callback returned error (%s, dest=\"%s\")", ++ NA_Error_to_string(callback_info->ret), ++ hg_core_na_addr_to_string(hg_core_handle->na_class, ++ hg_core_handle->na_addr, addr_buf, &addr_buf_size)); ++ } + + if (!(status & HG_CORE_OP_CANCELED) && + !(hg_atomic_get32(&hg_core_handle->flags) & HG_CORE_NO_RESPONSE)) { diff --git a/utils/build.config b/utils/build.config index fd4ff8c9208..e78b8ab927e 100644 --- a/utils/build.config +++ b/utils/build.config @@ -27,5 +27,5 @@ ucx=https://github.com/openucx/ucx.git [patch_versions] spdk=0001_external_isal_v26.01.patch -mercury=0001_dep_versions.patch,0002_ofi_counters.patch,0003_ofi_auth_key.patch,0004_ucx_connection_reset.patch,0005_ucx_put_flush.patch,0006_ucx_bind_bulk.patch +mercury=0001_dep_versions.patch,0002_ofi_counters.patch,0003_ofi_auth_key.patch,0004_ucx_connection_reset.patch,0005_ucx_put_flush.patch,0006_ucx_bind_bulk.patch,0007_canceled_events.patch argobots=0001_411e5b344642ebc82190fd8b125db512e5b449d1.diff,0002_bb0c908abfac4bfe37852eee621930634183c6aa.diff diff --git a/utils/rpms/mercury.changelog b/utils/rpms/mercury.changelog index ab937041061..fe6159649e9 100644 --- a/utils/rpms/mercury.changelog +++ b/utils/rpms/mercury.changelog @@ -1,4 +1,7 @@ %changelog +* Tue Sep 01 2026 Jerome Soumagne - 2.4.1-6 +- Add patch to fix handling of canceled events + * Mon Aug 31 2026 Tomasz Gromadzki - 2.4.1-5 - Rebuild to force update of defective packages. - Restore missing auto-generated runtime dependencies. diff --git a/utils/rpms/package_info.sh b/utils/rpms/package_info.sh index 2a65f0168e3..5891b96d2bf 100644 --- a/utils/rpms/package_info.sh +++ b/utils/rpms/package_info.sh @@ -39,7 +39,7 @@ export libfabric_version="1.22.0" export libfabric_release="5${distro_name}" export libfabric_full="${libfabric_version}-${libfabric_release}" export mercury_version="2.4.1" -export mercury_release="5${distro_name}" +export mercury_release="6${distro_name}" export mercury_full="${mercury_version}-${mercury_release}" export argobots_version="1.2" export argobots_release="4${distro_name}"