diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f4cb25..3e6599e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -417,6 +417,18 @@ if(GTest_FOUND) WORKING_DIRECTORY ${CMAKE_BINARY_DIR} LABELS "integration;docker;auth" ) + + # Fail fast if a suite hangs (e.g. a server/CLI-version regression stalls a + # subprocess) instead of letting one stuck test burn the whole CI job budget. + set_tests_properties( + PingIntegrationTests GetIntegrationTests GetWithMetadataIntegrationTests + PutIntegrationTests RemoveIntegrationTests RemoveWithVersionIntegrationTests + ReplaceWithVersionIntegrationTests PutIfAbsentIntegrationTests ReplaceIntegrationTests + ContainsKeyIntegrationTests ErrorHandlingIntegrationTests TopologyIntegrationTests + ConcurrentClientsTests HashAwareRoutingIntegrationTests ConcurrentMultiServerTests + RetryViewIntegrationTests PingRetryIntegrationTests AuthIntegrationTests + PROPERTIES TIMEOUT 300 + ) endif() # NOT WIN32 (integration tests) message(STATUS "Google Test found - unit tests enabled (integration tests: non-Windows only)") diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index d1a8627..1588f21 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -704,3 +704,35 @@ all green live against a Dockerized auth-enabled Infinispan 16.0.7. Documented i `documentation/topics/security.adoc` (new AsciiDoc user guide, `documentation/index.adoc`) with a runnable snippet in `examples/quickstart/README.md`. + +## 2026-10-12 — Integration tests hung on CI: Infinispan `:latest` CLI stopped exiting on EOF + +**Symptom.** Every CI run from Oct 9 on hung: `GetIntegrationTests` started and +produced no further output until the 30-min job timeout cancelled it (an orphan +`docker` process was left behind). Only the Ubuntu job runs integration tests +(Fedora is build-only, Windows doesn't build them), so it looked Ubuntu-specific. +`main` had last run green on 2026-09-25 and the branch on top of it was docs-only — +the code was identical, so the trigger was environmental, not the change. + +**Root cause.** The integration tests pull `quay.io/infinispan/server:`**`latest`** +(never pinned). Between Sep-25 and Oct-9 `:latest` advanced from 16.0.x to +**16.3.0**, and 16.3's `cli.sh`, when fed a command through a pipe +(`echo 'create cache …' | cli.sh -c `), now **runs the command but does not +exit on stdin EOF** — it drops into an interactive session and blocks forever. +16.0.x exited cleanly on EOF. The helper `createCacheViaCLI` (copy-pasted into 13 +integration files) wraps that in `system("… >/dev/null 2>&1")` with no timeout, so +the whole suite hung. Reproduced locally against both 16.3.0 (hangs) and 16.0.7 +(exits) via podman. + +**Fix.** Run the CLI in batch mode: `cli.sh -c `**`-f -`** (read commands from +stdin, exit at EOF). Applied to all 13 `createCacheViaCLI` copies. Also added a +per-test ctest **`TIMEOUT 300`** to every integration suite so a future stall fails +in minutes instead of burning the 30-min job budget (slowest real suite is +`TopologyIntegrationTests` at ~117s, so 300s has comfortable headroom). + +**Not pinned.** 16.3.0 is otherwise fully compatible — the full `ctest -L +integration` suite passes **18/18 green against real docker on `:latest` (16.3.0)** +(verified 2026-10-12). An earlier local `Connection reset by peer` on the Get ops +was a rootless-podman networking artifact (reproduced identically on known-good +16.0.7, and absent under real docker), not a version regression — so `:latest` was +left unpinned rather than frozen to an old release. diff --git a/docs/STATUS.md b/docs/STATUS.md index f9288dc..ed5016f 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -4,7 +4,7 @@ > If any other doc disagrees with this file, this file wins. Point-in-time > snapshots live in [`archive/`](archive/) and are historical only. > -> **Last updated:** 2026-09-25 +> **Last updated:** 2026-10-12 --- @@ -242,6 +242,12 @@ multiplexing. See "Working and shipped" below._ Windows and Fedora stay unit-only (Fedora runs inside a container with no Docker daemon; Windows doesn't build the integration tests). Windows/MSVC portability and `-Werror` build parity also landed (Sept 2026). +- **Integration tests pull `quay.io/infinispan/server:latest`** (unpinned). + Cache setup goes through `cli.sh -c -f -` (batch mode, exits on EOF) and + each ctest integration suite has `TIMEOUT 300`, so a server/CLI-version change + that stalls a subprocess fails fast instead of hanging the whole CI job. Full + suite verified green against `:latest` = 16.3.0 on 2026-10-12. See + [`DECISIONS.md`](DECISIONS.md). **Test status (verified 2026-09-25):** - Unit: **223/223** passing (`./build/unit_tests`, <1s) — +19 for the auth work diff --git a/tests/integration/ConcurrentClientsTest.cpp b/tests/integration/ConcurrentClientsTest.cpp index 87b3055..cfdb798 100644 --- a/tests/integration/ConcurrentClientsTest.cpp +++ b/tests/integration/ConcurrentClientsTest.cpp @@ -27,7 +27,7 @@ namespace { void createCacheViaCLI(const std::string& cacheName) { std::string cmd = "docker exec " + InfinispanTestEnvironment::containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + cacheName + - "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } diff --git a/tests/integration/ContainsKeyIntegrationTest.cpp b/tests/integration/ContainsKeyIntegrationTest.cpp index 9b4a857..67aa609 100644 --- a/tests/integration/ContainsKeyIntegrationTest.cpp +++ b/tests/integration/ContainsKeyIntegrationTest.cpp @@ -19,7 +19,7 @@ namespace { void createCacheViaCLI(const std::string& cacheName) { std::string cmd = "docker exec " + InfinispanTestEnvironment::containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + cacheName + - "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } diff --git a/tests/integration/ErrorHandlingIntegrationTest.cpp b/tests/integration/ErrorHandlingIntegrationTest.cpp index 132f577..c415157 100644 --- a/tests/integration/ErrorHandlingIntegrationTest.cpp +++ b/tests/integration/ErrorHandlingIntegrationTest.cpp @@ -14,7 +14,7 @@ namespace { void createCacheViaCLI(const std::string& cacheName) { std::string cmd = "docker exec " + InfinispanTestEnvironment::containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + cacheName + - "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } } // namespace diff --git a/tests/integration/GetIntegrationTest.cpp b/tests/integration/GetIntegrationTest.cpp index 3b52431..089c0a4 100644 --- a/tests/integration/GetIntegrationTest.cpp +++ b/tests/integration/GetIntegrationTest.cpp @@ -26,7 +26,7 @@ namespace { void createCacheViaCLI(const std::string& cacheName) { std::string cmd = "docker exec " + InfinispanTestEnvironment::containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + cacheName + - "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } diff --git a/tests/integration/GetWithMetadataIntegrationTest.cpp b/tests/integration/GetWithMetadataIntegrationTest.cpp index 61305c7..0e890a0 100644 --- a/tests/integration/GetWithMetadataIntegrationTest.cpp +++ b/tests/integration/GetWithMetadataIntegrationTest.cpp @@ -26,7 +26,7 @@ namespace { void createCacheViaCLI(const std::string& cacheName) { std::string cmd = "docker exec " + InfinispanTestEnvironment::containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + cacheName + - "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } diff --git a/tests/integration/PutIfAbsentIntegrationTest.cpp b/tests/integration/PutIfAbsentIntegrationTest.cpp index 447abfb..df92e14 100644 --- a/tests/integration/PutIfAbsentIntegrationTest.cpp +++ b/tests/integration/PutIfAbsentIntegrationTest.cpp @@ -19,7 +19,7 @@ namespace { void createCacheViaCLI(const std::string& cacheName) { std::string cmd = "docker exec " + InfinispanTestEnvironment::containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + cacheName + - "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } diff --git a/tests/integration/PutIntegrationTest.cpp b/tests/integration/PutIntegrationTest.cpp index dfa2963..7711d79 100644 --- a/tests/integration/PutIntegrationTest.cpp +++ b/tests/integration/PutIntegrationTest.cpp @@ -24,7 +24,7 @@ namespace { void createCacheViaCLI(const std::string& cacheName) { std::string cmd = "docker exec " + InfinispanTestEnvironment::containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + cacheName + - "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } diff --git a/tests/integration/RemoveIntegrationTest.cpp b/tests/integration/RemoveIntegrationTest.cpp index badaee2..08a44d6 100644 --- a/tests/integration/RemoveIntegrationTest.cpp +++ b/tests/integration/RemoveIntegrationTest.cpp @@ -22,7 +22,7 @@ namespace { void createCacheViaCLI(const std::string& cacheName) { std::string cmd = "docker exec " + InfinispanTestEnvironment::containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + cacheName + - "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } diff --git a/tests/integration/RemoveWithVersionIntegrationTest.cpp b/tests/integration/RemoveWithVersionIntegrationTest.cpp index b9ffa17..ab6c1af 100644 --- a/tests/integration/RemoveWithVersionIntegrationTest.cpp +++ b/tests/integration/RemoveWithVersionIntegrationTest.cpp @@ -23,7 +23,7 @@ namespace { void createCacheViaCLI(const std::string& cacheName) { std::string cmd = "docker exec " + InfinispanTestEnvironment::containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + cacheName + - "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } diff --git a/tests/integration/ReplTopologyChangeTest.cpp b/tests/integration/ReplTopologyChangeTest.cpp index 1461feb..baddafe 100644 --- a/tests/integration/ReplTopologyChangeTest.cpp +++ b/tests/integration/ReplTopologyChangeTest.cpp @@ -49,7 +49,7 @@ class ReplTopologyChangeTest : public ::testing::Test { std::string cmd = "docker exec " + server.containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + std::string(TEST_CACHE) + - "' | /opt/infinispan/bin/cli.sh -c http://localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } diff --git a/tests/integration/ReplaceIntegrationTest.cpp b/tests/integration/ReplaceIntegrationTest.cpp index d8bdacc..dff3cbc 100644 --- a/tests/integration/ReplaceIntegrationTest.cpp +++ b/tests/integration/ReplaceIntegrationTest.cpp @@ -19,7 +19,7 @@ namespace { void createCacheViaCLI(const std::string& cacheName) { std::string cmd = "docker exec " + InfinispanTestEnvironment::containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + cacheName + - "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } diff --git a/tests/integration/ReplaceWithVersionIntegrationTest.cpp b/tests/integration/ReplaceWithVersionIntegrationTest.cpp index b6cc55e..646238b 100644 --- a/tests/integration/ReplaceWithVersionIntegrationTest.cpp +++ b/tests/integration/ReplaceWithVersionIntegrationTest.cpp @@ -22,7 +22,7 @@ namespace { void createCacheViaCLI(const std::string& cacheName) { std::string cmd = "docker exec " + InfinispanTestEnvironment::containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + cacheName + - "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://admin:password@localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); } diff --git a/tests/integration/TopologyTestFixture.h b/tests/integration/TopologyTestFixture.h index 5240cc0..b29c546 100644 --- a/tests/integration/TopologyTestFixture.h +++ b/tests/integration/TopologyTestFixture.h @@ -199,7 +199,7 @@ class TopologyTest : public ::testing::Test { std::string cmd = "docker exec " + server.containerID + " bash -c \"echo 'create cache --template=org.infinispan.DIST_SYNC " + std::string(TEST_CACHE) + - "' | /opt/infinispan/bin/cli.sh -c http://localhost:11222\" >/dev/null 2>&1"; + "' | /opt/infinispan/bin/cli.sh -c http://localhost:11222 -f -\" >/dev/null 2>&1"; std::ignore = system(cmd.c_str()); }