From 80c543c82b74e866de60c3e8433b14e1d7ababb8 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 14 Nov 2026 13:33:35 +0000 Subject: [PATCH] httpclient: a request waiting out a pool's reconnect backoff ends when its reactor stops, rather than waiting forever (#584) After a failed connect the pool holds new connects back for 200 to 400 ms, and a request that found no connection inside that window queued with nothing in flight. Only the pool's sweep, a reactor ticker, could then open the next connection or fail the request at its acquire deadline, and a stopped reactor runs no ticker, so the request waited forever. RingHttpClient's per-origin pools share the acquire path, and had the same gap. Such a request now starts the retry itself: an open that waits out the rest of the backoff on the ring, one IRingHost.SubmitTimeout on a RingOpSource the pool holds, and then connects. It counts in _opening while it waits, so one runs at a time and the sweep's refill stands aside. The stop cancels the wait, and the retry then connects nothing and reports no failure, but wakes the waiters, which end with ObjectDisposedException as they already did behind a cancelled connect. A retry behind the backoff now fires when the backoff ends rather than on the next 250 ms tick, and ConnectionCount counts it while it waits. Tests, in Http's HttpClientBackoffTests: a request queued behind the backoff ends with ObjectDisposedException when its reactor stops, through HttpClientPool and through RingHttpClient; on main both are still waiting 2 s after the stop. A request waiting 2 s on an origin whose every open fails sees 3 to 12 opens; with the retry connecting at once it saw 22495. --- .../ioxide.httpclient/Http1/HttpClientPool.cs | 32 ++++- .../HttpClientBackoffTests.cs | 121 ++++++++++++++++++ tests/Ioxide.Tests.Http/Program.cs | 1 + 3 files changed, 150 insertions(+), 4 deletions(-) create mode 100644 tests/Ioxide.Tests.Http/HttpClientBackoffTests.cs diff --git a/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs b/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs index b4d801b8..62ac9819 100644 --- a/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs +++ b/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs @@ -34,6 +34,9 @@ public sealed class HttpClientPool : IDisposable private int _requests; // inside SendAsync: acquiring, waiting or sending private bool _disposed; + // What an open started behind the backoff gate waits on. One is enough: such an open starts only with none opening. + private readonly RingOpSource _backoff = new(); + // Warm: PoolSize connections opened up front and kept that way, the long-lived upstream shape. // Otherwise connections open as requests need them, which is how RingHttpClient's pools run. private readonly bool _warm; @@ -161,7 +164,7 @@ private async ValueTask AcquireAsync() // Honour the same backoff gate Sweep() does. Without it, a dead origin turns this loop // into a connect() storm: open, refuse, wake, reopen - thousands of syscalls a second - // on the reactor thread. Behind the gate the ticker paces the retries instead. One open + // on the reactor thread. Behind the gate the retry waits the backoff out, below. One open // in flight per waiting request, so a burst that finds no connection opens in parallel. if (_live < _options.PoolSize && _opening <= _waiters.Count && Native.MonotonicMs >= _reopenAtMs) { @@ -173,6 +176,12 @@ private async ValueTask AcquireAsync() throw new HttpClientException(AcquireTimeoutMessage()); } + // Nothing opening: the retry starts now, waiting the backoff out on the ring, so a stop ends this wait too. + if (_opening == 0 && _live < _options.PoolSize) + { + StartOpen(Math.Max(1, _reopenAtMs - Native.MonotonicMs)); + } + // No timer of its own: the sweep fails a waiter at its deadline. A Task.WhenAny with a // Task.Delay ran the hand-over through the thread pool and back for every request that // waited - most of them, in a pool that is the bottleneck. Completed from the reactor, @@ -223,7 +232,7 @@ private void Discard(HttpClientConnection connection) WakeWaiters(); // they must re-check: a replacement is opening } - private void StartOpen() + private void StartOpen(long backoffMs = 0) { if (_disposed) { @@ -232,13 +241,28 @@ private void StartOpen() _live++; _opening++; - _ = OpenOneAsync(); + _ = OpenOneAsync(backoffMs); } - private async Task OpenOneAsync() + private async Task OpenOneAsync(long backoffMs) { try { + if (backoffMs > 0) + { + ValueTask backoff = _backoff.Prepare(); + _host.SubmitTimeout(backoffMs * 1_000_000L, _backoff); + int waited = await backoff; + + // Not -ETIME, the wait running its course, but the stop: no connect, and no failure to report. + if (waited is not (-62 or 0) || _disposed) + { + _live--; + WakeWaiters(); + return; + } + } + HttpClientConnection connection = await HttpClientConnection.ConnectAsync(_host, _options); _reopenAtMs = 0; // healthy - allow immediate replenishment _lastOpenFailure = null; diff --git a/tests/Ioxide.Tests.Http/HttpClientBackoffTests.cs b/tests/Ioxide.Tests.Http/HttpClientBackoffTests.cs new file mode 100644 index 00000000..d6e0f6fc --- /dev/null +++ b/tests/Ioxide.Tests.Http/HttpClientBackoffTests.cs @@ -0,0 +1,121 @@ +using System.Net; +using System.Net.Sockets; +using ioxide; +using ioxide.httpclient; +using ioxide.timer; + +namespace Ioxide.Tests; + +/// +/// A request that finds no connection while its pool backs off after a failed open waits behind a retry that waits +/// the backoff out on the ring. The retries keep the backoff's pace, and the reactor's stop ends the request, as it +/// ends one waiting on a connect or a response. +/// +internal static class HttpClientBackoffTests +{ + public static void Register(Runner runner) + { + runner.Test("httpclient h1: a request waiting out the reconnect backoff ends when its reactor stops", () => + { + string outcome = EndedByTheStop(reactor => HttpClientPool.Start(reactor, new HttpClientOptions + { Port = (ushort)TestServer.DeadPort(), PoolSize = 1, AcquireTimeoutMs = 30_000 }).GetAsync("/")); + + Assert.Equal("ObjectDisposedException: The reactor has stopped. Object name: 'Reactor'.", outcome); + }); + + runner.Test("ring client: a request waiting out the reconnect backoff ends when its reactor stops", () => + { + string outcome = EndedByTheStop(reactor => RingHttpClient.Start(reactor, new RingHttpClientOptions + { Pool = new HttpClientOptions { PoolSize = 1, AcquireTimeoutMs = 30_000 } }) + .GetAsync($"http://127.0.0.1:{TestServer.DeadPort()}/")); + + Assert.Equal("ObjectDisposedException: The reactor has stopped. Object name: 'Reactor'.", outcome); + }); + + runner.Test("httpclient h1: a request waiting on an origin whose every open fails retries at the backoff's pace", () => + { + // Accepted, then closed before the handshake: the open fails as a refused one does, and the origin counts it. + using var origin = new TcpListener(IPAddress.Loopback, 0); + origin.Start(); + int opens = 0; + _ = Task.Run(async () => + { + try + { + while (true) + { + using Socket accepted = await origin.AcceptSocketAsync(); + Interlocked.Increment(ref opens); + } + } + catch (Exception) + { + // The listener stopped. + } + }); + + TlsClientContext tls = TestServer.Own(TlsClientContext.Create(new TlsClientOptions { ServerName = "localhost" })); + string outcome = OnReactor.Run(async reactor => + { + HttpClientPool pool = HttpClientPool.Start(reactor, new HttpClientOptions + { + Port = (ushort)((IPEndPoint)origin.LocalEndpoint).Port, + Tls = tls, + PoolSize = 1, + AcquireTimeoutMs = 2_000, + }); + try + { + using HttpClientResponse response = await pool.GetAsync("/"); + return $"answered {response.Status}"; + } + catch (HttpClientException e) + { + return e.Message; + } + }); + int counted = Volatile.Read(ref opens); + + Assert.True(outcome.StartsWith("no connection to "), $"expected the acquire to time out, got: {outcome}"); + + // The first open, then one per backoff of 200 to 400 ms, while the request waits its 2 s. + Assert.True(counted is >= 3 and <= 12, $"{counted} opens in the request's 2 s, where its backoff allows 6 to 11"); + }); + } + + // Makes the request on a reactor of its own and stops that reactor once 50 ms have passed: the refused connect is + // long done, and the backoff after it, 200 ms at least, has the request waiting. How the request ended, once stopped. + private static string EndedByTheStop(Func> request) + { + var made = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + (_, Reactor reactor, Thread thread) = TestServer.StartConfigured( + (_, connection) => { connection.DecRef(); return Task.CompletedTask; }, + new ServerConfig + { + RecvBufferSize = 4096, RecvSlots = 256, + Tcp = new TcpOptions { WriteSlabSize = 16 * 1024, PoolMax = 64, RecvQueueEntries = 64 }, + }, + r => _ = MakeAsync(r)); + + Assert.True(made.Task.Wait(10_000), "the request was never made"); + reactor.Stop(); + Assert.True(thread.Join(10_000), "the reactor did not stop"); + + Task call = made.Task.Result; + try + { + return call.Wait(2_000) ? $"answered {call.Result.Status}" : "still waiting 2 s after its reactor stopped"; + } + catch (AggregateException e) + { + return $"{e.InnerException!.GetType().Name}: {e.InnerException.Message.ReplaceLineEndings(" ")}"; + } + + async Task MakeAsync(Reactor r) + { + Task pending = request(r).AsTask(); + await new RingTimer(r).DelayAsync(50); + made.SetResult(pending); + } + } +} diff --git a/tests/Ioxide.Tests.Http/Program.cs b/tests/Ioxide.Tests.Http/Program.cs index a44780b0..b47345c2 100644 --- a/tests/Ioxide.Tests.Http/Program.cs +++ b/tests/Ioxide.Tests.Http/Program.cs @@ -24,6 +24,7 @@ private static int Main() HttpClientResponseLimitTests.Register(runner); HttpClientHttp10Tests.Register(runner); HttpClientThreadTests.Register(runner); + HttpClientBackoffTests.Register(runner); TlsClientTests.Register(runner); // Areas reserved for the failing-test review pass; empty until one lands.