Publish test containers on host ports below the ephemeral range

test/Capacitor.Cli.Tests.Integration/ContainerHostPorts.cs

@@ -0,0 +1,99 @@
+using System.Net;
+using System.Net.Sockets;
+using System.Runtime.CompilerServices;
+using Docker.DotNet;
+using DotNet.Testcontainers.Configurations;
+using DotNet.Testcontainers.Containers;
+
+namespace Capacitor.Cli.Tests.Integration;
+
+/// <summary>
+/// Host ports for every container a test publishes, Testcontainers' resource reaper included. Docker's own choice, and
+/// a port-0 bind, both come from Linux's ephemeral range (32768–60999), which a loaded runner's outbound sockets and
+/// their TIME_WAITs crowd, so a start fails with "address already in use" however often it is retried there. Below
+/// that range only an explicit bind takes a port, so a pick is checked free here and walks on from where the last one
+/// ended, which keeps two starts in one process from racing for the same port.
+/// </summary>
+public static class ContainerHostPorts {
+    public const int First = 20000;
+    public const int Last  = 32767;
+
+    const int Span = Last - First + 1;
+
+    /// <summary>How many attempts a container start gets while each failure is a host-port clash.</summary>
+    const int MaxAttempts = 10;
+
+    static int _cursor = Random.Shared.Next(Span);
+
+    /// <summary>Marked obsolete, yet the reaper's port binding still reads it; the reaper's own pin fails the day a
+    /// Testcontainers upgrade stops honouring it.</summary>
+    [ModuleInitializer]
+    internal static void PublishTheResourceReaperHere() {
+#pragma warning disable CS0618
+        TestcontainersSettings.ResourceReaperPublicHostPort = _ => Next();
+#pragma warning restore CS0618
+    }
+
+    /// <summary>The next port in the range that nothing on this machine holds.</summary>
+    public static ushort Next() {
+        for (var tried = 0; tried < Span; tried++) {
+            var port = First + (int)((uint)Interlocked.Increment(ref _cursor) % Span);
+            if (IsFree(port)) return (ushort)port;
+        }
+
+        throw new InvalidOperationException($"Every host port from {First} to {Last} is held");
+    }
+
+    /// <summary>Starts a container <paramref name="build"/> publishes on a fresh <see cref="Next"/> pick each time,
+    /// rebuilding it after each clash with another process that took the picked port first.</summary>
+    public static async Task<T> StartAsync<T>(Func<T> build, CancellationToken ct = default) where T : IContainer {
+        for (var attempt = 1; ; attempt++) {
+            var container = build();
+
+            try {
+                await container.StartAsync(ct);
+
+                return container;
+            } catch (Exception ex) when (IsClash(ex) && attempt < MaxAttempts) {
+                await Console.Error.WriteLineAsync($"{container.Image.FullName}: start attempt {attempt} hit a host-port clash, retrying: {ex.Message}");
+                await container.DisposeAsync();
+                await Task.Delay(TimeSpan.FromSeconds(1), ct);
+            } catch {
+                await container.DisposeAsync();
+                throw;
+            }
+        }
+    }
+
+    /// <summary>"address already in use" when a process holds the picked port, "port is already allocated" when another
+    /// container published it first.</summary>
+    internal static bool IsClash(Exception ex) =>
+        ex is DockerApiException { StatusCode: HttpStatusCode.InternalServerError } api
+     && (api.ResponseBody.Contains("address already in use", StringComparison.Ordinal)
+      || api.ResponseBody.Contains("port is already allocated", StringComparison.Ordinal));
+
+    // Docker publishes on IPv6 as well as IPv4, so a port held on either is taken. A host without IPv6 refuses that
+    // bind for its own reasons, which say nothing about the port.
+    static bool IsFree(int port) {
+        try {
+            using var v4 = new TcpListener(IPAddress.Any, port);
+            v4.Start();
+        } catch (SocketException) {
+            return false;
+        }
+
+        if (!Socket.OSSupportsIPv6) return true;
+
+        try {
+            using var v6 = new TcpListener(IPAddress.IPv6Any, port);
+            v6.Server.DualMode = false;
+            v6.Start();
+        } catch (SocketException ex) when (ex.SocketErrorCode != SocketError.AddressAlreadyInUse) {
+            return true;
+        } catch (SocketException) {
+            return false;
+        }
+
+        return true;
+    }
+}

test/Capacitor.Cli.Tests.Integration/ContainerHostPortsTests.cs

@@ -0,0 +1,89 @@
+using System.Globalization;
+using System.Net;
+using System.Net.Sockets;
+using Docker.DotNet;
+using DotNet.Testcontainers.Configurations;
+using DotNet.Testcontainers.Containers;
+
+namespace Capacitor.Cli.Tests.Integration;
+
+/// <summary>Every container a test starts is published below Linux's ephemeral range, from which Docker's own choice
+/// and a port-0 bind are drawn, on a port nothing here holds. The pick walks a process-wide cursor, so the test that
+/// needs to know which port comes next runs alone.</summary>
+public class ContainerHostPortsTests {
+    const int EphemeralFloor = 32768;
+
+    [Test]
+    [NotInParallel]
+    public async Task A_full_cycle_of_picks_stays_in_range_and_never_hands_out_a_held_port() {
+        var (held, holder) = HoldTheNextPort();
+        using var _ = holder;
+
+        var drawn = Enumerable.Range(0, ContainerHostPorts.Last - ContainerHostPorts.First + 1)
+            .Select(_ => (int)ContainerHostPorts.Next())
+            .ToList();
+
+        await Assert.That(drawn).DoesNotContain(held);
+        await Assert.That(drawn.All(p => p is >= ContainerHostPorts.First and < EphemeralFloor)).IsTrue();
+    }
+
+    /// <summary>Docker publishes on IPv6 as well as IPv4, so a port something holds on IPv6 alone is just as taken.</summary>
+    [Test]
+    [NotInParallel]
+    public async Task A_port_held_on_ipv6_alone_is_passed_over() {
+        Skip.When(!Socket.OSSupportsIPv6, "this host has no IPv6");
+
+        var (held, holder) = HoldTheNextPort(ipv6Only: true);
+        using var _ = holder;
+
+        await Assert.That((int)ContainerHostPorts.Next()).IsNotEqualTo(held);
+    }
+
+    /// <summary>Docker refuses a taken host port in two words: the kernel's, when another process holds it, and its own,
+    /// when another container already published it.</summary>
+    [Test]
+    [Arguments("failed to bind port 0.0.0.0:33201/tcp: Error starting userland proxy: listen tcp4 0.0.0.0:33201: bind: address already in use", true)]
+    [Arguments("Bind for 0.0.0.0:20417 failed: port is already allocated", true)]
+    [Arguments("could not kill container: abc is zombie and can not be killed", false)]
+    public async Task Only_a_bind_refusal_is_a_host_port_clash(string message, bool clash) =>
+        await Assert.That(ContainerHostPorts.IsClash(new DockerApiException(HttpStatusCode.InternalServerError,
+            $"{{\"message\":\"driver failed programming external connectivity on endpoint gifted_wozniak (8b56): {message}\"}}"))).IsEqualTo(clash);
+
+    /// <summary>The resource reaper starts with the first container of the process, so it is running by the time the
+    /// emulator is up.</summary>
+    [Test]
+    [Timeout(300_000)]
+    public async Task The_emulator_and_the_resource_reaper_publish_below_the_ephemeral_range(CancellationToken ct) {
+        Skip.When(Environment.GetEnvironmentVariable("GITHUB_ACTIONS") == "true" && !OperatingSystem.IsLinux(),
+            "this CI runner cannot run Linux containers");
+
+        var unavailable = await WorkOSEmulator.UnavailableReasonAsync();
+        Skip.When(unavailable is not null, unavailable ?? "");
+
+        await using var emulator = await WorkOSEmulator.StartAsync(ct);
+
+        using var docker = TestcontainersSettings.OS.DockerEndpointAuthConfig.GetDockerClientBuilder(Guid.Empty).Build();
+        var reaper = await docker.Containers.InspectContainerAsync($"testcontainers-ryuk-{ResourceReaper.DefaultSessionId:D}", ct);
+        var reaperPort = int.Parse(reaper.NetworkSettings.Ports["8080/tcp"][0].HostPort, CultureInfo.InvariantCulture);
+
+        await Assert.That(new Uri(emulator.BaseUrl).Port).IsLessThan(EphemeralFloor);
+        await Assert.That(reaperPort).IsLessThan(EphemeralFloor);
+    }
+
+    /// <summary>Holds the port one past the latest pick, which is the one the next pick lands on.</summary>
+    static (int Port, TcpListener Holder) HoldTheNextPort(bool ipv6Only = false) {
+        var port = ContainerHostPorts.Next() + 1;
+        if (port > ContainerHostPorts.Last) port = ContainerHostPorts.First;
+
+        var holder = new TcpListener(ipv6Only ? IPAddress.IPv6Any : IPAddress.Any, port);
+        if (ipv6Only) holder.Server.DualMode = false;
+
+        try {
+            holder.Start();
+        } catch (SocketException) {
+            // something else already holds it, which serves as well
+        }
+
+        return (port, holder);
+    }
+}

test/Capacitor.Cli.Tests.Integration/WorkOSEmulator.cs

@@ -1,8 +1,8 @@
 using System.Text;
-using Capacitor.Cli.Core.Auth;
 using DotNet.Testcontainers.Builders;
 using DotNet.Testcontainers.Configurations;
 using DotNet.Testcontainers.Containers;
+using DotNet.Testcontainers.Images;
 
 namespace Capacitor.Cli.Tests.Integration;
 
@@ -15,7 +15,6 @@ public sealed class WorkOSEmulator : IAsyncDisposable {
     public const string OwnerId        = "user_01K0000000000000000000000A";
     public const string OrganizationId = "org_01K0000000000000000000000K";
 
-    const string Image     = "ghcr.io/workos/emulate:0.15.0";
     const int    Port      = 4100;
     const string SeedPath  = "/app/workos-emulate.config.yaml";
 
@@ -68,31 +67,22 @@ public sealed class WorkOSEmulator : IAsyncDisposable {
     }
 
     public static async Task<WorkOSEmulator> StartAsync(CancellationToken ct) {
-        // A fixed host port, so a restart keeps the base a stored token names. Passing any argument
-        // replaces the image's defaults, and without --host 0.0.0.0 it listens on its own loopback.
-        var hostPort  = OAuthLoginFlow.GetAvailablePort();
-        var container = new ContainerBuilder(Image)
-            .WithPortBinding(hostPort, Port)
+        // The host port stays fixed across a restart, because a stored token names the base URL; something that takes
+        // the port while the emulator is stopped fails that one test. Passing any argument replaces the image's
+        // defaults, and without --host 0.0.0.0 it listens on its own loopback.
+        var container = await ContainerHostPorts.StartAsync(() => new ContainerBuilder(new DockerImage("ghcr.io/workos/emulate", tag: "0.15.0"))
+            .WithPortBinding(ContainerHostPorts.Next(), Port)
             .WithResourceMapping(Encoding.UTF8.GetBytes(Seed), SeedPath)
             .WithCommand("--host", "0.0.0.0", "--seed", SeedPath)
             .WithWaitStrategy(Wait.ForUnixContainer().UntilHttpRequestIsSucceeded(r => r.ForPort(Port).ForPath("/health")))
-            .Build();
+            .Build(), ct);
 
-        var emulator = new WorkOSEmulator(container, hostPort);
-
-        try {
-            await container.StartAsync(ct);
-        } catch {
-            await emulator.DisposeAsync();
-
-            throw;
-        }
-
-        return emulator;
+        return new WorkOSEmulator(container, container.GetMappedPublicPort(Port));
     }
 
-    /// <summary>Stops and starts the same container: the emulator keeps all state in memory, so every
-    /// token it issued is forgotten.</summary>
+    /// <summary>Stops and starts the same container on the same host port: the emulator keeps all state in memory, so
+    /// every token it issued is forgotten. The port is not retried, so one taken while it is stopped fails the
+    /// start.</summary>
     public async Task RestartAsync(CancellationToken ct) {
         await _container.StopAsync(ct);
         await _container.StartAsync(ct);

test/Capacitor.Cli.Tests.Unit/ContainerHostPortGuardTests.cs

@@ -0,0 +1,108 @@
+using System.Runtime.CompilerServices;
+using System.Text.RegularExpressions;
+
+namespace Capacitor.Cli.Tests.Unit;
+
+/// <summary>
+/// Docker's own host-port choice, and a port-0 bind, share Linux's ephemeral range with a loaded runner's sockets, so
+/// every published container port takes its host port from <c>ContainerHostPorts.Next()</c>. Pinned as a guard because
+/// a binding on Docker's choice passes everywhere a machine is quiet and fails only on a busy runner. The KurrentDB and
+/// Postgres builders publish their port on Docker's choice unless told otherwise, so each builder's own chain needs a
+/// picked binding in the statement that constructs it.
+/// </summary>
+public partial class ContainerHostPortGuardTests {
+    [GeneratedRegex(@"\.WithPortBinding\s*\((?!\s*ContainerHostPorts\.Next\s*\(\s*\)\s*,)")]
+    private static partial Regex DockerChosenBinding();
+
+    [GeneratedRegex(@"new\s+(?:KurrentDb|PostgreSql)Builder\s*\(|(?:KurrentDb|PostgreSql)Builder\s+\w+\s*=\s*new\s*\(")]
+    private static partial Regex PublishingBuilder();
+
+    [GeneratedRegex(@"\.WithPortBinding\s*\(\s*ContainerHostPorts\.Next\s*\(\s*\)\s*,")]
+    private static partial Regex PickedBinding();
+
+    [GeneratedRegex(@"\.Build\s*\(\s*\)")]
+    private static partial Regex Build();
+
+    [Test]
+    [Arguments(".WithPortBinding(4100, assignRandomHostPort: true)")]
+    [Arguments(".WithPortBinding(4100, true)")]
+    [Arguments(".WithPortBinding(hostPort, Port)")]
+    [Arguments(".WithPortBinding(OAuthLoginFlow.GetAvailablePort(), Port)")]
+    [Arguments("new KurrentDbBuilder(image).Build()")]
+    [Arguments("KurrentDbBuilder b = new(image); b.Build()")]
+    [Arguments("new PostgreSqlBuilder(image).Build(); new ContainerBuilder(i).WithPortBinding(ContainerHostPorts.Next(), 80).Build()")]
+    [Arguments("KurrentDbBuilder b = new(image); b = b.WithPortBinding(ContainerHostPorts.Next(), 2113); b.Build()")]
+    [Arguments("KurrentDbBuilder Unbuilt() => new KurrentDbBuilder(image); var c = new ContainerBuilder(i).WithPortBinding(ContainerHostPorts.Next(), 80).Build();")]
+    [Arguments("Pair(new KurrentDbBuilder(a), new PostgreSqlBuilder(b).WithPortBinding(ContainerHostPorts.Next(), 5432).Build())")]
+    [Arguments("new KurrentDbBuilder(image).WithEnvironment(\"NOTE\", \".WithPortBinding(ContainerHostPorts.Next(), 2113)\").Build()")]
+    [Arguments("var q = '\"'; new KurrentDbBuilder(image).Build(); var s = \"x\";")]
+    [Arguments("/* \" */ new KurrentDbBuilder(image).Build(); var s = \"x\";")]
+    public async Task Pattern_flags_a_port_docker_chooses(string code) =>
+        await Assert.That(PublishesOnDockersChoice(code)).IsTrue();
+
+    [Test]
+    [Arguments(".WithPortBinding(ContainerHostPorts.Next(), Port)")]
+    [Arguments("new KurrentDbBuilder(image).WithPortBinding(ContainerHostPorts.Next(), 2113).Build()")]
+    [Arguments("new KurrentDbBuilder(image).WithCreateParameterModifier(p => { p.Name = \"k\"; }).WithPortBinding(ContainerHostPorts.Next(), 2113).Build()")]
+    [Arguments("// .WithPortBinding(2113, true)\n.WithPortBinding(ContainerHostPorts.Next(), Port)")]
+    public async Task Pattern_passes_a_port_picked_below_the_ephemeral_range(string code) =>
+        await Assert.That(PublishesOnDockersChoice(code)).IsFalse();
+
+    [Test]
+    public async Task No_test_publishes_a_container_port_on_dockers_choice() {
+        var root = RepoRoot();
+
+        var offenders = Directory.EnumerateFiles(Path.Combine(root, "test"), "*.cs", SearchOption.AllDirectories)
+            .Where(path => !path.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Any(s => s is "bin" or "obj"))
+            .Where(path => PublishesOnDockersChoice(File.ReadAllText(path)))
+            .Select(path => Path.GetRelativePath(root, path).Replace('\\', '/'))
+            .ToList();
+
+        await Assert.That(offenders).IsEmpty()
+            .Because("bind each published port with .WithPortBinding(ContainerHostPorts.Next(), containerPort)");
+    }
+
+    static bool PublishesOnDockersChoice(string source) {
+        var code = SourceCode.WithoutCommentsOrStrings(source);
+
+        return DockerChosenBinding().IsMatch(code)
+            || PublishingBuilder().Matches(code).Any(builder => !PickedBinding().IsMatch(ChainOf(code, builder)));
+    }
+
+    /// <summary>The code from a builder's construction to whichever comes first: its <c>.Build()</c>, the end of its
+    /// statement, or the next builder's construction. A binding added in a later statement does not count.</summary>
+    static string ChainOf(string code, Match builder) {
+        var start = builder.Index + builder.Length;
+
+        int[] ends = [
+            Build().Match(code, start) is { Success: true } build ? build.Index : code.Length,
+            StatementEnd(code, start),
+            PublishingBuilder().Match(code, start) is { Success: true } next ? next.Index : code.Length
+        ];
+
+        return code[start..ends.Min()];
+    }
+
+    /// <summary>The first <c>;</c> outside every bracket opened after <paramref name="start"/>, which lies inside the builder's
+    /// own constructor call, so a statement inside a lambda in the chain does not end it.</summary>
+    static int StatementEnd(string code, int start) {
+        var depth = 1;
+
+        for (var i = start; i < code.Length; i++)
+            switch (code[i]) {
+                case '(' or '[' or '{': depth++; break;
+                case ')' or ']' or '}': depth--; break;
+                case ';' when depth <= 0: return i;
+            }
+
+        return code.Length;
+    }
+
+    static string RepoRoot([CallerFilePath] string here = "") {
+        var dir = Path.GetDirectoryName(here);
+        while (dir is not null && !File.Exists(Path.Combine(dir, "Capacitor.slnx")))
+            dir = Path.GetDirectoryName(dir);
+
+        return dir ?? throw new InvalidOperationException($"Could not locate repo root walking up from {here}");
+    }
+}