diff --git a/test/server/auto_head_options_engines_421_linux_test.go b/test/server/auto_head_options_engines_421_linux_test.go index b0ff7ea6..aa1ecbba 100644 --- a/test/server/auto_head_options_engines_421_linux_test.go +++ b/test/server/auto_head_options_engines_421_linux_test.go @@ -303,41 +303,12 @@ func startServer421(t *testing.T, eng celeris.EngineType, setup func(*celeris.Se // chosen here). func startServerConfig421(t *testing.T, cfg celeris.Config, setup func(*celeris.Server)) string { t.Helper() - retryUntil := time.Now().Add(30 * time.Second) - for tries := 1; ; tries++ { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - addr := ln.Addr().String() - _ = ln.Close() - cfg.Addr = addr - s := celeris.New(cfg) + // setup comes first (it may call Server.Use); the listener is handed to the + // server, so the port cannot be taken between the pick and the bind + // (celeris#1104, startPicked1104). + _, addr := startPickedPing1104(t, cfg, func(s *celeris.Server) { setup(s) - s.GET("/ping", func(c *celeris.Context) error { return c.String(http.StatusOK, "ok") }) - startDone := make(chan error, 1) - go func() { startDone <- s.Start() }() - err = waitReady761(addr, startDone) - if err == nil { - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _ = s.Shutdown(ctx) - select { - case <-startDone: - case <-time.After(15 * time.Second): - t.Errorf("Start did not return within 15s of Shutdown") - } - }) - return addr - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = s.Shutdown(ctx) - cancel() - if isRingENOMEM(err) && time.Now().Before(retryUntil) { - time.Sleep(20 * time.Millisecond) - continue - } - t.Fatalf("server did not start (try %d): %v", tries, err) - } + ping1104(s) + }) + return addr } diff --git a/test/server/bodylimit_default_889_linux_test.go b/test/server/bodylimit_default_889_linux_test.go index 5661dba1..8a781b24 100644 --- a/test/server/bodylimit_default_889_linux_test.go +++ b/test/server/bodylimit_default_889_linux_test.go @@ -69,42 +69,13 @@ var bodyCfgs889 = []bodyCfg889{ // the number of body bytes the handler saw. func bodyServer889(t *testing.T, eng celeris.EngineType, limit int64) string { t.Helper() - retryUntil := time.Now().Add(30 * time.Second) - for tries := 1; ; tries++ { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - addr := ln.Addr().String() - _ = ln.Close() - s := celeris.New(celeris.Config{Addr: addr, Engine: eng, Protocol: celeris.Auto, MaxRequestBodySize: limit}) - s.GET("/ping", func(c *celeris.Context) error { return c.String(http.StatusOK, "ok") }) + // The listener is handed to the server, so the port cannot be taken between + // the pick and the bind (celeris#1104, startPicked1104). + _, addr := startPickedPing1104(t, celeris.Config{Engine: eng, Protocol: celeris.Auto, MaxRequestBodySize: limit}, func(s *celeris.Server) { + ping1104(s) s.POST("/up", func(c *celeris.Context) error { return c.String(http.StatusOK, "n=%d", len(c.Body())) }) - startDone := make(chan error, 1) - go func() { startDone <- s.Start() }() - err = waitReady761(addr, startDone) - if err == nil { - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _ = s.Shutdown(ctx) - select { - case <-startDone: - case <-time.After(15 * time.Second): - t.Errorf("Start did not return within 15s of Shutdown") - } - }) - return addr - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = s.Shutdown(ctx) - cancel() - if isRingENOMEM(err) && time.Now().Before(retryUntil) { - time.Sleep(20 * time.Millisecond) - continue - } - t.Fatalf("server did not start (try %d): %v", tries, err) - } + }) + return addr } const bodyUpgradeHead889 = "POST /up HTTP/1.1\r\nHost: a\r\nConnection: Upgrade, HTTP2-Settings\r\nUpgrade: h2c\r\n" + diff --git a/test/server/chunked_promotion_889_linux_test.go b/test/server/chunked_promotion_889_linux_test.go index da142a67..07337655 100644 --- a/test/server/chunked_promotion_889_linux_test.go +++ b/test/server/chunked_promotion_889_linux_test.go @@ -33,49 +33,19 @@ var enginesNative889 = []struct { // the body length, a sync GET /ping and an Async() GET /slow, on one engine. func start889(t *testing.T, cfg celeris.Config, extra ...func(*celeris.Server)) (string, *celeris.Server) { t.Helper() - retryUntil := time.Now().Add(30 * time.Second) - for tries := 1; ; tries++ { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - addr := ln.Addr().String() - _ = ln.Close() - cfg.Addr = addr - s := celeris.New(cfg) + // The listener is handed to the server, so the port cannot be taken between + // the pick and the bind (celeris#1104, startPicked1104). + s, addr := startPickedPing1104(t, cfg, func(s *celeris.Server) { up := func(c *celeris.Context) error { return c.String(http.StatusOK, "n=%d", len(c.Body())) } s.POST("/up", up) s.POST("/aup", up).Async() - s.GET("/ping", func(c *celeris.Context) error { return c.String(http.StatusOK, "ok") }) + ping1104(s) s.GET("/slow", func(c *celeris.Context) error { return c.String(http.StatusOK, "slow") }).Async() for _, f := range extra { f(s) } - startDone := make(chan error, 1) - go func() { startDone <- s.Start() }() - err = waitReady761(addr, startDone) - if err == nil { - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _ = s.Shutdown(ctx) - select { - case <-startDone: - case <-time.After(15 * time.Second): - t.Errorf("Start did not return within 15s of Shutdown") - } - }) - return addr, s - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = s.Shutdown(ctx) - cancel() - if strings.Contains(err.Error(), "cannot allocate memory") && time.Now().Before(retryUntil) { - time.Sleep(20 * time.Millisecond) - continue - } - t.Fatalf("server did not start (try %d): %v", tries, err) - } + }) + return addr, s } func get889(t *testing.T, c net.Conn, br *bufio.Reader, path string) string { diff --git a/test/server/detect_split_first_segment_870_linux_test.go b/test/server/detect_split_first_segment_870_linux_test.go index 2b1f9ca3..613c8878 100644 --- a/test/server/detect_split_first_segment_870_linux_test.go +++ b/test/server/detect_split_first_segment_870_linux_test.go @@ -4,7 +4,6 @@ package celeris_test import ( "bufio" - "context" "encoding/binary" "fmt" "io" @@ -74,16 +73,9 @@ func startServer870(t *testing.T, cfg celeris.Config) string { // that reads its metrics. func startServerSrv870(t *testing.T, cfg celeris.Config) (string, *celeris.Server) { t.Helper() - retryUntil := time.Now().Add(30 * time.Second) - for tries := 1; ; tries++ { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - addr := ln.Addr().String() - _ = ln.Close() - cfg.Addr = addr - s := celeris.New(cfg) + // The listener is handed to the server, so the port cannot be taken between + // the pick and the bind (celeris#1104, startPicked1104). + s, addr := startPickedPing1104(t, cfg, func(s *celeris.Server) { // With AsyncHandlers on, a route that only answers runs inline and its conn // is never promoted to a dispatch goroutine (AsyncPromotedConns stays 0), so // the route is marked async: the conn is promoted at its first request and @@ -92,35 +84,12 @@ func startServerSrv870(t *testing.T, cfg celeris.Config) (string, *celeris.Serve if cfg.AsyncHandlers { h.Async() } - s.GET("/ping", func(c *celeris.Context) error { return c.String(http.StatusOK, "ok") }) + ping1104(s) s.POST("/len", func(c *celeris.Context) error { return c.String(http.StatusOK, "%d", len(c.Body())) }) - startDone := make(chan error, 1) - go func() { startDone <- s.Start() }() - err = waitReady761(addr, startDone) - if err == nil { - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _ = s.Shutdown(ctx) - select { - case <-startDone: - case <-time.After(15 * time.Second): - t.Errorf("Start did not return within 15s of Shutdown") - } - }) - return addr, s - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = s.Shutdown(ctx) - cancel() - if isRingENOMEM(err) && time.Now().Before(retryUntil) { - time.Sleep(20 * time.Millisecond) - continue - } - t.Fatalf("server did not start (try %d): %v", tries, err) - } + }) + return addr, s } // sendPieces870 dials addr and writes each piece as its own segment, pieceGap870 diff --git a/test/server/h2_stream_context_836_linux_test.go b/test/server/h2_stream_context_836_linux_test.go index 36ea6e6d..87028411 100644 --- a/test/server/h2_stream_context_836_linux_test.go +++ b/test/server/h2_stream_context_836_linux_test.go @@ -477,40 +477,11 @@ func TestH2CUnmatchedThroughTimeoutSingleflight836(t *testing.T) { // readiness probe /ping is registered after it. func startGlobalChainServer836(t *testing.T, cfg celeris.Config, setup func(*celeris.Server)) string { t.Helper() - retryUntil := time.Now().Add(30 * time.Second) - for { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - addr := ln.Addr().String() - _ = ln.Close() - cfg.Addr = addr - s := celeris.New(cfg) + // The listener is handed to the server, so the port cannot be taken between + // the pick and the bind (celeris#1104, startPicked1104). + _, addr := startPickedPing1104(t, cfg, func(s *celeris.Server) { setup(s) - s.GET("/ping", func(c *celeris.Context) error { return c.String(http.StatusOK, "ok") }) - startDone := make(chan error, 1) - go func() { startDone <- s.Start() }() - if err = waitReady761(addr, startDone); err == nil { - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _ = s.Shutdown(ctx) - select { - case <-startDone: - case <-time.After(15 * time.Second): - t.Errorf("Start did not return within 15s of Shutdown") - } - }) - return addr - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = s.Shutdown(ctx) - cancel() - if isRingENOMEM(err) && time.Now().Before(retryUntil) { - time.Sleep(20 * time.Millisecond) - continue - } - t.Fatalf("server did not start: %v", err) - } + ping1104(s) + }) + return addr } diff --git a/test/server/h2upgrade_disabled_964_linux_test.go b/test/server/h2upgrade_disabled_964_linux_test.go index ed45a839..7183539e 100644 --- a/test/server/h2upgrade_disabled_964_linux_test.go +++ b/test/server/h2upgrade_disabled_964_linux_test.go @@ -4,7 +4,6 @@ package celeris_test import ( "bufio" - "context" "fmt" "io" "net" @@ -36,43 +35,13 @@ func bptr964(v bool) *bool { return &v } // whatever cfg says (the zero value is Auto). func startServer964(t *testing.T, cfg celeris.Config) string { t.Helper() - retryUntil := time.Now().Add(30 * time.Second) - for tries := 1; ; tries++ { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - addr := ln.Addr().String() - _ = ln.Close() - cfg.Addr = addr - s := celeris.New(cfg) + // The listener is handed to the server, so the port cannot be taken between + // the pick and the bind (celeris#1104, startPicked1104). + _, addr := startPickedPing1104(t, cfg, func(s *celeris.Server) { s.GET("/h", func(c *celeris.Context) error { return c.String(http.StatusOK, "h1-answer") }) - s.GET("/ping", func(c *celeris.Context) error { return c.String(http.StatusOK, "ok") }) - startDone := make(chan error, 1) - go func() { startDone <- s.Start() }() - err = waitReady761(addr, startDone) - if err == nil { - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _ = s.Shutdown(ctx) - select { - case <-startDone: - case <-time.After(15 * time.Second): - t.Errorf("Start did not return within 15s of Shutdown") - } - }) - return addr - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = s.Shutdown(ctx) - cancel() - if isRingENOMEM(err) && time.Now().Before(retryUntil) { - time.Sleep(20 * time.Millisecond) - continue - } - t.Fatalf("server did not start (try %d): %v", tries, err) - } + ping1104(s) + }) + return addr } // TestEnableH2UpgradeFalseDisablesUpgradeOnAuto964 drives celeris#964 on every diff --git a/test/server/iouring_async_cancel_refusal_886_linux_test.go b/test/server/iouring_async_cancel_refusal_886_linux_test.go index 82ab8e6a..ee2a0cbe 100644 --- a/test/server/iouring_async_cancel_refusal_886_linux_test.go +++ b/test/server/iouring_async_cancel_refusal_886_linux_test.go @@ -120,8 +120,11 @@ func TestServerAdaptiveStartsOnEpollWhenIOUringRefused886(t *testing.T) { if err != nil { t.Fatalf("listen: %v", err) } + // The listener is handed to the server, not closed and bound again: adaptive + // gives it to the epoll fallback when the io_uring start engine is refused + // before Listen, and the port is not free between the pick and the bind + // until that engine rebinds it (celeris#1104). addr := ln.Addr().String() - _ = ln.Close() var logs lockedBuf886 s := celeris.New(celeris.Config{ @@ -133,7 +136,7 @@ func TestServerAdaptiveStartsOnEpollWhenIOUringRefused886(t *testing.T) { s.GET("/ping", func(c *celeris.Context) error { return c.String(http.StatusOK, "ok") }) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { done <- s.StartWithContext(ctx) }() + go func() { done <- s.StartWithListenerAndContext(ctx, ln) }() defer func() { cancel() sctx, scancel := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/test/server/large_response_linux_test.go b/test/server/large_response_linux_test.go index 1302c610..d1eef245 100644 --- a/test/server/large_response_linux_test.go +++ b/test/server/large_response_linux_test.go @@ -534,46 +534,13 @@ func startServer761(t *testing.T, eng celeris.EngineType, asyncServer bool, rout // chosen here). func startServerConfig761(t testing.TB, cfg celeris.Config, routes func(*celeris.Server)) string { t.Helper() - retryUntil := time.Now().Add(30 * time.Second) - for tries := 1; ; tries++ { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - addr := ln.Addr().String() - _ = ln.Close() - cfg.Addr = addr - s := celeris.New(cfg) - s.GET("/ping", func(c *celeris.Context) error { return c.String(http.StatusOK, "ok") }) + // The listener is handed to the server, so the port cannot be taken between + // the pick and the bind (celeris#1104, startPicked1104). + _, addr := startPickedPing1104(t, cfg, func(s *celeris.Server) { + ping1104(s) routes(s) - startDone := make(chan error, 1) - go func() { startDone <- s.Start() }() - err = waitReady761(addr, startDone) - if err == nil { - if tries > 1 { - t.Logf("server start retried on ring ENOMEM: %d tries", tries) - } - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _ = s.Shutdown(ctx) - select { - case <-startDone: - case <-time.After(15 * time.Second): - t.Errorf("Start did not return within 15s of Shutdown") - } - }) - return addr - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = s.Shutdown(ctx) - cancel() - if isRingENOMEM(err) && time.Now().Before(retryUntil) { - time.Sleep(20 * time.Millisecond) - continue - } - t.Fatalf("server did not start: %v", err) - } + }) + return addr } // waitReady761 polls /ping until it answers 200, or returns the error the diff --git a/test/server/shutdown_h2_pool_linux_test.go b/test/server/shutdown_h2_pool_linux_test.go index 8410939b..9ccfc1c0 100644 --- a/test/server/shutdown_h2_pool_linux_test.go +++ b/test/server/shutdown_h2_pool_linux_test.go @@ -202,45 +202,29 @@ type h2PoolServer759 struct { shutErr chan error } +// startH2PoolServer759 starts a server on a listener it picked, under a context +// the server's cancel stops (StartWithListenerAndContext, which shares +// StartWithContext's shutdown), and waits until it answers /ping; see +// startServer760 for the retry and startPickedCtx1104 for the listener. func startH2PoolServer759(t *testing.T, eng celeris.EngineType, budget time.Duration, routes func(*celeris.Server)) *h2PoolServer759 { t.Helper() - retryUntil := time.Now().Add(30 * time.Second) - for tries := 1; ; tries++ { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - addr := ln.Addr().String() - _ = ln.Close() - s := celeris.New(celeris.Config{Engine: eng, Addr: addr, ShutdownTimeout: budget}) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) // a start that never succeeds leaves nothing running + s, addr, done := startPickedCtx1104(ctx, t, celeris.Config{Engine: eng, ShutdownTimeout: budget}, func(s *celeris.Server) { s.GET("/ping", func(c *celeris.Context) error { return c.String(http.StatusOK, "ok") }) routes(s) - ctx, cancel := context.WithCancel(context.Background()) - srv := &h2PoolServer759{s: s, addr: addr, cancel: cancel, startDone: make(chan error, 1)} - go func() { srv.startDone <- s.StartWithContext(ctx) }() - err = waitDrainOrderReady(addr, srv.startDone) - if err == nil { - if tries > 1 { - t.Logf("server start retried on ring ENOMEM: %d tries", tries) - } - t.Cleanup(func() { - cancel() - select { - case err := <-srv.startDone: - srv.startDone <- err - case <-time.After(30 * time.Second): - t.Errorf("StartWithContext did not return within 30s of the cleanup cancel") - } - }) - return srv - } + }, func(_ *celeris.Server, addr string, done <-chan error) error { return waitDrainOrderReady(addr, done) }) + srv := &h2PoolServer759{s: s, addr: addr, cancel: cancel, startDone: done} + t.Cleanup(func() { cancel() - if isRingENOMEM(err) && time.Now().Before(retryUntil) { - time.Sleep(20 * time.Millisecond) - continue + select { + case err := <-srv.startDone: + srv.startDone <- err + case <-time.After(30 * time.Second): + t.Errorf("StartWithContext did not return within 30s of the cleanup cancel") } - t.Fatalf("server did not start: %v", err) - } + }) + return srv } // beginShutdown starts the shutdown: a cancel of StartWithContext's context diff --git a/test/server/shutdown_send_drain_linux_test.go b/test/server/shutdown_send_drain_linux_test.go index 3557b744..a40ed9c3 100644 --- a/test/server/shutdown_send_drain_linux_test.go +++ b/test/server/shutdown_send_drain_linux_test.go @@ -518,9 +518,12 @@ type server760 struct { startDone chan error } -// startServer760 starts a server with routes on StartWithContext, with -// ShutdownTimeout budget (and WriteTimeout writeTimeout, 0 for the default), -// and waits until it answers /ping. An io_uring start +// startServer760 starts a server with routes on a listener it picked, under a +// context it cancels to stop it (StartWithListenerAndContext, which shares +// StartWithContext's shutdown), with ShutdownTimeout budget (and WriteTimeout +// writeTimeout, 0 for the default), and waits until it answers /ping. The +// listener is handed to the server, not closed and bound again (see +// startPickedCtx1104). An io_uring start // that fails only with ENOMEM is retried, with a new server, for up to 30 s: // the kernel charges ring memory to RLIMIT_MEMLOCK per UID and gives it back // some milliseconds after a ring closes, so at the CI runner's 8 MiB a start @@ -528,46 +531,26 @@ type server760 struct { // leaked (see startC714DetachServer). func startServer760(t *testing.T, eng celeris.EngineType, budget, writeTimeout time.Duration, cfgFn func(*celeris.Config), routes func(*celeris.Server)) *server760 { t.Helper() - retryUntil := time.Now().Add(30 * time.Second) - for tries := 1; ; tries++ { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - addr := ln.Addr().String() - _ = ln.Close() - cfg := celeris.Config{Engine: eng, Addr: addr, ShutdownTimeout: budget, WriteTimeout: writeTimeout} - if cfgFn != nil { - cfgFn(&cfg) - } - s := celeris.New(cfg) + cfg := celeris.Config{Engine: eng, ShutdownTimeout: budget, WriteTimeout: writeTimeout} + if cfgFn != nil { + cfgFn(&cfg) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) // a start that never succeeds leaves nothing running + s, addr, done := startPickedCtx1104(ctx, t, cfg, func(s *celeris.Server) { s.GET("/ping", func(c *celeris.Context) error { return c.String(http.StatusOK, "ok") }) routes(s) - ctx, cancel := context.WithCancel(context.Background()) - srv := &server760{s: s, addr: addr, cancel: cancel, startDone: make(chan error, 1)} - go func() { srv.startDone <- s.StartWithContext(ctx) }() - err = waitReady760(addr, srv.startDone) - if err == nil { - if tries > 1 { - t.Logf("server start retried on ring ENOMEM: %d tries", tries) - } - t.Cleanup(func() { - cancel() - select { - case <-srv.startDone: - case <-time.After(30 * time.Second): - t.Errorf("StartWithContext did not return within 30s of the cleanup cancel") - } - }) - return srv - } + }, func(_ *celeris.Server, addr string, done <-chan error) error { return waitReady760(addr, done) }) + srv := &server760{s: s, addr: addr, cancel: cancel, startDone: done} + t.Cleanup(func() { cancel() - if isRingENOMEM(err) && time.Now().Before(retryUntil) { - time.Sleep(20 * time.Millisecond) - continue + select { + case <-srv.startDone: + case <-time.After(30 * time.Second): + t.Errorf("StartWithContext did not return within 30s of the cleanup cancel") } - t.Fatalf("server did not start: %v", err) - } + }) + return srv } // waitReady760 polls /ping until it answers 200, or returns the error the diff --git a/test/server/start_picked_family_linux_test.go b/test/server/start_picked_family_linux_test.go new file mode 100644 index 00000000..a9035c1c --- /dev/null +++ b/test/server/start_picked_family_linux_test.go @@ -0,0 +1,106 @@ +//go:build linux + +package celeris_test + +import ( + "fmt" + "net" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/goceleris/celeris" +) + +// celeris#1104: every starter of test/server that picked a port, closed it and +// had Start bind it could be given a port another test of the process picked at +// the same time (package F, startPicked1104, moved three starters off it). This +// forces two concurrent calls of each remaining starter onto the same picked +// port (listenForStart1104: the first picker gets it, a picker that finds it +// held gets a fresh one) and requires them to end on different ports. A starter +// that frees the port between the pick and the bind gives both of them the same +// one, and one of the two servers is not the one that answers there. +// +// It runs on the std engine, the only one that serves on the listener it is +// handed. The epoll, io_uring and adaptive engines close it inside Listen and +// rebind the address with SO_REUSEPORT, so a hand-off only shortens their window +// and two servers can still end on one port: the same test with the engine +// swapped ended two servers on one port with epoll in 2 of 5 runs of up to 630 +// subtests (2 failures in the run counted), with io_uring in 2 of 3 runs of 630 +// (1 failure each), and the review of celeris#1139 saw 3 of 105 and 2 of 210. No +// retry sees it, as both binds succeed. The starters are tested here, on std, +// where the guarantee holds. +// +// The two starts of a pair run as parallel subtests, not as bare goroutines, so a +// start that fails (the starters call t.Fatal) ends its own subtest. A failed +// start leaves its address empty and is reported by its subtest alone. +func TestStartersForcedOntoOnePortEndOnPortsOfTheirOwn1104(t *testing.T) { + std := celeris.Config{Engine: celeris.Std} + none := func(*celeris.Server) {} + starters := []struct { + name string + start func(t *testing.T) string + }{ + {"startServerConfig761", func(t *testing.T) string { return startServerConfig761(t, std, none) }}, + {"startServerConfig421", func(t *testing.T) string { return startServerConfig421(t, std, none) }}, + {"bodyServer889", func(t *testing.T) string { return bodyServer889(t, celeris.Std, 0) }}, + {"start889", func(t *testing.T) string { a, _ := start889(t, std); return a }}, + {"startServer870", func(t *testing.T) string { return startServer870(t, std) }}, + {"startGlobalChainServer836", func(t *testing.T) string { return startGlobalChainServer836(t, std, none) }}, + {"startServer964", func(t *testing.T) string { return startServer964(t, std) }}, + {"startServer760", func(t *testing.T) string { return startServer760(t, celeris.Std, 5*time.Second, 0, nil, none).addr }}, + {"startH2PoolServer759", func(t *testing.T) string { return startH2PoolServer759(t, celeris.Std, 5*time.Second, none).addr }}, + } + for _, st := range starters { + for round := 0; round < 3; round++ { + t.Run(fmt.Sprintf("%s/round%d", st.name, round), func(t *testing.T) { + pick, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := pick.Addr().String() + _ = pick.Close() + defer func(old func() (net.Listener, error)) { listenForStart1104 = old }(listenForStart1104) + listenForStart1104 = func() (net.Listener, error) { + if ln, err := net.Listen("tcp", port); err == nil { + return ln, nil + } + return net.Listen("tcp", "127.0.0.1:0") + } + addrs := make([]string, 2) + // A starter's server stops when its subtest ends, so each waits for + // the other to be up first (the barrier), or the second could be given + // the port of a first that is gone. + var up sync.WaitGroup + up.Add(len(addrs)) + bothUp := make(chan struct{}) + go func() { up.Wait(); close(bothUp) }() + var overlapped atomic.Int32 + // The inner Run returns once both parallel subtests are done. + t.Run("pair", func(t *testing.T) { + for i := range addrs { + t.Run(fmt.Sprintf("starter%d", i), func(t *testing.T) { + t.Parallel() + defer func() { + up.Done() + select { + case <-bothUp: + overlapped.Add(1) + case <-time.After(30 * time.Second): + } + }() + addrs[i] = st.start(t) + }) + } + }) + if overlapped.Load() != int32(len(addrs)) { + t.Errorf("the two starters were not up at the same time (is -parallel 1?)") + } + if addrs[0] != "" && addrs[0] == addrs[1] { + t.Errorf("both starters ended on %s: one of the two servers failed to bind it and the other's /ping answered for it", addrs[0]) + } + }) + } + } +} diff --git a/test/server/start_picked_linux_test.go b/test/server/start_picked_linux_test.go index ce13e6ae..6079fd72 100644 --- a/test/server/start_picked_linux_test.go +++ b/test/server/start_picked_linux_test.go @@ -6,9 +6,9 @@ import ( "context" "fmt" "net" - "time" - + "net/http" "testing" + "time" "github.com/goceleris/celeris" ) @@ -22,13 +22,34 @@ var listenForStart1104 = func() (net.Listener, error) { return net.Listen("tcp", // startPicked1104 builds a server with cfg (Addr is set here) and routes, starts // it on a listener it picked, and returns it with its address and Start's result // channel once ready reports it up. The listener is handed to the server -// (StartWithListener), so the port is never free between the pick and the bind: -// another cell picking the same one gets a different port, and a readiness probe -// can only be answered by this server. A ring ENOMEM is retried for the window -// (startRetrying1104; a fresh server and port per try) and a start that fails -// otherwise fails t. ready is given the server, its address and Start's result +// (StartWithListener). On the std engine, which serves on that listener, the +// port is never free between the pick and the bind: another cell picking the +// same one gets a different port, and a readiness probe can only be answered by +// this server. The epoll, io_uring and adaptive engines close the supplied +// listener inside Listen and rebind its address with SO_REUSEPORT, so there the +// hand-off only shortens the window to the close and the rebind, and two native +// servers that both win the port both bind it, without an error, and share it +// (see TestStartersForcedOntoOnePortEndOnPortsOfTheirOwn1104). A ring ENOMEM is +// retried for the window (startRetrying1104; a fresh server and port per try) +// and a start that fails otherwise fails t. ready is given the server, its address and Start's result // channel and must return Start's error if it comes first. func startPicked1104(t testing.TB, cfg celeris.Config, routes func(*celeris.Server), ready func(s *celeris.Server, addr string, done <-chan error) error) (*celeris.Server, string, chan error) { + t.Helper() + return startPickedWith1104(t, cfg, routes, (*celeris.Server).StartWithListener, ready) +} + +// startPickedCtx1104 is startPicked1104 for the starters that stop their server +// by cancelling the context Start runs under: it calls StartWithListenerAndContext, +// which shares StartWithContext's shutdown (listenUntilCancelled). ctx is not +// cancelled here; a try that fails is stopped with Shutdown. +func startPickedCtx1104(ctx context.Context, t testing.TB, cfg celeris.Config, routes func(*celeris.Server), ready func(s *celeris.Server, addr string, done <-chan error) error) (*celeris.Server, string, chan error) { + t.Helper() + return startPickedWith1104(t, cfg, routes, func(s *celeris.Server, ln net.Listener) error { return s.StartWithListenerAndContext(ctx, ln) }, ready) +} + +// startPickedWith1104 is the body of startPicked1104 and startPickedCtx1104: +// start is how the server is started on the listener. +func startPickedWith1104(t testing.TB, cfg celeris.Config, routes func(*celeris.Server), start func(*celeris.Server, net.Listener) error, ready func(s *celeris.Server, addr string, done <-chan error) error) (*celeris.Server, string, chan error) { t.Helper() var ( s *celeris.Server @@ -45,7 +66,7 @@ func startPicked1104(t testing.TB, cfg celeris.Config, routes func(*celeris.Serv s = celeris.New(cfg) routes(s) done = make(chan error, 1) - go func(s *celeris.Server, ln net.Listener, done chan error) { done <- s.StartWithListener(ln) }(s, ln, done) + go func(s *celeris.Server, ln net.Listener, done chan error) { done <- start(s, ln) }(s, ln, done) if err := ready(s, addr, done); err != nil { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) _ = s.Shutdown(ctx) @@ -57,3 +78,28 @@ func startPicked1104(t testing.TB, cfg celeris.Config, routes func(*celeris.Serv }) return s, addr, done } + +// startPickedPing1104 is startPicked1104 for the starters of this package that +// register their routes (and a /ping, in whatever order they need) and wait for +// /ping to answer: it also registers the cleanup they all had, a Shutdown of the +// server and a wait for Start to return. +func startPickedPing1104(t testing.TB, cfg celeris.Config, routes func(*celeris.Server)) (*celeris.Server, string) { + t.Helper() + s, addr, done := startPicked1104(t, cfg, routes, func(_ *celeris.Server, addr string, done <-chan error) error { return waitReady761(addr, done) }) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = s.Shutdown(ctx) + select { + case <-done: + case <-time.After(15 * time.Second): + t.Errorf("Start did not return within 15s of Shutdown") + } + }) + return s, addr +} + +// ping1104 registers the /ping route startPickedPing1104's readiness probe uses. +func ping1104(s *celeris.Server) { + s.GET("/ping", func(c *celeris.Context) error { return c.String(http.StatusOK, "ok") }) +} diff --git a/test/server/std_cancel_shutdown_timeout_test.go b/test/server/std_cancel_shutdown_timeout_test.go index 17545e42..6f1f58fb 100644 --- a/test/server/std_cancel_shutdown_timeout_test.go +++ b/test/server/std_cancel_shutdown_timeout_test.go @@ -42,8 +42,10 @@ func runStdCancelBudgetCase(t *testing.T, watchCtx bool) { if err != nil { t.Fatal(err) } + // The server is handed this listener (std serves on it), so the port is not + // free between the pick and the bind: no other test of the process can be + // given it, and nothing but this server can answer the /ping probe below. addr := ln.Addr().String() - _ = ln.Close() entered := make(chan struct{}) release := make(chan struct{}) @@ -82,7 +84,7 @@ func runStdCancelBudgetCase(t *testing.T, watchCtx bool) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() startDone := make(chan error, 1) - go func() { startDone <- s.StartWithContext(ctx) }() + go func() { startDone <- s.StartWithListenerAndContext(ctx, ln) }() probe := &http.Client{Timeout: 300 * time.Millisecond} for deadline := time.Now().Add(15 * time.Second); ; { if resp, err := probe.Get("http://" + addr + "/ping"); err == nil {