From 92f60f91cd7eb1c9b5fc9c0f8c7ed948b106f055 Mon Sep 17 00:00:00 2001 From: Ziad Hatahet Date: Sun, 15 Nov 2026 21:51:51 -0500 Subject: [PATCH] [vertx] One Vertx and SO_REUSEPORT listener per core A Vertx folds every server on a host:port into one listening socket, accepted on its single acceptor thread, so that thread caps new connections per second. On the 64-core host limited-conn sits at 630K req/s with 28 CPUs busy. Run one Vertx per core, each with a single event loop and one server verticle, and set setReusePort(true) on the 8080, 8081 and 8082 servers, so each core binds its own socket and the kernel spreads new connections across them. Main now exits if a deployment fails. --- frameworks/vertx/README.md | 11 +++--- frameworks/vertx/meta.json | 2 +- .../vertx/src/main/java/httparena/Main.java | 34 +++++++++++++------ .../main/java/httparena/ServerVerticle.java | 10 ++++-- 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/frameworks/vertx/README.md b/frameworks/vertx/README.md index cbbaf105de..c49a36fc69 100644 --- a/frameworks/vertx/README.md +++ b/frameworks/vertx/README.md @@ -1,6 +1,6 @@ # vertx -Eclipse Vert.x with vertx-web on Netty, one server verticle per core. +Eclipse Vert.x with vertx-web on Netty, one Vert.x instance and `SO_REUSEPORT` listener per core. ## Stack @@ -31,13 +31,14 @@ Eclipse Vert.x with vertx-web on Netty, one server verticle per core. - Compression through `HttpServerOptions.setCompressionSupported(true)` - TLS through OpenSSL (BoringSSL from `netty-tcnative-boringssl-static`) via `OpenSSLEngineOptions`, rather than the JDK's `SSLEngine` that Vert.x uses when no engine is set -- The verticle is deployed once per core, all instances sharing port 8080 +- One `Vertx` per core, each with a single event loop and one server verticle. Every server sets + `setReusePort(true)`, so each core binds its own `SO_REUSEPORT` socket on each port and the + kernel spreads new connections across them. Inside one `Vertx`, servers on the same host:port + share a single listening socket and a single acceptor thread - `/echo` is read from the request stream, so the 20 MB body is never buffered `json-tls` and `static-tls` listen on `8081` when `/certs/server.crt` and `/certs/server.key` are -mounted. The verticle is deployed once per core and each instance binds both ports, so the TLS -listener is spread across every event loop rather than parked on one. ALPN is off: those two -profiles want HTTP/1.1 negotiated and no h2 offered. +mounted. ALPN is off: those two profiles want HTTP/1.1 negotiated and no h2 offered. Static file bodies are read from disk on every request - `sendFile` hands the descriptor to the kernel rather than holding the bytes - and the pre-compressed sibling is picked per request. diff --git a/frameworks/vertx/meta.json b/frameworks/vertx/meta.json index 35c62f4adc..9408630e4e 100644 --- a/frameworks/vertx/meta.json +++ b/frameworks/vertx/meta.json @@ -10,7 +10,7 @@ "response": true }, "engine": "Netty", - "description": "Eclipse Vert.x 5 with vertx-web on Netty's epoll transport, one server verticle per core. Routing and query/path params through the Router API, JsonObject serialization, gzip through the server compression option, TLS through BoringSSL (netty-tcnative).", + "description": "Eclipse Vert.x 5 with vertx-web on Netty's epoll transport, one Vert.x instance with its own SO_REUSEPORT listener per core. Routing and query/path params through the Router API, JsonObject serialization, gzip through the server compression option, TLS through BoringSSL (netty-tcnative).", "repo": "https://github.com/eclipse-vertx/vert.x", "enabled": true, "tests": [ diff --git a/frameworks/vertx/src/main/java/httparena/Main.java b/frameworks/vertx/src/main/java/httparena/Main.java index e2fc77f043..6ca94bb9f8 100644 --- a/frameworks/vertx/src/main/java/httparena/Main.java +++ b/frameworks/vertx/src/main/java/httparena/Main.java @@ -5,7 +5,6 @@ import java.nio.file.Files; import java.nio.file.Path; -import io.vertx.core.DeploymentOptions; import io.vertx.core.Vertx; import io.vertx.core.VertxOptions; import io.vertx.core.buffer.Buffer; @@ -26,21 +25,36 @@ public static void main(String[] args) throws Exception { } catch (Exception ignored) { } - // Netty's epoll transport rather than the JDK's NIO selector. Exit if it fails to - // load, so a run can never quietly fall back to NIO. - Vertx vertx = Vertx.vertx(new VertxOptions().setPreferNativeTransport(true)); - if (!vertx.isNativeTransportEnabled()) { - System.err.println("epoll native transport unavailable: " + vertx.unavailableNativeTransportCause()); - System.exit(1); - } // The TLS port runs on OpenSSL, and without it every 8081 listen fails. Exit with the // reason up front, the same as for epoll, rather than leave it to a failed deployment. if (!OpenSSLEngineOptions.isAvailable()) { System.err.println("OpenSSL unavailable: " + OpenSsl.unavailabilityCause()); System.exit(1); } + + // One Vertx per core, each with a single event loop. A Vertx folds every server on the + // same host:port into one listening socket, accepted on its one acceptor thread whatever + // setReusePort says, so that thread caps the connections it can take per second. With a + // Vertx per core, each binds its own SO_REUSEPORT socket (set in ServerVerticle) and the + // kernel spreads new connections across them. Item[] items = dataset; - int instances = Runtime.getRuntime().availableProcessors(); - vertx.deployVerticle(() -> new ServerVerticle(items), new DeploymentOptions().setInstances(instances)); + int cores = Runtime.getRuntime().availableProcessors(); + VertxOptions options = new VertxOptions() + .setPreferNativeTransport(true) + .setEventLoopPoolSize(1); + for (int i = 0; i < cores; i++) { + Vertx vertx = Vertx.vertx(options); + // Netty's epoll transport rather than the JDK's NIO selector. Exit if it fails to + // load, so a run can never quietly fall back to NIO. + if (!vertx.isNativeTransportEnabled()) { + System.err.println("epoll native transport unavailable: " + vertx.unavailableNativeTransportCause()); + System.exit(1); + } + // A listener that fails to bind would leave this core serving nothing. + vertx.deployVerticle(new ServerVerticle(items)).onFailure(e -> { + e.printStackTrace(); + System.exit(1); + }); + } } } diff --git a/frameworks/vertx/src/main/java/httparena/ServerVerticle.java b/frameworks/vertx/src/main/java/httparena/ServerVerticle.java index 165e529459..6e5f3746d7 100644 --- a/frameworks/vertx/src/main/java/httparena/ServerVerticle.java +++ b/frameworks/vertx/src/main/java/httparena/ServerVerticle.java @@ -105,9 +105,13 @@ public Future start() { .handler(this::crudUpdate); router.post("/echo").handler(this::echo); + // SO_REUSEPORT on every port: Main runs one Vertx per core, each deploying this + // verticle once, so each core has its own listening socket on each port and accepts + // its own connections. HttpServerOptions options = new HttpServerOptions() .setHost("0.0.0.0") .setPort(8080) + .setReusePort(true) .setCompressionSupported(true); Future plain = vertx.createHttpServer(options).requestHandler(router).listen(); @@ -115,12 +119,11 @@ public Future start() { HttpServerOptions h2c = new HttpServerOptions() .setHost("0.0.0.0") .setPort(8082) + .setReusePort(true) .setHttp2ClearTextEnabled(true); Future cleartext = vertx.createHttpServer(h2c).requestHandler(router).listen(); - // json-tls and static-tls on 8081, the same router behind TLS. This verticle is - // deployed once per core, so every instance binds both ports and the listener is - // spread across all the event loops rather than parked on one. ALPN is off: the + // json-tls and static-tls on 8081, the same router behind TLS. ALPN is off: the // two profiles want HTTP/1.1 negotiated and no h2 offered. The harness only mounts // /certs for the TLS profiles. OpenSSL (BoringSSL through netty-tcnative) rather than // the JDK's SSLEngine, which Vert.x falls back to when no engine is set. @@ -128,6 +131,7 @@ public Future start() { HttpServerOptions tls = new HttpServerOptions() .setHost("0.0.0.0") .setPort(8081) + .setReusePort(true) .setCompressionSupported(true) .setSsl(true) .setUseAlpn(false)