diff --git a/frameworks/vertx/README.md b/frameworks/vertx/README.md index cbbaf105de..c49a36fc69 100644 --- a/frameworks/vertx/README.md +++ b/frameworks/vertx/README.md @@ -1,6 +1,6 @@ # vertx -Eclipse Vert.x with vertx-web on Netty, one server verticle per core. +Eclipse Vert.x with vertx-web on Netty, one Vert.x instance and `SO_REUSEPORT` listener per core. ## Stack @@ -31,13 +31,14 @@ Eclipse Vert.x with vertx-web on Netty, one server verticle per core. - Compression through `HttpServerOptions.setCompressionSupported(true)` - TLS through OpenSSL (BoringSSL from `netty-tcnative-boringssl-static`) via `OpenSSLEngineOptions`, rather than the JDK's `SSLEngine` that Vert.x uses when no engine is set -- The verticle is deployed once per core, all instances sharing port 8080 +- One `Vertx` per core, each with a single event loop and one server verticle. Every server sets + `setReusePort(true)`, so each core binds its own `SO_REUSEPORT` socket on each port and the + kernel spreads new connections across them. Inside one `Vertx`, servers on the same host:port + share a single listening socket and a single acceptor thread - `/echo` is read from the request stream, so the 20 MB body is never buffered `json-tls` and `static-tls` listen on `8081` when `/certs/server.crt` and `/certs/server.key` are -mounted. The verticle is deployed once per core and each instance binds both ports, so the TLS -listener is spread across every event loop rather than parked on one. ALPN is off: those two -profiles want HTTP/1.1 negotiated and no h2 offered. +mounted. ALPN is off: those two profiles want HTTP/1.1 negotiated and no h2 offered. Static file bodies are read from disk on every request - `sendFile` hands the descriptor to the kernel rather than holding the bytes - and the pre-compressed sibling is picked per request. diff --git a/frameworks/vertx/meta.json b/frameworks/vertx/meta.json index 35c62f4adc..9408630e4e 100644 --- a/frameworks/vertx/meta.json +++ b/frameworks/vertx/meta.json @@ -10,7 +10,7 @@ "response": true }, "engine": "Netty", - "description": "Eclipse Vert.x 5 with vertx-web on Netty's epoll transport, one server verticle per core. Routing and query/path params through the Router API, JsonObject serialization, gzip through the server compression option, TLS through BoringSSL (netty-tcnative).", + "description": "Eclipse Vert.x 5 with vertx-web on Netty's epoll transport, one Vert.x instance with its own SO_REUSEPORT listener per core. Routing and query/path params through the Router API, JsonObject serialization, gzip through the server compression option, TLS through BoringSSL (netty-tcnative).", "repo": "https://github.com/eclipse-vertx/vert.x", "enabled": true, "tests": [ diff --git a/frameworks/vertx/src/main/java/httparena/Main.java b/frameworks/vertx/src/main/java/httparena/Main.java index e2fc77f043..6ca94bb9f8 100644 --- a/frameworks/vertx/src/main/java/httparena/Main.java +++ b/frameworks/vertx/src/main/java/httparena/Main.java @@ -5,7 +5,6 @@ import java.nio.file.Files; import java.nio.file.Path; -import io.vertx.core.DeploymentOptions; import io.vertx.core.Vertx; import io.vertx.core.VertxOptions; import io.vertx.core.buffer.Buffer; @@ -26,21 +25,36 @@ public static void main(String[] args) throws Exception { } catch (Exception ignored) { } - // Netty's epoll transport rather than the JDK's NIO selector. Exit if it fails to - // load, so a run can never quietly fall back to NIO. - Vertx vertx = Vertx.vertx(new VertxOptions().setPreferNativeTransport(true)); - if (!vertx.isNativeTransportEnabled()) { - System.err.println("epoll native transport unavailable: " + vertx.unavailableNativeTransportCause()); - System.exit(1); - } // The TLS port runs on OpenSSL, and without it every 8081 listen fails. Exit with the // reason up front, the same as for epoll, rather than leave it to a failed deployment. if (!OpenSSLEngineOptions.isAvailable()) { System.err.println("OpenSSL unavailable: " + OpenSsl.unavailabilityCause()); System.exit(1); } + + // One Vertx per core, each with a single event loop. A Vertx folds every server on the + // same host:port into one listening socket, accepted on its one acceptor thread whatever + // setReusePort says, so that thread caps the connections it can take per second. With a + // Vertx per core, each binds its own SO_REUSEPORT socket (set in ServerVerticle) and the + // kernel spreads new connections across them. Item[] items = dataset; - int instances = Runtime.getRuntime().availableProcessors(); - vertx.deployVerticle(() -> new ServerVerticle(items), new DeploymentOptions().setInstances(instances)); + int cores = Runtime.getRuntime().availableProcessors(); + VertxOptions options = new VertxOptions() + .setPreferNativeTransport(true) + .setEventLoopPoolSize(1); + for (int i = 0; i < cores; i++) { + Vertx vertx = Vertx.vertx(options); + // Netty's epoll transport rather than the JDK's NIO selector. Exit if it fails to + // load, so a run can never quietly fall back to NIO. + if (!vertx.isNativeTransportEnabled()) { + System.err.println("epoll native transport unavailable: " + vertx.unavailableNativeTransportCause()); + System.exit(1); + } + // A listener that fails to bind would leave this core serving nothing. + vertx.deployVerticle(new ServerVerticle(items)).onFailure(e -> { + e.printStackTrace(); + System.exit(1); + }); + } } } diff --git a/frameworks/vertx/src/main/java/httparena/ServerVerticle.java b/frameworks/vertx/src/main/java/httparena/ServerVerticle.java index 165e529459..6e5f3746d7 100644 --- a/frameworks/vertx/src/main/java/httparena/ServerVerticle.java +++ b/frameworks/vertx/src/main/java/httparena/ServerVerticle.java @@ -105,9 +105,13 @@ public Future start() { .handler(this::crudUpdate); router.post("/echo").handler(this::echo); + // SO_REUSEPORT on every port: Main runs one Vertx per core, each deploying this + // verticle once, so each core has its own listening socket on each port and accepts + // its own connections. HttpServerOptions options = new HttpServerOptions() .setHost("0.0.0.0") .setPort(8080) + .setReusePort(true) .setCompressionSupported(true); Future plain = vertx.createHttpServer(options).requestHandler(router).listen(); @@ -115,12 +119,11 @@ public Future start() { HttpServerOptions h2c = new HttpServerOptions() .setHost("0.0.0.0") .setPort(8082) + .setReusePort(true) .setHttp2ClearTextEnabled(true); Future cleartext = vertx.createHttpServer(h2c).requestHandler(router).listen(); - // json-tls and static-tls on 8081, the same router behind TLS. This verticle is - // deployed once per core, so every instance binds both ports and the listener is - // spread across all the event loops rather than parked on one. ALPN is off: the + // json-tls and static-tls on 8081, the same router behind TLS. ALPN is off: the // two profiles want HTTP/1.1 negotiated and no h2 offered. The harness only mounts // /certs for the TLS profiles. OpenSSL (BoringSSL through netty-tcnative) rather than // the JDK's SSLEngine, which Vert.x falls back to when no engine is set. @@ -128,6 +131,7 @@ public Future start() { HttpServerOptions tls = new HttpServerOptions() .setHost("0.0.0.0") .setPort(8081) + .setReusePort(true) .setCompressionSupported(true) .setSsl(true) .setUseAlpn(false)