Dist: run the runtime with compact object headers and ship its base CDS archive

AGENTS.md

@@ -154,14 +154,21 @@ This repository has multiple entry points that may trigger compilation/build beh
 
 ### Language server startup archive
 
+The compiler and the language server run with `-XX:+UseCompactObjectHeaders`: the heap is millions of small objects, and
+the 4 bytes saved in each header took 6% off the heap of the language server on castle fight (488 to 457 MB after a
+collection), at the same start time. Every launcher passes it (`launchers/*`, wurst4vscode, grill); a new one must too.
+
 wurst4vscode starts the server with `-XX:+AutoCreateSharedArchive`: the JVM writes an AppCDS archive next to the
 compiler jar when the first session ends, and the sessions after it start from it. That archive sits on the
-runtime's own base archive, which `deploy.gradle` makes (`jlink --generate-cds-archive`; macOS copies a full JDK, which
-has one) and `assembleSlimCompilerDist` checks for. Without it the JVM silently runs without any archive, which is
-how the earlier attempt went unnoticed. jlink writes the archive with a JVM of its own, which takes on the options it is started with: the
-CI exports `JAVA_TOOL_OPTIONS=-XX:+UseCompactObjectHeaders`, so `deploy.gradle` removes it (and the other JVM option variables) for that
-step, or only `classes_coh.jsa` is made, which a runtime started without that option cannot use. The extension passes `-Xlog:disable` because the JVM reports archive trouble
-on stdout, which is the protocol stream: never print anything of your own there.
+runtime's own base archive, which must be the one for compact object headers (`classes_coh.jsa`): a JVM uses a base
+archive only for the object header mode it was started with. `deploy.gradle` makes it after jlink with
+`java -XX:+UseCompactObjectHeaders -Xshare:dump` (jlink's own `--generate-cds-archive` makes the archive for the options
+of the JVM it starts, which the `JAVA_TOOL_OPTIONS` of the build machine change) and ships only that one; macOS copies a
+full JDK, which has it. `assembleSlimCompilerDist` starts the runtime with `-Xshare:on`, which fails when there is no
+usable archive. Without one the JVM silently runs without any, which is how the earlier attempt went unnoticed. The
+build runs those JVMs without `JAVA_TOOL_OPTIONS`, `_JAVA_OPTIONS` and `JDK_JAVA_OPTIONS`. The extension passes
+`-Xlog:disable` because the JVM reports archive trouble on stdout, which is the protocol stream: never print anything
+of your own there.
 
 ### Build-map pipeline (centralized)
 

CHANGELOG.md

@@ -2,9 +2,11 @@
 
 - The runtime of the distribution carries the base class data sharing archive of the JVM, which it never did: the
   slim runtime is built with jlink, and the AppCDS archive the build tried to make for the language server needs a
-  base archive to sit on, so it could not be made and was never shipped. The build now makes the base archive
-  (`jlink --generate-cds-archive`) and fails when the runtime has none, and the dead archive task and its
-  `-languageServerAppCdsTrain` option are gone. With wurst4vscode starting the server with an archive of its own
+  base archive to sit on, so it could not be made and was never shipped. The build now makes the base archive and
+  fails when the runtime cannot start from it, and the dead archive task and its
+  `-languageServerAppCdsTrain` option are gone. The launchers (and wurst4vscode and grill) start the JVM with
+  compact object headers, which took 6% off the heap of the language server on castle fight, and the archive is
+  the one for that mode. With wurst4vscode starting the server with an archive of its own
   (written next to the compiler jar when the first session ends), the language server of a small project is ready
   after 3.7 s instead of 4.4 s, and castle fight after 17.3 s instead of 19.0 s (medians of four starts).
 

de.peeeq.wurstscript/deploy.gradle

@@ -59,6 +59,7 @@ def jdepsPathProvider = javaHomeProvider.map { home ->
 def jlinkPathProvider = javaHomeProvider.map { home ->
     new File(home, "bin/jlink${os.isWindows() ? '.exe' : ''}").absolutePath
 }
+def javaExecutableName = os.isWindows() ? "java.exe" : "java"
 def jmodsDirProvider = javaHomeProvider.map { home ->
     new File(home, "jmods").absolutePath
 }
@@ -97,6 +98,31 @@ abstract class JdepsRawTask extends DefaultTask {
     }
 }
 
+/**
+ * Runs the java of a runtime image the way the distribution starts it. The language server and the compiler run with
+ * compact object headers (the heap is millions of small objects), and a base class data sharing archive is made for
+ * the object header mode of the JVM which writes it, so the archive is made and checked with that option.
+ * The options of the build machine are removed: CI exports JAVA_TOOL_OPTIONS for its tests, and the archive must not
+ * depend on them.
+ */
+class RuntimeJava {
+    static final List<String> COMPACT_HEADERS = ["-XX:+UseCompactObjectHeaders"]
+
+    static String run(File java, List<String> options) {
+        def command = [java.absolutePath] + COMPACT_HEADERS + options
+        def pb = new ProcessBuilder(command)
+        pb.redirectErrorStream(true)
+        ["JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "JDK_JAVA_OPTIONS"].each { pb.environment().remove(it) }
+        def process = pb.start()
+        def output = process.inputStream.getText("UTF-8")
+        def exit = process.waitFor()
+        if (exit != 0) {
+            throw new org.gradle.api.GradleException("`" + command.join(" ") + "` exited with " + exit + ": " + output)
+        }
+        return output
+    }
+}
+
 def fatJarProvider = tasks.named("shadowJar").flatMap { it.archiveFile } // Provider<RegularFile>
 
 tasks.register("jdepsRaw", JdepsRawTask) {
@@ -172,35 +198,30 @@ tasks.register("jlinkRuntime25", Exec) {
         logger.lifecycle("[jlink] Modules: ${mods}")
 
         executable = jlinkPathProvider.get()
-        // The base archive is written by a JVM which jlink starts, and takes on the object header mode of the options
-        // that JVM is given. CI exports JAVA_TOOL_OPTIONS=-XX:+UseCompactObjectHeaders for the tests, which would
-        // leave only classes_coh.jsa, which a runtime started without that option cannot use.
-        ["JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "JDK_JAVA_OPTIONS"].each { environment.remove(it) }
         args "--verbose",
                 "--module-path", jmodsDirProvider.get(),
                 "--add-modules", mods,
                 "--no-header-files",
                 "--no-man-pages",
                 "--strip-debug",
                 "--compress=zip-9",
-                // The language server starts from an AppCDS archive which the JVM writes next to the compiler
-                // jar (wurst4vscode, -XX:+AutoCreateSharedArchive). That archive sits on top of the base archive
-                // of the runtime; without one the JVM silently runs without any.
-                "--generate-cds-archive",
                 "--output", outDir.absolutePath
     }
 
     doLast {
-        // checked on the image itself: the dist folder is only added to, and may still hold an archive of an earlier build
-        def image = jreImageDir.get().asFile
-        if (!["bin/server/classes.jsa", "lib/server/classes.jsa"].any { new File(image, it).exists() }) {
-            throw new GradleException("jlink made no base CDS archive (classes.jsa under bin/server or lib/server) in " + image)
-        }
+        // The language server starts from an AppCDS archive which the JVM writes next to the compiler jar
+        // (wurst4vscode, -XX:+AutoCreateSharedArchive). That archive sits on top of the base archive of the runtime;
+        // without one the JVM silently runs without any. Only the archive for compact object headers is made: it is
+        // the mode every launcher uses, and a second archive would add 14 MB to the download. (jlink's own
+        // --generate-cds-archive writes the archive for the options of the JVM it starts, which is not under our control.)
+        RuntimeJava.run(new File(jreImageDir.get().asFile, "bin/" + javaExecutableName), ["-Xshare:dump"])
     }
 }
 
 // 3) Assemble folder layout: jre + compiler.jar (no manifest)
-tasks.register("assembleSlimCompilerDist", Copy) {
+// A Sync, not a Copy: the folder is packaged as it is, so a file of an earlier build (the base archive of another
+// header mode, say) must not stay in it.
+tasks.register("assembleSlimCompilerDist", Sync) {
     description = "Assembles dist folder with JRE and compiler.jar (no manifest)."
     group = "distribution"
 
@@ -215,28 +236,23 @@ tasks.register("assembleSlimCompilerDist", Copy) {
         }
     } else {
         dependsOn("jlinkRuntime25", "shadowJar")
-        from(jreImageDir) {
-            into("wurst-runtime")
-            // for heaps above 32 GB, which this does not run with; it would add 14 MB to the download
-            exclude("**/classes_nocoops*.jsa")
-        }
+        from(jreImageDir) { into("wurst-runtime") }
     }
 
     from(fatJar) { into("wurst-compiler") }
     into(distRoot)
 
     doLast {
         logger.lifecycle("[dist] Folder ready at: ${distRoot.get().asFile}")
+        // -Xshare:on makes the JVM fail when it has no usable base archive for compact object headers, where it
+        // would otherwise run without one and nobody would notice
         def runtime = new File(distRoot.get().asFile, "wurst-runtime")
-        if (!["bin/server/classes.jsa", "lib/server/classes.jsa"].any { new File(runtime, it).exists() }) {
-            throw new GradleException("The runtime has no base CDS archive (classes.jsa under bin/server or lib/server), " +
-                    "so the language server cannot start from an AppCDS archive.")
-        }
+        RuntimeJava.run(new File(runtime, "bin/" + javaExecutableName), ["-Xshare:on", "-version"])
     }
 }
 
 // Add launch scripts
-tasks.named("assembleSlimCompilerDist", Copy) { t ->
+tasks.named("assembleSlimCompilerDist", Sync) { t ->
     if (os.isWindows()) {
         from("launchers/wurstscript.cmd") { into(".") }
         from("launchers/wurstscript")     { into(".") }

de.peeeq.wurstscript/launchers/grill

@@ -23,4 +23,4 @@ if [[ ! -x "$RUNTIME" ]]; then
   exit 3
 fi
 
-exec "$RUNTIME" -Dfile.encoding=UTF-8 --enable-native-access=ALL-UNNAMED -jar "$JAR" "$@"
+exec "$RUNTIME" -XX:+UseCompactObjectHeaders -Dfile.encoding=UTF-8 --enable-native-access=ALL-UNNAMED -jar "$JAR" "$@"

de.peeeq.wurstscript/launchers/grill.cmd

@@ -31,7 +31,7 @@ if not exist "%JAVA%" (
     goto :restore
 )
 
-"%JAVA%" -Dfile.encoding=UTF-8 --enable-native-access=ALL-UNNAMED -jar "%GRILL_JAR%" %*
+"%JAVA%" -XX:+UseCompactObjectHeaders -Dfile.encoding=UTF-8 --enable-native-access=ALL-UNNAMED -jar "%GRILL_JAR%" %*
 
 :restore
 rem Restore previous code page if we captured it

de.peeeq.wurstscript/launchers/wurstscript

@@ -23,4 +23,4 @@ if [[ ! -x "$RUNTIME" ]]; then
   exit 3
 fi
 
-exec "$RUNTIME" -Dfile.encoding=UTF-8 -jar "$JAR" "$@"
+exec "$RUNTIME" -XX:+UseCompactObjectHeaders -Dfile.encoding=UTF-8 -jar "$JAR" "$@"

de.peeeq.wurstscript/launchers/wurstscript.cmd

@@ -29,7 +29,7 @@ if not exist "%JAVA%" (
     goto :restore
 )
 
-"%JAVA%" -Dfile.encoding=UTF-8 -jar "%JAR%" %*
+"%JAVA%" -XX:+UseCompactObjectHeaders -Dfile.encoding=UTF-8 -jar "%JAR%" %*
 
 :restore
 if defined _OLDCP chcp %_OLDCP% >NUL