diff --git a/README.md b/README.md index 7315d562..29ed1f8d 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,23 @@ In XTMF2, open **Settings**, open **RunServers**, and add a remote endpoint. Ent The GUI pins the server certificate to this fingerprint and authenticates with the token before creating the RunServer bus. A mismatched certificate or token is rejected. Do not expose the TCP port to untrusted networks; use firewall rules or a private network as appropriate. +## Distributed estimation + +Estimation runs can evaluate candidate parameter vectors concurrently across multiple connected RunServers. Normal model runs and calibration runs continue to use one RunServer. + +To start a distributed estimation run: + +1. Connect the required RunServers from **Settings** > **RunServers**. +2. Open the model system and choose **Run Estimation**. +3. Select two or more RunServers in the run configuration dialog and choose the orchestrator RunServer. The other selected servers are workers. +4. If the model has an estimation `InputDirectory`, review its worker-local value for each selected worker. These values are saved in the GUI settings by model node ID and are reused on later estimation runs. + +The orchestrator RunServer owns the estimation algorithm and connects directly to the workers. Each worker constructs and validates its own local copy of the model. Input-directory overrides are applied only to that worker's copy; the saved model system is not modified. The orchestrator must be able to reach every worker's configured TCP endpoint. + +After submission, the GUI is not required for the optimization to continue. The orchestrator persists `estimation-completion.json` in the run directory when the job finishes, including the best parameter values and completion status. + +RunServers that receive path overrides must be running the current shared-estimation worker build. Runs without path overrides retain the version-1 shared-estimation protocol and remain compatible with workers that support the original distributed-estimation protocol. + ## Main Branches There are 4 major branches for XTMF 2 intended for different purposes: diff --git a/XTMF2.sln b/XTMF2.sln index 3e9aa680..7dabf2f5 100644 --- a/XTMF2.sln +++ b/XTMF2.sln @@ -33,6 +33,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "XTMF2.Run", "src\XTMF2.Run\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "XTMF2.GUI", "src\XTMF2.GUI\XTMF2.GUI.csproj", "{A16D11CD-0D70-4558-8CEA-52F0601C215A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "XTMF2.RunServer.Updater", "src\XTMF2.RunServer.Updater\XTMF2.RunServer.Updater.csproj", "{E3239050-BF93-44A7-BC91-6F865087EBB7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -139,6 +141,18 @@ Global {A16D11CD-0D70-4558-8CEA-52F0601C215A}.Release|x64.Build.0 = Release|Any CPU {A16D11CD-0D70-4558-8CEA-52F0601C215A}.Release|x86.ActiveCfg = Release|Any CPU {A16D11CD-0D70-4558-8CEA-52F0601C215A}.Release|x86.Build.0 = Release|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|x64.ActiveCfg = Debug|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|x64.Build.0 = Debug|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|x86.ActiveCfg = Debug|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|x86.Build.0 = Debug|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|Any CPU.Build.0 = Release|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|x64.ActiveCfg = Release|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|x64.Build.0 = Release|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|x86.ActiveCfg = Release|Any CPU + {E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -152,6 +166,7 @@ Global {A1B2C3D4-E5F6-7890-ABCD-EF1234567890} = {77789778-93F5-46A6-9114-CCE855E5919C} {F6999FCE-4C4F-4CF6-9B70-0DEA31FB5E6D} = {9040CC77-712E-414D-8453-391F9F348218} {A16D11CD-0D70-4558-8CEA-52F0601C215A} = {9040CC77-712E-414D-8453-391F9F348218} + {E3239050-BF93-44A7-BC91-6F865087EBB7} = {9040CC77-712E-414D-8453-391F9F348218} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {15025AE3-C5AA-4CD4-BC38-D1C1BA8904AF} diff --git a/src/XTMF2.Bus/CreateStreams.cs b/src/XTMF2.Bus/CreateStreams.cs new file mode 100644 index 00000000..e69de29b diff --git a/src/XTMF2.Client/Program.cs b/src/XTMF2.Client/Program.cs index ac9ec8c5..10da6f74 100644 --- a/src/XTMF2.Client/Program.cs +++ b/src/XTMF2.Client/Program.cs @@ -19,6 +19,7 @@ You should have received a copy of the GNU General Public License using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -27,15 +28,22 @@ You should have received a copy of the GNU General Public License using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using XTMF2.Bus; +using XTMF2.Bus.Optimization; using XTMF2.Configuration; namespace XTMF2.Client { public class Program { + private static string[] _originalArguments = Array.Empty(); + private static readonly Stopwatch StartupStopwatch = Stopwatch.StartNew(); + [MTAThread] static void Main(string[] args) { + _originalArguments = args; + LogStartup("Process started; parsing command-line arguments."); + LogDeploymentStartupSummary(); if (args.Length == 0) { Console.WriteLine("Usage: XTMF.Run [-setup-security DIRECTORY] [-loadDLL dllPath] [-tcp ADDRESS PORT -security DIRECTORY] [-namedPipe PIPE_NAME]"); @@ -150,14 +158,18 @@ private static bool IsLoopbackAddress(string address) private static void RunTcpServer(string address, int port, string? securityDirectory, List extraDlls) { + LogStartup($"Initializing TCP server on {address}:{port}."); X509Certificate2? certificate = null; string? token = null; if (securityDirectory is not null) { + var securityStopwatch = Stopwatch.StartNew(); + LogStartup("Loading TCP server security credentials."); try { certificate = RunServerSecurity.LoadCertificate(securityDirectory); token = RunServerSecurity.LoadToken(securityDirectory); + LogStartup($"TCP server security credentials loaded in {securityStopwatch.ElapsedMilliseconds} ms."); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or CryptographicException) { @@ -166,6 +178,8 @@ private static void RunTcpServer(string address, int port, string? securityDirec } } + var listenerStopwatch = Stopwatch.StartNew(); + LogStartup("Binding the TCP listener."); if (!CreateStreams.CreateTcpListener(address, port, out var listener, out var boundPort, out var error)) { Console.WriteLine("Error creating TCP RunServer listener\r\n" + error); @@ -177,8 +191,12 @@ private static void RunTcpServer(string address, int port, string? securityDirec if (certificate is not null) Console.WriteLine($"Certificate fingerprint: {RunServerSecurity.GetFingerprint(certificate)}"); Console.Out.Flush(); + LogStartup($"TCP listener bound in {listenerStopwatch.ElapsedMilliseconds} ms on {address}:{boundPort}."); + SignalDeploymentReady(); using (tcpListener) using (var shutdown = new CancellationTokenSource()) + using (var remoteEstimationRegistry = new RemoteSharedEstimationRegistry()) + using (var remoteRunRegistry = new RemoteRunRegistry()) { Console.CancelKeyPress += (_, eventArgs) => { @@ -187,6 +205,7 @@ private static void RunTcpServer(string address, int port, string? securityDirec tcpListener.Stop(); }; + LogStartup($"READY: accepting TCP connections on {address}:{boundPort}."); while (!shutdown.IsCancellationRequested) { TcpClient? client = null; @@ -212,13 +231,16 @@ private static void RunTcpServer(string address, int port, string? securityDirec Console.Out.Flush(); var acceptedClient = client; _ = securityDirectory is null - ? Task.Run(() => RunTcpClientUnsecured(acceptedClient, extraDlls)) - : Task.Run(() => RunTcpClient(acceptedClient, certificate!, token!, extraDlls)); + ? Task.Run(() => RunTcpClientUnsecured(acceptedClient, extraDlls, + remoteEstimationRegistry, remoteRunRegistry)) + : Task.Run(() => RunTcpClient(acceptedClient, certificate!, token!, extraDlls, + remoteEstimationRegistry, remoteRunRegistry)); } } } - private static void RunTcpClientUnsecured(TcpClient client, List extraDlls) + private static void RunTcpClientUnsecured(TcpClient client, List extraDlls, + RemoteSharedEstimationRegistry remoteEstimationRegistry, RemoteRunRegistry remoteRunRegistry) { var remoteEndpoint = client.Client.RemoteEndPoint?.ToString() ?? "unknown endpoint"; Console.WriteLine($"Local RunServer connection accepted from {remoteEndpoint}"); @@ -228,7 +250,10 @@ private static void RunTcpClientUnsecured(TcpClient client, List extraDl { try { - RunClient(stream, extraDlls, usePrivateWorkspace: true); + RunClient(stream, extraDlls, usePrivateWorkspace: true, + allowDeployment: false, + remoteEstimationRegistry: remoteEstimationRegistry, + remoteRunRegistry: remoteRunRegistry); } catch (Exception ex) { @@ -238,14 +263,16 @@ private static void RunTcpClientUnsecured(TcpClient client, List extraDl } } - private static void RunTcpClient(TcpClient client, X509Certificate2 certificate, string token, List extraDlls) + private static void RunTcpClient(TcpClient client, X509Certificate2 certificate, string token, List extraDlls, + RemoteSharedEstimationRegistry remoteEstimationRegistry, RemoteRunRegistry remoteRunRegistry) { var remoteEndpoint = client.Client.RemoteEndPoint?.ToString() ?? "unknown endpoint"; + var authenticationStopwatch = Stopwatch.StartNew(); using (client) { if (!CreateStreams.AuthenticateSecureTcpClient(client, certificate, token, out var stream, out var error)) { - Console.WriteLine($"RunServer security validation failed for {remoteEndpoint}: {error}"); + Console.WriteLine($"RunServer TLS/token authentication failed for {remoteEndpoint} after {authenticationStopwatch.ElapsedMilliseconds} ms: {error}"); Console.Out.Flush(); return; } @@ -255,7 +282,10 @@ private static void RunTcpClient(TcpClient client, X509Certificate2 certificate, { try { - RunClient(stream, extraDlls, usePrivateWorkspace: true); + RunClient(stream, extraDlls, usePrivateWorkspace: true, + allowDeployment: true, + remoteEstimationRegistry: remoteEstimationRegistry, + remoteRunRegistry: remoteRunRegistry); } catch (Exception ex) { @@ -268,16 +298,191 @@ private static void RunTcpClient(TcpClient client, X509Certificate2 certificate, } } - private static void RunClient(Stream serverStream, List extraDlls, SystemConfiguration? config = null, bool usePrivateWorkspace = false) + private static void RunClient(Stream serverStream, List extraDlls, SystemConfiguration? config = null, + bool usePrivateWorkspace = false, RemoteSharedEstimationRegistry? remoteEstimationRegistry = null, + RemoteRunRegistry? remoteRunRegistry = null, bool allowDeployment = false) { + var runtimeStopwatch = Stopwatch.StartNew(); + LogStartup("Initializing the XTMF runtime for a client connection."); var runtime = XTMFRuntime.CreateRuntime(config); + LogStartup($"XTMF runtime initialized in {runtimeStopwatch.ElapsedMilliseconds} ms."); var loadedConfig = runtime.SystemConfiguration; foreach (var dll in extraDlls) { + var assemblyStopwatch = Stopwatch.StartNew(); + LogStartup($"Loading additional assembly '{Path.GetFileName(dll)}'."); loadedConfig.LoadAssembly(dll); + LogStartup($"Additional assembly loaded in {assemblyStopwatch.ElapsedMilliseconds} ms."); + } + using var ownedRemoteEstimationRegistry = remoteEstimationRegistry is null + ? new RemoteSharedEstimationRegistry() + : null; + var registry = remoteEstimationRegistry ?? ownedRemoteEstimationRegistry!; + using var clientBus = new RunServerBus(serverStream, true, runtime, extraDlls, + System.Diagnostics.Debugger.IsAttached, usePrivateWorkspace, remoteRunRegistry, allowDeployment); + using var sharedEstimationWorker = clientBus.AttachSharedEstimationWorker(); + using var remoteCoordinator = new RemoteSharedEstimationCoordinatorSession(clientBus, registry); + clientBus.SetDeploymentGate( + () => + { + remoteRunRegistry?.BeginDrain(); + registry.BeginDrain(); + }, + () => + { + remoteRunRegistry?.EndDrain(); + registry.EndDrain(); + }, + () => + (remoteRunRegistry?.IsIdle ?? true) && registry.IsIdle, + RestartWithOriginalArguments); + clientBus.SetSharedActivityProviders(registry.GetActiveActivities, + sharedEstimationWorker.GetActiveActivities); + try + { + clientBus.ProcessRequests(); + } + finally + { + remoteRunRegistry?.Detach(clientBus); + } + } + + private static void LogStartup(string message) + { + Console.Error.WriteLine($"[RunServer +{StartupStopwatch.ElapsedMilliseconds} ms] {message}"); + Console.Error.Flush(); + } + + private static bool RestartWithOriginalArguments(string stagingRoot) + { + if (!Directory.Exists(stagingRoot) || !File.Exists(Path.Combine(stagingRoot, "deployment.zip"))) + return false; + + var processPath = Environment.ProcessPath; + if (string.IsNullOrWhiteSpace(processPath)) + return false; + + string? deploymentDirectory = null; + try + { + var processDirectory = GetInstallationDirectory(processPath); + if (string.IsNullOrWhiteSpace(processDirectory)) + return false; + + deploymentDirectory = Path.Combine(processDirectory, ".deployments", + DateTime.UtcNow.ToString("yyyyMMddHHmmssfff") + "-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(deploymentDirectory); + var archivePath = Path.Combine(deploymentDirectory, "deployment.zip"); + File.Copy(Path.Combine(stagingRoot, "deployment.zip"), archivePath); + + var updaterPath = Path.Combine(processDirectory, + OperatingSystem.IsWindows() ? "XTMF2.RunServer.Updater.exe" : "XTMF2.RunServer.Updater"); + if (!File.Exists(updaterPath)) + throw new FileNotFoundException("The RunServer updater executable is missing.", updaterPath); + + var updaterInfo = new ProcessStartInfo(updaterPath) + { + UseShellExecute = false, + WorkingDirectory = processDirectory + }; + updaterInfo.ArgumentList.Add("--old-pid"); + updaterInfo.ArgumentList.Add(Environment.ProcessId.ToString()); + updaterInfo.ArgumentList.Add("--install-dir"); + updaterInfo.ArgumentList.Add(processDirectory); + updaterInfo.ArgumentList.Add("--server-path"); + updaterInfo.ArgumentList.Add(processPath); + updaterInfo.ArgumentList.Add("--server-assembly"); + updaterInfo.ArgumentList.Add(typeof(Program).Assembly.Location); + updaterInfo.ArgumentList.Add("--deployment-dir"); + updaterInfo.ArgumentList.Add(deploymentDirectory); + updaterInfo.ArgumentList.Add("--staging-root"); + updaterInfo.ArgumentList.Add(stagingRoot); + foreach (var argument in _originalArguments) + { + updaterInfo.ArgumentList.Add("--server-arg"); + updaterInfo.ArgumentList.Add(argument); + } + if (System.Diagnostics.Process.Start(updaterInfo) is null) + throw new InvalidOperationException("Unable to start the RunServer updater."); + Environment.Exit(0); + return true; + } + catch (Exception exception) when (exception is IOException or InvalidDataException or UnauthorizedAccessException or + InvalidOperationException or System.ComponentModel.Win32Exception or ArgumentException) + { + Console.Error.WriteLine($"Unable to restart RunServer: {exception.Message}"); + if (deploymentDirectory is not null) + { + try + { + Directory.Delete(deploymentDirectory, recursive: true); + } + catch (Exception cleanupException) when (cleanupException is IOException or UnauthorizedAccessException) + { + Console.Error.WriteLine($"Unable to clean up failed deployment: {cleanupException.Message}"); + } + } + return false; + } + } + + private static void SignalDeploymentReady() + { + var readyFile = Environment.GetEnvironmentVariable("XTMF2_DEPLOYMENT_READY_FILE"); + if (!string.IsNullOrWhiteSpace(readyFile)) + { + try + { + File.WriteAllText(readyFile, "ready"); + } + catch (IOException exception) + { + Console.Error.WriteLine($"Unable to signal RunServer readiness: {exception.Message}"); + } + } + Environment.SetEnvironmentVariable("XTMF2_DEPLOYMENT_READY_FILE", null); + } + + private static bool IsDotnetHost(string processPath) + { + var fileName = Path.GetFileNameWithoutExtension(processPath); + return string.Equals(fileName, "dotnet", StringComparison.OrdinalIgnoreCase); + } + + private static string GetInstallationDirectory(string processPath) + { + if (IsDotnetHost(processPath)) + return Path.GetDirectoryName(typeof(Program).Assembly.Location) ?? Environment.CurrentDirectory; + return Path.GetDirectoryName(processPath) ?? Environment.CurrentDirectory; + } + + private static void LogDeploymentStartupSummary() + { + var manifestPath = Environment.GetEnvironmentVariable("XTMF2_DEPLOYMENT_MANIFEST"); + if (string.IsNullOrWhiteSpace(manifestPath) || !File.Exists(manifestPath)) + return; + + try + { + var modules = File.ReadAllLines(manifestPath) + .Where(module => module.Length <= 256 && + !module.Any(char.IsControl) && + (module.Equals("XTMF2.dll", StringComparison.Ordinal) || + module.StartsWith("Modules/", StringComparison.Ordinal))) + .Take(4096) + .ToArray(); + Console.WriteLine($"RunServer restarted with deployment containing {modules.Length} module file(s):"); + foreach (var module in modules) + Console.WriteLine($" {module}"); + Console.Out.Flush(); + Environment.SetEnvironmentVariable("XTMF2_DEPLOYMENT_MANIFEST", null); + } + catch (IOException exception) + { + Console.WriteLine($"RunServer restarted, but deployment overview could not be read: {exception.Message}"); + Console.Out.Flush(); } - using var clientBus = new RunServerBus(serverStream, true, runtime, extraDlls, System.Diagnostics.Debugger.IsAttached, usePrivateWorkspace); - clientBus.ProcessRequests(); } } } \ No newline at end of file diff --git a/src/XTMF2.Client/Properties/launchSettings.json b/src/XTMF2.Client/Properties/launchSettings.json index fccb7da5..8d3b4bac 100644 --- a/src/XTMF2.Client/Properties/launchSettings.json +++ b/src/XTMF2.Client/Properties/launchSettings.json @@ -3,6 +3,10 @@ "XTMF.Run": { "commandName": "Project", "commandLineArgs": "-loadDLL \"../../../../../tests/XTMF2.UnitTests/bin/Debug/netcoreapp3.1/XTMF2.UnitTests.dll\" -namedPipe 123" + }, + "XTMF.RunServer": { + "commandName": "Project", + "commandLineArgs": "-tcp 0.0.0.0 1447 -security .\\runserver-security\\" } } } \ No newline at end of file diff --git a/src/XTMF2.Client/XTMF2.RunServer.csproj b/src/XTMF2.Client/XTMF2.RunServer.csproj index dccff50c..7754176b 100644 --- a/src/XTMF2.Client/XTMF2.RunServer.csproj +++ b/src/XTMF2.Client/XTMF2.RunServer.csproj @@ -26,6 +26,7 @@ + diff --git a/src/XTMF2.GUI/App.axaml b/src/XTMF2.GUI/App.axaml index fe3db5ba..71d38d79 100644 --- a/src/XTMF2.GUI/App.axaml +++ b/src/XTMF2.GUI/App.axaml @@ -332,6 +332,12 @@ + +