diff --git a/README.md b/README.md index 29ed1f8..6e1f2d5 100644 --- a/README.md +++ b/README.md @@ -54,12 +54,29 @@ This creates: * `runserver-key.pem`: the certificate private key. Keep this file private. * `runserver-token.txt`: the authentication token. Keep this file private. -The setup command prints the certificate's SHA-256 fingerprint. The server also prints the same fingerprint each time it starts; this is safe to share with GUI users and is not the authentication token. Start the remote server with the security directory: +The setup command prints the certificate's SHA-256 fingerprint. The server also prints the same fingerprint each time it starts; this is safe to share with GUI users and is not the authentication token. Create a `XTMF2.RunServer.json` file in the server's configuration directory (or pass its path with `-config`): + +```json +{ + "version": 1, + "serverStorageDirectory": "./xtmf-data", + "listenAddress": "0.0.0.0", + "port": 5000, + "securityDirectory": "./runserver-security", + "additionalAssemblies": [], + "allowDeployment": true, + "runWorkspaceBaseDirectory": "./run-workspaces" +} +``` + +This is RunServer configuration version 1. Paths in the JSON file are resolved relative to the file. The server derives its `RemoteRuns` and `Deployments` directories from `serverStorageDirectory`; each run gets its own subdirectory under `runWorkspaceBaseDirectory`, which defaults to `serverStorageDirectory/Runs`. Listening outside loopback and enabling deployment both require TLS/token security. Keep the security files private. Start the server with: ```bash -dotnet run -c Release --project src/XTMF2.Client/XTMF2.RunServer.csproj -- -tcp 0.0.0.0 5000 -security ./runserver-security +dotnet run -c Release --project src/XTMF2.Client/XTMF2.RunServer.csproj -- -config ./XTMF2.RunServer.json ``` +If no arguments are supplied, RunServer looks for `XTMF2.RunServer.config` beside its executable. The GUI-launched local RunServer receives the GUI's configured storage directory directly and binds to loopback on an ephemeral port. Each remote RunServer operator manages the JSON file on that server; GUI endpoint settings only describe how clients connect. + In XTMF2, open **Settings**, open **RunServers**, and add a remote endpoint. Enter: 1. The remote machine's address and TCP port. diff --git a/src/XTMF2.Client/Program.cs b/src/XTMF2.Client/Program.cs index fb37024..c11842a 100644 --- a/src/XTMF2.Client/Program.cs +++ b/src/XTMF2.Client/Program.cs @@ -21,6 +21,9 @@ You should have received a copy of the GNU General Public License using System.Diagnostics; using System.Linq; using System.IO; +using System.Globalization; +using System.Text; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using System.Net.Sockets; @@ -36,21 +39,41 @@ namespace XTMF2.Client public class Program { private static string[] _originalArguments = Array.Empty(); - private static readonly Stopwatch StartupStopwatch = Stopwatch.StartNew(); [MTAThread] static void Main(string[] args) { - _originalArguments = args; - LogStartup("Process started; parsing command-line arguments."); - LogDeploymentStartupSummary(); + ConfigureTimestampedConsole(); if (args.Length == 0) { - Console.WriteLine("Usage: XTMF.Run [-setup-security DIRECTORY] [-loadDLL dllPath] [-tcp ADDRESS PORT -security DIRECTORY] [-namedPipe PIPE_NAME]"); + var defaultConfigPath = Path.Combine(AppContext.BaseDirectory, "XTMF2.RunServer.config"); + if (!File.Exists(defaultConfigPath)) + { + Console.WriteLine($"RunServer configuration file not found: {defaultConfigPath}"); + Console.WriteLine("Usage: XTMF2.RunServer [-config path] [-setup-security DIRECTORY]"); + return; + } + StartConfiguredServer(defaultConfigPath); return; } + + if (string.Equals(args[0], "-config", StringComparison.OrdinalIgnoreCase)) + { + if (args.Length != 2) + { + Console.WriteLine("Usage: XTMF2.RunServer -config path"); + return; + } + StartConfiguredServer(args[1]); + return; + } + + _originalArguments = args; + LogStartup("Process started; parsing command-line arguments."); + LogDeploymentStartupSummary(); List dllsToLoad = new List(); string? error = null; + string? serverStorageDirectory = null; for (int i = 0; i < args.Length; i++) { switch (args[i].ToLowerInvariant()) @@ -66,7 +89,7 @@ static void Main(string[] args) } break; case "-config": - Console.WriteLine("Custom configurations are not supported yet."); + Console.WriteLine("-config must be the first argument and must be followed by exactly one configuration path."); return; case "-setup-security": if (i + 1 >= args.Length) @@ -76,6 +99,14 @@ static void Main(string[] args) } SetupSecurity(args[++i]); return; + case "-serverstoragedirectory": + if (i + 1 >= args.Length) + { + Console.WriteLine("Expected a directory after -serverStorageDirectory."); + return; + } + serverStorageDirectory = args[++i]; + break; case "-remote": Console.WriteLine("Remote connections are not supported yet."); return; @@ -101,7 +132,7 @@ static void Main(string[] args) Console.WriteLine("Remote TCP RunServers require -security DIRECTORY."); return; } - RunTcpServer(tcpAddress, tcpPort, securityDirectory, dllsToLoad); + RunTcpServer(tcpAddress, tcpPort, securityDirectory, dllsToLoad, serverStorageDirectory); break; case "-namedpipe": if (args.Length == ++i) @@ -122,11 +153,24 @@ static void Main(string[] args) Console.WriteLine("Unable to create a connection to the host!"); return; } - using (var remoteRunRegistry = new RemoteRunRegistry()) + if (serverStorageDirectory is null) { RunClient(serverStream, dllsToLoad, config: null, usePrivateWorkspace: false, remoteEstimationRegistry: null, - remoteRunRegistry: remoteRunRegistry, allowDeployment: false); + remoteRunRegistry: new RemoteRunRegistry(), allowDeployment: false); + } + else + { + var storagePaths = new XTMFStoragePaths(serverStorageDirectory); + Directory.CreateDirectory(storagePaths.DataRoot); + LogNamedPipeConfiguration(storagePaths, dllsToLoad); + using var remoteEstimationRegistry = new RemoteSharedEstimationRegistry(); + using var remoteRunRegistry = new RemoteRunRegistry( + storagePaths.RemoteRunDirectory, storagePaths.RunWorkspaceBaseDirectory); + RunClient(serverStream, dllsToLoad, config: null, + usePrivateWorkspace: true, remoteEstimationRegistry: remoteEstimationRegistry, + remoteRunRegistry: remoteRunRegistry, allowDeployment: false, + storagePaths: storagePaths); } } finally @@ -141,6 +185,22 @@ static void Main(string[] args) } } + private static void StartConfiguredServer(string configPath) + { + var fullConfigPath = Path.GetFullPath(configPath); + _originalArguments = ["-config", fullConfigPath]; + try + { + RunTcpServer(RunServerConfiguration.Load(fullConfigPath)); + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException + or System.Text.Json.JsonException or ArgumentException or CryptographicException) + { + Console.Error.WriteLine($"Unable to start RunServer: {exception.Message}"); + Console.Error.Flush(); + } + } + private static void SetupSecurity(string directory) { try @@ -149,7 +209,7 @@ private static void SetupSecurity(string directory) Console.WriteLine($"RunServer security files created in {directory}"); Console.WriteLine($"Certificate fingerprint: {fingerprint}"); Console.WriteLine("Token: read runserver-token.txt and enter it in the XTMF2 GUI."); - Console.WriteLine($"Start with: XTMF2.RunServer -tcp 0.0.0.0 PORT -security {directory}"); + Console.WriteLine("Add the security directory to XTMF2.RunServer.config, then start with: XTMF2.RunServer -config XTMF2.RunServer.config"); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or CryptographicException) { @@ -161,20 +221,39 @@ private static bool IsLoopbackAddress(string address) => string.Equals(address, "localhost", StringComparison.OrdinalIgnoreCase) || (IPAddress.TryParse(address, out var parsed) && IPAddress.IsLoopback(parsed)); - private static void RunTcpServer(string address, int port, string? securityDirectory, List extraDlls) + private static void RunTcpServer(string address, int port, string? securityDirectory, + List extraDlls, string? serverStorageDirectory) + { + var configuration = new RunServerConfiguration + { + ServerStorageDirectory = serverStorageDirectory ?? XTMFStoragePaths.GetDefaultRunServerDataRoot(), + ListenAddress = address, + Port = port, + SecurityDirectory = securityDirectory, + AdditionalAssemblies = extraDlls, + AllowDeployment = securityDirectory is not null, + }; + configuration.Validate(); + RunTcpServer(configuration); + } + + private static void RunTcpServer(RunServerConfiguration configuration) { - LogStartup($"Initializing TCP server on {address}:{port}."); + configuration.Validate(); + var storagePaths = configuration.CreateStoragePaths(); + Directory.CreateDirectory(storagePaths.DataRoot); + LogConfiguration(configuration, storagePaths); + LogStartup($"Initializing TCP server on {configuration.ListenAddress}:{configuration.Port}."); X509Certificate2? certificate = null; string? token = null; - if (securityDirectory is not null) + if (configuration.SecurityDirectory is not null) { - var securityStopwatch = Stopwatch.StartNew(); LogStartup("Loading TCP server security credentials."); try { - certificate = RunServerSecurity.LoadCertificate(securityDirectory); - token = RunServerSecurity.LoadToken(securityDirectory); - LogStartup($"TCP server security credentials loaded in {securityStopwatch.ElapsedMilliseconds} ms."); + certificate = RunServerSecurity.LoadCertificate(configuration.SecurityDirectory); + token = RunServerSecurity.LoadToken(configuration.SecurityDirectory); + LogStartup("TCP server security credentials loaded."); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or CryptographicException) { @@ -183,25 +262,25 @@ private static void RunTcpServer(string address, int port, string? securityDirec } } - var listenerStopwatch = Stopwatch.StartNew(); LogStartup("Binding the TCP listener."); - if (!CreateStreams.CreateTcpListener(address, port, out var listener, out var boundPort, out var error)) + if (!CreateStreams.CreateTcpListener(configuration.ListenAddress, configuration.Port, out var listener, out var boundPort, out var error)) { Console.WriteLine("Error creating TCP RunServer listener\r\n" + error); return; } var tcpListener = listener!; - Console.WriteLine($"RunServer listening on {address}:{boundPort}"); + Console.WriteLine($"RunServer listening on {configuration.ListenAddress}:{boundPort}"); if (certificate is not null) Console.WriteLine($"Certificate fingerprint: {RunServerSecurity.GetFingerprint(certificate)}"); Console.Out.Flush(); - LogStartup($"TCP listener bound in {listenerStopwatch.ElapsedMilliseconds} ms on {address}:{boundPort}."); + LogStartup($"TCP listener bound on {configuration.ListenAddress}:{boundPort}."); SignalDeploymentReady(); using (tcpListener) using (var shutdown = new CancellationTokenSource()) using (var remoteEstimationRegistry = new RemoteSharedEstimationRegistry()) - using (var remoteRunRegistry = new RemoteRunRegistry()) + using (var remoteRunRegistry = new RemoteRunRegistry( + storagePaths.RemoteRunDirectory, storagePaths.RunWorkspaceBaseDirectory)) { Console.CancelKeyPress += (_, eventArgs) => { @@ -210,7 +289,7 @@ private static void RunTcpServer(string address, int port, string? securityDirec tcpListener.Stop(); }; - LogStartup($"READY: accepting TCP connections on {address}:{boundPort}."); + LogStartup($"READY: accepting TCP connections on {configuration.ListenAddress}:{boundPort}."); while (!shutdown.IsCancellationRequested) { TcpClient? client = null; @@ -235,16 +314,17 @@ private static void RunTcpServer(string address, int port, string? securityDirec Console.WriteLine($"RunServer connection attempt from {client.Client.RemoteEndPoint?.ToString() ?? "unknown endpoint"}"); Console.Out.Flush(); var acceptedClient = client; - _ = securityDirectory is null - ? Task.Run(() => RunTcpClientUnsecured(acceptedClient, extraDlls, + _ = configuration.SecurityDirectory is null + ? Task.Run(() => RunTcpClientUnsecured(acceptedClient, configuration, storagePaths, remoteEstimationRegistry, remoteRunRegistry)) - : Task.Run(() => RunTcpClient(acceptedClient, certificate!, token!, extraDlls, + : Task.Run(() => RunTcpClient(acceptedClient, certificate!, token!, configuration, storagePaths, remoteEstimationRegistry, remoteRunRegistry)); } } } - private static void RunTcpClientUnsecured(TcpClient client, List extraDlls, + private static void RunTcpClientUnsecured(TcpClient client, RunServerConfiguration configuration, + XTMFStoragePaths storagePaths, RemoteSharedEstimationRegistry remoteEstimationRegistry, RemoteRunRegistry remoteRunRegistry) { var remoteEndpoint = client.Client.RemoteEndPoint?.ToString() ?? "unknown endpoint"; @@ -255,9 +335,11 @@ private static void RunTcpClientUnsecured(TcpClient client, List extraDl { try { - RunClient(stream, extraDlls, config: null, usePrivateWorkspace: true, + RunClient(stream, configuration.AdditionalAssemblies, config: null, + usePrivateWorkspace: true, remoteEstimationRegistry: remoteEstimationRegistry, - remoteRunRegistry: remoteRunRegistry, allowDeployment: false); + remoteRunRegistry: remoteRunRegistry, allowDeployment: configuration.AllowDeployment, + storagePaths: storagePaths); } catch (Exception ex) { @@ -267,16 +349,16 @@ private static void RunTcpClientUnsecured(TcpClient client, List extraDl } } - private static void RunTcpClient(TcpClient client, X509Certificate2 certificate, string token, List extraDlls, + private static void RunTcpClient(TcpClient client, X509Certificate2 certificate, string token, + RunServerConfiguration configuration, XTMFStoragePaths storagePaths, RemoteSharedEstimationRegistry remoteEstimationRegistry, RemoteRunRegistry remoteRunRegistry) { var remoteEndpoint = client.Client.RemoteEndPoint?.ToString() ?? "unknown endpoint"; - var authenticationStopwatch = Stopwatch.StartNew(); using (client) { if (!CreateStreams.AuthenticateSecureTcpClient(client, certificate, token, out var stream, out var error)) { - Console.WriteLine($"RunServer TLS/token authentication failed for {remoteEndpoint} after {authenticationStopwatch.ElapsedMilliseconds} ms: {error}"); + Console.WriteLine($"RunServer TLS/token authentication failed for {remoteEndpoint}: {error}"); Console.Out.Flush(); return; } @@ -286,9 +368,11 @@ private static void RunTcpClient(TcpClient client, X509Certificate2 certificate, { try { - RunClient(stream, extraDlls, config: null, usePrivateWorkspace: true, + RunClient(stream, configuration.AdditionalAssemblies, config: null, + usePrivateWorkspace: true, remoteEstimationRegistry: remoteEstimationRegistry, - remoteRunRegistry: remoteRunRegistry, allowDeployment: true); + remoteRunRegistry: remoteRunRegistry, allowDeployment: configuration.AllowDeployment, + storagePaths: storagePaths); } catch (Exception ex) { @@ -303,26 +387,27 @@ private static void RunTcpClient(TcpClient client, X509Certificate2 certificate, private static void RunClient(Stream serverStream, List extraDlls, SystemConfiguration? config, bool usePrivateWorkspace, RemoteSharedEstimationRegistry? remoteEstimationRegistry, - RemoteRunRegistry remoteRunRegistry, bool allowDeployment) + RemoteRunRegistry remoteRunRegistry, bool allowDeployment, XTMFStoragePaths? storagePaths = null) { - var runtimeStopwatch = Stopwatch.StartNew(); LogStartup("Initializing the XTMF runtime for a client connection."); - var runtime = XTMFRuntime.CreateRuntime(config); - LogStartup($"XTMF runtime initialized in {runtimeStopwatch.ElapsedMilliseconds} ms."); + var runtime = storagePaths is null + ? XTMFRuntime.CreateRuntime(config) + : XTMFRuntime.CreateRuntime(storagePaths); + LogStartup("XTMF runtime initialized."); var loadedConfig = runtime.SystemConfiguration; foreach (var dll in extraDlls) { - var assemblyStopwatch = Stopwatch.StartNew(); LogStartup($"Loading additional assembly '{Path.GetFileName(dll)}'."); loadedConfig.LoadAssembly(dll); - LogStartup($"Additional assembly loaded in {assemblyStopwatch.ElapsedMilliseconds} ms."); + LogStartup("Additional assembly loaded."); } using var ownedRemoteEstimationRegistry = remoteEstimationRegistry is null ? new RemoteSharedEstimationRegistry() : null; var registry = remoteEstimationRegistry ?? ownedRemoteEstimationRegistry!; using var clientBus = new RunServerBus(serverStream, true, runtime, extraDlls, - System.Diagnostics.Debugger.IsAttached, usePrivateWorkspace, remoteRunRegistry, allowDeployment); + System.Diagnostics.Debugger.IsAttached, usePrivateWorkspace, remoteRunRegistry, allowDeployment, + storagePaths); using var sharedEstimationWorker = clientBus.AttachSharedEstimationWorker(); using var remoteCoordinator = new RemoteSharedEstimationCoordinatorSession(clientBus, registry); clientBus.SetDeploymentGate( @@ -353,10 +438,122 @@ private static void RunClient(Stream serverStream, List extraDlls, Syste private static void LogStartup(string message) { - Console.Error.WriteLine($"[RunServer +{StartupStopwatch.ElapsedMilliseconds} ms] {message}"); + Console.Error.WriteLine($"[RunServer] {message}"); Console.Error.Flush(); } + private static void LogConfiguration(RunServerConfiguration configuration, XTMFStoragePaths storagePaths) + { + var effectiveConfiguration = new RunServerConfiguration + { + Version = configuration.Version, + ServerStorageDirectory = storagePaths.DataRoot, + ListenAddress = configuration.ListenAddress, + Port = configuration.Port, + SecurityDirectory = configuration.SecurityDirectory is null + ? null + : Path.GetFullPath(configuration.SecurityDirectory), + AdditionalAssemblies = configuration.AdditionalAssemblies + .Select(assembly => Path.GetFullPath(assembly)) + .ToList(), + AllowDeployment = configuration.AllowDeployment, + RunWorkspaceBaseDirectory = storagePaths.RunWorkspaceBaseDirectory, + }; + Console.WriteLine("Effective RunServer configuration:"); + Console.WriteLine(JsonSerializer.Serialize(effectiveConfiguration, new JsonSerializerOptions + { + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + })); + } + + private static void LogNamedPipeConfiguration(XTMFStoragePaths storagePaths, List additionalAssemblies) + { + Console.WriteLine("Effective RunServer configuration:"); + Console.WriteLine(JsonSerializer.Serialize(new + { + transport = "namedPipe", + serverStorageDirectory = storagePaths.DataRoot, + runWorkspaceBaseDirectory = storagePaths.RunWorkspaceBaseDirectory, + additionalAssemblies = additionalAssemblies.Select(assembly => Path.GetFullPath(assembly)).ToArray(), + allowDeployment = false, + }, new JsonSerializerOptions { WriteIndented = true })); + } + + private static void ConfigureTimestampedConsole() + { + Console.SetOut(new TimestampedTextWriter(Console.Out)); + Console.SetError(new TimestampedTextWriter(Console.Error)); + } + + private sealed class TimestampedTextWriter(TextWriter inner) : TextWriter + { + private readonly object _sync = new(); + private bool _atLineStart = true; + + public override Encoding Encoding => inner.Encoding; + + public override void Write(char value) + { + lock (_sync) + WriteCore(value); + } + + public override void Write(string? value) + { + if (value is null) + return; + lock (_sync) + { + foreach (var character in value) + WriteCore(character); + } + } + + public override void WriteLine() + { + lock (_sync) + { + foreach (var character in NewLine) + WriteCore(character); + inner.Flush(); + } + } + + public override void WriteLine(string? value) + { + lock (_sync) + { + if (value is not null) + { + foreach (var character in value) + WriteCore(character); + } + foreach (var character in NewLine) + WriteCore(character); + inner.Flush(); + } + } + + public override void Flush() + { + lock (_sync) + inner.Flush(); + } + + private void WriteCore(char value) + { + if (_atLineStart) + { + inner.Write($"[{DateTimeOffset.Now.ToString("yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture)}] "); + _atLineStart = false; + } + inner.Write(value); + if (value == '\n') + _atLineStart = true; + } + } + private static bool RestartWithOriginalArguments(string stagingRoot) { if (!Directory.Exists(stagingRoot) || !File.Exists(Path.Combine(stagingRoot, "deployment.zip"))) diff --git a/src/XTMF2.GUI/App.axaml.cs b/src/XTMF2.GUI/App.axaml.cs index cd1c044..9545178 100644 --- a/src/XTMF2.GUI/App.axaml.cs +++ b/src/XTMF2.GUI/App.axaml.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Threading.Tasks; using XTMF2; +using XTMF2.Configuration; using XTMF2.GUI.Resources; using XTMF2.GUI.Views; @@ -53,7 +54,7 @@ public override void OnFrameworkInitializationCompleted() try { // Create the XTMF Runtime - Runtime = XTMFRuntime.CreateRuntime(); + Runtime = XTMFRuntime.CreateRuntime(new XTMFStoragePaths(Properties.Settings.Default.DataRoot)); } catch (XTMFCodeStyleError codeError) { diff --git a/src/XTMF2.GUI/Properties/Settings.cs b/src/XTMF2.GUI/Properties/Settings.cs index b046fb4..37ca68d 100644 --- a/src/XTMF2.GUI/Properties/Settings.cs +++ b/src/XTMF2.GUI/Properties/Settings.cs @@ -21,6 +21,7 @@ You should have received a copy of the GNU General Public License using System.IO; using System.Linq; using System.Text.Json; +using XTMF2.Configuration; namespace XTMF2.GUI.Properties; @@ -80,6 +81,7 @@ public Settings() public string AiModel { get; set; } = "llama3.2"; public string OllamaEndpoint { get; set; } = "http://localhost:11434"; public int AiMaxCompactionCycles { get; set; } = 100; + public string DataRoot { get; set; } = XTMFStoragePaths.GetDefaultDesktopDataRoot(); public List RunServers { get; set; } = new() { RunServerEndpoint.CreateLocal() }; public void Save() @@ -127,6 +129,9 @@ private static Settings Load() settings.AiMaxCompactionCycles = loaded.AiMaxCompactionCycles is < 1 or > 100 ? 100 : loaded.AiMaxCompactionCycles; + settings.DataRoot = string.IsNullOrWhiteSpace(loaded.DataRoot) + ? XTMFStoragePaths.GetDefaultDesktopDataRoot() + : loaded.DataRoot; settings.RunServers = NormalizeRunServers(loaded.RunServers); } } diff --git a/src/XTMF2.GUI/RunController.cs b/src/XTMF2.GUI/RunController.cs index 31e2fc6..7620b92 100644 --- a/src/XTMF2.GUI/RunController.cs +++ b/src/XTMF2.GUI/RunController.cs @@ -28,6 +28,7 @@ You should have received a copy of the GNU General Public License using XTMF2.Bus; using XTMF2.Bus.Optimization; using XTMF2.Controllers; +using XTMF2.Configuration; using XTMF2.Editing; using XTMF2.GUI.ViewModels; using XTMF2.GUI.Properties; @@ -157,36 +158,44 @@ private static bool InitializeInSeparateProcess(XTMFRuntime runtime, out RunCont { var xtmfGUIFilePath = typeof(XTMF2.GUI.Program).Assembly.Location; var xtmfClientFileName = Path.Combine(Path.GetDirectoryName(xtmfGUIFilePath)!, "XTMF2.RunServer.dll"); + var pipeName = $"XTMF2-RunServer-{Environment.ProcessId}-{Guid.NewGuid():N}"; Process? client = null; try { var startInfo = new ProcessStartInfo() { FileName = "dotnet", - Arguments = $"\"{xtmfClientFileName}\" -tcp 127.0.0.1 0", UseShellExecute = false, CreateNoWindow = OperatingSystem.IsWindows(), WorkingDirectory = Environment.CurrentDirectory, RedirectStandardOutput = true, RedirectStandardError = true }; + startInfo.ArgumentList.Add(xtmfClientFileName); + startInfo.ArgumentList.Add("-serverStorageDirectory"); + startInfo.ArgumentList.Add(runtime.SystemConfiguration.StoragePaths.DataRoot); + startInfo.ArgumentList.Add("-namedPipe"); + startInfo.ArgumentList.Add(pipeName); client = new() { StartInfo = startInfo, EnableRaisingEvents = true }; - client.ErrorDataReceived += (_, eventArgs) => + client.OutputDataReceived += (_, eventArgs) => { if (eventArgs.Data is not null) Debug.WriteLine($"[Local RunServer] {eventArgs.Data}"); }; - client.Start(); - client.BeginErrorReadLine(); - var listenLine = client.StandardOutput.ReadLine(); - if (!TryParseListeningPort(listenLine, out var hostPort)) + client.ErrorDataReceived += (_, eventArgs) => { - error = "The local RunServer did not report a listening TCP port."; - controller = null; - return false; - } - if (!XTMF2.Bus.CreateStreams.CreateTcpClient("127.0.0.1", hostPort, out var hostStream, out error)) + if (eventArgs.Data is not null) + Debug.WriteLine($"[Local RunServer] {eventArgs.Data}"); + }; + if (!XTMF2.Bus.CreateStreams.CreateNewNamedPipeHost(pipeName, out var hostStream, out error, () => + { + client.Start(); + client.BeginOutputReadLine(); + client.BeginErrorReadLine(); + }) || hostStream is null) { + if (client is { HasExited: false }) + client.Kill(entireProcessTree: true); controller = null; return false; } @@ -820,18 +829,6 @@ private bool SendRun( return true; } - private static bool TryParseListeningPort(string? line, out int port) - { - port = 0; - const string prefix = "RunServer listening on "; - if (line is null || !line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) - return false; - - var endpoint = line[prefix.Length..]; - var separator = endpoint.LastIndexOf(':'); - return separator >= 0 && int.TryParse(endpoint[(separator + 1)..], out port) && port is > 0 and <= 65535; - } - private RunController(XTMFRuntime runtime, HostBus hostBus, Process? runServerProcess = null) { Runtime = runtime; diff --git a/src/XTMF2.GUI/StorageRootMigration.cs b/src/XTMF2.GUI/StorageRootMigration.cs new file mode 100644 index 0000000..14bf687 --- /dev/null +++ b/src/XTMF2.GUI/StorageRootMigration.cs @@ -0,0 +1,248 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; + +namespace XTMF2.GUI; + +internal enum StorageRootMigrationMode +{ + Copy, + Move, +} + +internal sealed record StorageRootMigrationProgress(string Status, double? Percentage); + +internal static class StorageRootMigration +{ + private const int BufferSize = 1024 * 1024; + + public static void Migrate(string sourceRoot, string destinationRoot, StorageRootMigrationMode mode, + IProgress? progress = null, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceRoot); + ArgumentException.ThrowIfNullOrWhiteSpace(destinationRoot); + + var source = Path.GetFullPath(sourceRoot); + var destination = Path.GetFullPath(destinationRoot); + if (!Directory.Exists(source)) + throw new DirectoryNotFoundException($"The current data directory does not exist: {source}"); + if (IsSameOrNestedPath(source, destination) || IsSameOrNestedPath(destination, source)) + throw new InvalidOperationException("The current and new data directories must not overlap."); + + progress?.Report(new StorageRootMigrationProgress("Scanning current data...", null)); + var directories = new List(); + var files = Scan(source, directories, cancellationToken); + var totalWork = files.Aggregate(0L, (total, file) => checked(total + Math.Max(file.Length, 1))); + long completedWork = 0; + + progress?.Report(new StorageRootMigrationProgress("Preparing destination folder structure...", null)); + Directory.CreateDirectory(destination); + foreach (var relativeDirectory in directories.OrderBy(path => path, StringComparer.Ordinal)) + { + cancellationToken.ThrowIfCancellationRequested(); + Directory.CreateDirectory(Path.Combine(destination, relativeDirectory)); + } + + var projectFiles = files.Where(file => IsRootDirectory(file.RelativePath, "Users")).ToArray(); + var runFiles = files.Where(file => IsRootDirectory(file.RelativePath, "Runs") || + IsRootDirectory(file.RelativePath, "RemoteRuns")).ToArray(); + var otherFiles = files.Except(projectFiles).Except(runFiles).ToArray(); + + completedWork = CopyGroup(projectFiles, "Copying projects and model systems...", source, destination, + totalWork, completedWork, progress, cancellationToken); + completedWork = CopyGroup(otherFiles, "Copying other XTMF data...", source, destination, + totalWork, completedWork, progress, cancellationToken); + completedWork = CopyGroup(runFiles, "Copying run data...", source, destination, + totalWork, completedWork, progress, cancellationToken); + + if (mode == StorageRootMigrationMode.Move) + { + progress?.Report(new StorageRootMigrationProgress("Removing copied source data...", null)); + cancellationToken.ThrowIfCancellationRequested(); + Directory.Delete(source, recursive: true); + } + + progress?.Report(new StorageRootMigrationProgress("Migration complete.", 100)); + } + + private static List Scan(string sourceRoot, List directories, + CancellationToken cancellationToken) + { + var files = new List(); + var pendingDirectories = new Stack(); + pendingDirectories.Push(sourceRoot); + + while (pendingDirectories.TryPop(out var currentDirectory)) + { + cancellationToken.ThrowIfCancellationRequested(); + foreach (var entry in Directory.EnumerateFileSystemEntries(currentDirectory)) + { + var attributes = File.GetAttributes(entry); + if ((attributes & FileAttributes.ReparsePoint) != 0) + throw new IOException($"Storage migration does not support symbolic links: {entry}"); + + var relativePath = Path.GetRelativePath(sourceRoot, entry); + if ((attributes & FileAttributes.Directory) != 0) + { + directories.Add(relativePath); + pendingDirectories.Push(entry); + } + else + { + files.Add(new MigrationFile(entry, relativePath, new FileInfo(entry).Length)); + } + } + } + + return files.OrderBy(file => file.RelativePath, StringComparer.Ordinal).ToList(); + } + + private static long CopyGroup(IReadOnlyList files, string status, string sourceRoot, + string destinationRoot, long totalWork, long completedWork, + IProgress? progress, CancellationToken cancellationToken) + { + progress?.Report(new StorageRootMigrationProgress(status, Percentage(completedWork, totalWork))); + var lastReportedPercentage = (int)Percentage(completedWork, totalWork); + foreach (var file in files) + { + cancellationToken.ThrowIfCancellationRequested(); + var destinationPath = Path.Combine(destinationRoot, file.RelativePath); + var copiedForFile = 0L; + CopyFile(file, sourceRoot, destinationRoot, destinationPath, bytesCopied => + { + copiedForFile = bytesCopied; + var percentage = Percentage(completedWork + copiedForFile, totalWork); + if ((int)percentage > lastReportedPercentage) + { + lastReportedPercentage = (int)percentage; + progress?.Report(new StorageRootMigrationProgress(status, percentage)); + } + }, cancellationToken); + + completedWork += Math.Max(file.Length, 1); + var completedPercentage = Percentage(completedWork, totalWork); + if ((int)completedPercentage > lastReportedPercentage) + { + lastReportedPercentage = (int)completedPercentage; + progress?.Report(new StorageRootMigrationProgress(status, completedPercentage)); + } + } + + return completedWork; + } + + private static void CopyFile(MigrationFile file, string sourceRoot, string destinationRoot, + string destinationPath, Action reportBytesCopied, CancellationToken cancellationToken) + { + var destinationDirectory = Path.GetDirectoryName(destinationPath)!; + Directory.CreateDirectory(destinationDirectory); + var temporaryPath = Path.Combine(destinationDirectory, + $".{Path.GetFileName(destinationPath)}.{Guid.NewGuid():N}.xtmf-copy"); + + try + { + var migratedManifest = RewriteRunManifest(file, sourceRoot, destinationRoot); + if (migratedManifest is not null) + { + if (new FileInfo(file.SourcePath).Length != file.Length) + throw new IOException($"The source file changed while it was being copied: {file.SourcePath}"); + using var target = new FileStream(temporaryPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, + BufferSize, FileOptions.SequentialScan); + target.Write(migratedManifest); + target.Flush(flushToDisk: true); + reportBytesCopied(file.Length); + } + else + { + using var source = new FileStream(file.SourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, + BufferSize, FileOptions.SequentialScan); + using var target = new FileStream(temporaryPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, + BufferSize, FileOptions.SequentialScan); + var buffer = new byte[BufferSize]; + long bytesCopied = 0; + int bytesRead; + while ((bytesRead = source.Read(buffer, 0, buffer.Length)) > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + target.Write(buffer, 0, bytesRead); + bytesCopied += bytesRead; + reportBytesCopied(bytesCopied); + } + target.Flush(flushToDisk: true); + + if (bytesCopied != file.Length || source.Length != file.Length) + throw new IOException($"The source file changed while it was being copied: {file.SourcePath}"); + } + + File.Move(temporaryPath, destinationPath, overwrite: true); + } + finally + { + if (File.Exists(temporaryPath)) + File.Delete(temporaryPath); + } + } + + private static byte[]? RewriteRunManifest(MigrationFile file, string sourceRoot, string destinationRoot) + { + if (!IsRootDirectory(file.RelativePath, "RemoteRuns") || + !string.Equals(Path.GetExtension(file.RelativePath), ".json", StringComparison.OrdinalIgnoreCase)) + return null; + + JsonNode? manifest; + try + { + manifest = JsonNode.Parse(File.ReadAllBytes(file.SourcePath)); + } + catch (JsonException) + { + return null; + } + + if (manifest is not JsonObject manifestObject) + return null; + + var workingDirectoryProperty = manifestObject.FirstOrDefault(property => + string.Equals(property.Key, "WorkingDirectory", StringComparison.OrdinalIgnoreCase)); + if (workingDirectoryProperty.Value is not JsonValue workingDirectoryValue || + !workingDirectoryValue.TryGetValue(out var workingDirectory) || + string.IsNullOrWhiteSpace(workingDirectory) || !Path.IsPathFullyQualified(workingDirectory)) + return null; + + var fullWorkingDirectory = Path.GetFullPath(workingDirectory); + var relativeWorkingDirectory = Path.GetRelativePath(sourceRoot, fullWorkingDirectory); + if (!IsRelativePathWithinRoot(relativeWorkingDirectory)) + return null; + + manifestObject[workingDirectoryProperty.Key] = Path.GetFullPath( + Path.Combine(destinationRoot, relativeWorkingDirectory)); + return JsonSerializer.SerializeToUtf8Bytes(manifest); + } + + private static bool IsRootDirectory(string relativePath, string directoryName) + { + var separatorIndex = relativePath.IndexOfAny([Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar]); + var rootDirectory = separatorIndex < 0 ? relativePath : relativePath[..separatorIndex]; + return string.Equals(rootDirectory, directoryName, StringComparison.OrdinalIgnoreCase); + } + + private static bool IsSameOrNestedPath(string root, string candidate) + { + var relative = Path.GetRelativePath(root, candidate); + return relative == "." || IsRelativePathWithinRoot(relative); + } + + private static bool IsRelativePathWithinRoot(string relativePath) + => !Path.IsPathRooted(relativePath) && relativePath != ".." && + !relativePath.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) && + !relativePath.StartsWith($"..{Path.AltDirectorySeparatorChar}", StringComparison.Ordinal); + + private static double Percentage(long completed, long total) + => total == 0 ? 100 : Math.Clamp(completed * 100d / total, 0, 100); + + private sealed record MigrationFile(string SourcePath, string RelativePath, long Length); +} \ No newline at end of file diff --git a/src/XTMF2.GUI/Views/SettingsWindow.axaml b/src/XTMF2.GUI/Views/SettingsWindow.axaml index 9174d6c..8756bcc 100644 --- a/src/XTMF2.GUI/Views/SettingsWindow.axaml +++ b/src/XTMF2.GUI/Views/SettingsWindow.axaml @@ -71,6 +71,56 @@ + + + + + + + +