diff --git a/Compression.Tests/Compression.Tests.csproj b/Compression.Tests/Compression.Tests.csproj index 023dbbe39..d95663c74 100644 --- a/Compression.Tests/Compression.Tests.csproj +++ b/Compression.Tests/Compression.Tests.csproj @@ -146,6 +146,10 @@ ZfsVectors.%(Filename)%(Extension) + + + FlexVectors.%(Filename)%(Extension) + G722Vectors.%(Filename)%(Extension) diff --git a/Compression.Tests/ConversionMatrix/ConversionMatrixTests.cs b/Compression.Tests/ConversionMatrix/ConversionMatrixTests.cs index 664cfe99d..02a88f539 100644 --- a/Compression.Tests/ConversionMatrix/ConversionMatrixTests.cs +++ b/Compression.Tests/ConversionMatrix/ConversionMatrixTests.cs @@ -130,6 +130,7 @@ public static IEnumerable Grid() { { "Bbc", "[name/charset/size constraint] BBC Micro DFS disk; name-synthesizing FS, payload not found by content" }, { "Cpm", "[name/charset/size constraint] CP/M disk; 8.3-folding name-synth FS, payload not found by content" }, { "Lif", "[name/charset/size constraint] HP-71 LIF disk pads file content to a 256-byte record so bytes differ" }, + { "Flex", "[name/charset/size constraint] FLEX keeps no byte length: files are whole 252-byte sector records, zero-padded, so bytes differ" }, { "Creg", "[name/charset/size constraint] a Windows 9x hive's root key has no key-name record, so the matrix payload's root-level files have nowhere to live; CREG refuses them rather than inventing a key" }, { "Ps1MemoryCard", "[name/charset/size constraint] PS1 memory-card saves allocate whole 8 KiB blocks and arbitrary conversion payloads are zero-padded to the stored block size" }, { "Rt11", "[name/charset/size constraint] RT-11 disk pads content to a 512-byte block so bytes differ" }, diff --git a/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs new file mode 100644 index 000000000..bbbc898af --- /dev/null +++ b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs @@ -0,0 +1,37 @@ +using Compression.Registry; +using FileFormat.FirmwareHex; + +namespace Compression.Tests.FirmwareHex; + +[TestFixture] +public sealed class FirmwareHexPayloadTests { + [TestCase(false)] + [TestCase(true)] + public void Create_RejectsMultiplePayloadsWithoutWritingOutput(bool emptyFirst) { + var descriptor = new IntelHexFormatDescriptor(); + using var output = new MemoryStream(); + var inputs = new[] { + ArchiveInputInfo.InMemory("first.bin", emptyFirst ? [] : [1, 2, 3]), + ArchiveInputInfo.InMemory("second.bin", [9, 8, 7]), + }; + + Assert.Throws(() => + ((IArchiveCreatable)descriptor).Create(output, inputs, new FormatCreateOptions())); + Assert.That(output.Length, Is.Zero); + } + + [Test] + public void Add_RejectsAnotherPayloadWithoutChangingTheArchive() { + var descriptor = new IntelHexFormatDescriptor(); + using var archive = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(archive, + [ArchiveInputInfo.InMemory("firmware.bin", [1, 2, 3])], new FormatCreateOptions()); + var original = archive.ToArray(); + archive.Position = 0; + + Assert.Throws(() => ((IArchiveModifiable)descriptor).Add(archive, + [ArchiveInputInfo.InMemory("ADD.BIN", [9, 8, 7])])); + Assert.That(archive.ToArray(), Is.EqualTo(original)); + } + +} diff --git a/Compression.Tests/Flex/FlexExternalConformanceTests.cs b/Compression.Tests/Flex/FlexExternalConformanceTests.cs new file mode 100644 index 000000000..14bab4a84 --- /dev/null +++ b/Compression.Tests/Flex/FlexExternalConformanceTests.cs @@ -0,0 +1,123 @@ +using Compression.Registry; +using FileSystem.Flex; + +namespace Compression.Tests.Flex; + +/// +/// Two-way oracle for FLEX disks with flexfloppy (laurent-fr/flextools, used as a +/// black box): it must catalogue and extract what we create and edit, and we must read +/// what it formats and adds — including into a disk we wrote. +/// +/// +/// Skips unless flexfloppy is found in WSL/Linux, at $CWB_FLEXFLOPPY, on the +/// PATH, or at ~/flex/flextools/flexfloppy/flexfloppy (where +/// Flex/ReferenceVectors/generate.sh expects it). +/// +[TestFixture] +[Category("ExternalFsInterop")] +[Category("Wsl")] +public sealed class FlexExternalConformanceTests { + + private string _tmp = null!; + + [SetUp] + public void SetUp() { + this._tmp = Path.Combine(Path.GetTempPath(), $"cwb_flex_{Guid.NewGuid():N}"); + Directory.CreateDirectory(this._tmp); + } + + [TearDown] + public void TearDown() { + try { Directory.Delete(this._tmp, true); } catch { /* best effort */ } + } + + private static string? _tool; + + private static string Tool() { + if (!FsInteropToolbox.WslAvailable) Assert.Ignore("WSL/Linux shell not available."); + _tool ??= FsInteropToolbox.RunWsl( + "for c in \"$CWB_FLEXFLOPPY\" \"$(command -v flexfloppy)\" \"$HOME/flex/flextools/flexfloppy/flexfloppy\"; do " + + "[ -n \"$c\" ] && [ -x \"$c\" ] && { echo \"$c\"; break; }; done").StdOut.Trim(); + if (_tool.Length == 0) Assert.Ignore("flexfloppy (https://github.com/laurent-fr/flextools) is not installed."); + return _tool; + } + + private static string W(string p) => FsInteropToolbox.WinToWsl(p); + + private (int Code, string Out) Ff(string args) { + var r = FsInteropToolbox.RunWsl($"'{Tool()}' {args} 2>&1"); + return (r.ExitCode, r.StdOut + r.StdErr); + } + + private static byte[] Padded(byte[] data) { + var p = new byte[Math.Max(1, (data.Length + 251) / 252) * 252]; + data.CopyTo(p, 0); + return p; + } + + [Test] + public void GivenOurDisk_WhenFlexfloppyExtractsIt_ThenEveryFileMatches() { + var files = new Dictionary { + ["HELLO.TXT"] = "HELLO\r"u8.ToArray(), + ["ONE.BIN"] = new byte[252], + ["BIG.DAT"] = Enumerable.Range(0, 5000).Select(i => (byte)(i * 13)).ToArray(), + }; + var img = Path.Combine(this._tmp, "ours.dsk"); + using (var fs = File.Create(img)) + new FlexFormatDescriptor().Create(fs, files.Select(kv => ArchiveInputInfo.InMemory(kv.Key, kv.Value)).ToList(), + new FormatCreateOptions { FormatSpecific = FormatCreateOptions.FormatSpecificFrom([new("VolumeLabel", "OURS")]) }); + + var cat = this.Ff($"--in {W(img)} --cat"); + Assert.That(cat.Code, Is.Zero, cat.Out); + Assert.That(cat.Out, Does.Contain("OURS")); + this.AssertFlexfloppyExtracts(img, files.ToDictionary(kv => kv.Key, kv => Padded(kv.Value))); + } + + [Test] + public void GivenFlexfloppyDisk_WhenWeAddAndRemoveAndFlexfloppyAddsMore_ThenBothSidesAgree() { + var img = Path.Combine(this._tmp, "theirs.dsk"); + var src = Path.Combine(this._tmp, "THEIRS.TXT"); + File.WriteAllBytes(src, Enumerable.Range(0, 700).Select(i => (byte)('A' + i % 26)).ToArray()); + Assert.That(this.Ff($"--new --tracks 40 --sectors 18 --label THEIRS --out {W(img)}").Code, Is.Zero); + Assert.That(this.Ff($"--in {W(img)} --add {W(src)}").Code, Is.Zero); + + // We read what flexfloppy wrote … + var disk = new FlexDisk(new MemoryStream(File.ReadAllBytes(img))); + var theirs = disk.Entries.Single(e => e.FullName == "THEIRS.TXT"); + Assert.That(disk.Extract(theirs).AsSpan(0, 700).ToArray(), Is.EqualTo(File.ReadAllBytes(src))); + + // … edit it in place … + var ours = Enumerable.Range(0, 3000).Select(i => (byte)(i ^ 0x5A)).ToArray(); + using (var fs = File.Open(img, FileMode.Open, FileAccess.ReadWrite)) { + var d = new FlexFormatDescriptor(); + d.Add(fs, [ArchiveInputInfo.InMemory("OURS.BIN", ours), ArchiveInputInfo.InMemory("GONE.TXT", [1, 2, 3])]); + d.Remove(fs, ["GONE.TXT"]); + } + + // … and flexfloppy adds into our edit and extracts everything. + var later = Path.Combine(this._tmp, "LATER.TXT"); + File.WriteAllBytes(later, "added by flexfloppy after us\r"u8.ToArray()); + var add = this.Ff($"--in {W(img)} --add {W(later)}"); + Assert.That(add.Code, Is.Zero, add.Out); + var extracted = this.AssertFlexfloppyExtracts(img, new() { ["OURS.BIN"] = Padded(ours) }); + Assert.That(extracted.Keys, Is.EquivalentTo(new[] { "THEIRS.TXT", "OURS.BIN", "LATER.TXT" }), "GONE.TXT stays deleted for flexfloppy"); + + var after = new FlexDisk(new MemoryStream(File.ReadAllBytes(img))); + Assert.That(after.Extract(after.Entries.Single(e => e.FullName == "LATER.TXT")).AsSpan(0, 29).ToArray(), Is.EqualTo(File.ReadAllBytes(later))); + Assert.That(after.Entries.Select(e => e.FullName), Is.EquivalentTo(extracted.Keys)); + } + + /// Runs flexfloppy --extract and checks the given files; returns everything it extracted. + private Dictionary AssertFlexfloppyExtracts(string img, Dictionary expected) { + var outDir = Path.Combine(this._tmp, $"x_{Guid.NewGuid():N}"); + Directory.CreateDirectory(outDir); + var r = this.Ff($"--in {W(img)} --extract {W(outDir)}"); + Assert.That(r.Code, Is.Zero, r.Out); + var got = Directory.GetFiles(outDir).ToDictionary(p => Path.GetFileName(p), File.ReadAllBytes); + foreach (var (n, d) in expected) { + Assert.That(got.ContainsKey(n), Is.True, $"flexfloppy did not extract {n}:\n{r.Out}"); + Assert.That(got[n], Is.EqualTo(d), $"{n} as flexfloppy extracts it"); + } + return got; + } +} diff --git a/Compression.Tests/Flex/FlexTests.cs b/Compression.Tests/Flex/FlexTests.cs new file mode 100644 index 000000000..f8cc0fc3b --- /dev/null +++ b/Compression.Tests/Flex/FlexTests.cs @@ -0,0 +1,223 @@ +using System.Buffers.Binary; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using Compression.Registry; +using FileSystem.Flex; + +namespace Compression.Tests.Flex; + +/// +/// FLEX disks: the reader against a disk flexfloppy formatted and filled (and, +/// when present in test-corpus/flex, the real TSC FLEX 9 distribution disk), the +/// writer's NEWDISK layout against the TSC Adaptation Guide, in-place edits and refusals. +/// The two-way flexfloppy oracle is . +/// +[TestFixture] +public class FlexTests { + + internal static byte[] Resource(string name) { + using var res = typeof(FlexTests).Assembly.GetManifestResourceStream($"FlexVectors.{name}") + ?? throw new InvalidOperationException($"{name} not embedded"); + using var ms = new MemoryStream(); + if (name.EndsWith(".gz", StringComparison.Ordinal)) { + using var gz = new GZipStream(res, CompressionMode.Decompress); + gz.CopyTo(ms); + } else res.CopyTo(ms); + return ms.ToArray(); + } + + internal static Dictionary Manifest(string name) + => Encoding.ASCII.GetString(Resource(name)).Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(l => l.TrimEnd('\r').Split(" ", 2)).ToDictionary(p => p[1], p => p[0]); + + internal static string Sha(byte[] d) => Convert.ToHexStringLower(SHA256.HashData(d)); + + private static Dictionary ReadAll(byte[] img) { + var d = new FlexDisk(new MemoryStream(img, writable: false)); + return d.Entries.ToDictionary(e => e.FullName, d.Extract); + } + + // ── third-party images ──────────────────────────────────────────────── + + [Test] + public void GivenFlexfloppyDisk_WhenRead_ThenEveryFileMatchesWhatFlexfloppyExtracts() { + var files = ReadAll(Resource("flexfloppy.dsk.gz")); + var manifest = Manifest("flexfloppy.manifest"); + Assert.That(files.Keys, Is.EquivalentTo(manifest.Keys)); + foreach (var (n, sha) in manifest) Assert.That(Sha(files[n]), Is.EqualTo(sha), n); + } + + [Test] + public void GivenRealTscFlex9Disk_WhenRead_ThenEveryFileMatchesWhatFlexfloppyExtracts() { + var path = Path.Combine(Documentation.FilesystemReadmeIsCurrentTests.RepositoryRoot(), "test-corpus", "flex", "TSCFLX09.dsk"); + if (!File.Exists(path)) + Assert.Ignore("test-corpus/flex/TSCFLX09.dsk (TSC FLEX 9.1 distribution disk, sha256 cd61d5f3…a5c) is not present; see Flex/ReferenceVectors/README.md."); + var img = File.ReadAllBytes(path); + var d = new FlexDisk(new MemoryStream(img)); + Assert.That((d.Label, d.VolumeNumber, d.Tracks, d.SectorsPerTrack, d.FreeSectors), Is.EqualTo(("FLEX-SWT", (ushort)6809, 35, 10, (ushort)161))); + var manifest = Manifest("TSCFLX09.manifest"); + var files = d.Entries.ToDictionary(e => e.FullName, d.Extract); + Assert.That(files.Keys, Is.EquivalentTo(manifest.Keys)); + foreach (var (n, sha) in manifest) Assert.That(Sha(files[n]), Is.EqualTo(sha), n); + Assert.That(d.Entries.Single(e => e.FullName == "ERRORS.SYS").IsRandom, Is.True, "ERRORS.SYS is a random file"); + Assert.That(d.Entries.Single(e => e.FullName == "FLEX.SYS").Created, Is.EqualTo(new DateTime(1980, 10, 3, 0, 0, 0, DateTimeKind.Utc))); + } + + // ── NEWDISK layout (Adaptation Guide 9.1) ───────────────────────────── + + [Test] + public void GivenEmptyDisk_WhenFormatted_ThenSirDirectoryAndFreeChainFollowNewdisk() { + var img = FlexWriter.Format(35, 10, "VOLUME", 7, new DateTime(1984, 5, 6)); + var sir = img.AsSpan(0x200, 256).ToArray(); + Assert.Multiple(() => { + Assert.That(sir.AsSpan(0, 16).IndexOfAnyExcept((byte)0), Is.EqualTo(-1), "bytes 0-15 zero"); + Assert.That(Encoding.ASCII.GetString(sir.AsSpan(16, 6)), Is.EqualTo("VOLUME")); + Assert.That(BinaryPrimitives.ReadUInt16BigEndian(sir.AsSpan(27)), Is.EqualTo(7), "volume number"); + Assert.That(sir.AsSpan(29, 4).ToArray(), Is.EqualTo(new byte[] { 1, 1, 34, 10 }), "first and last data sector"); + Assert.That(BinaryPrimitives.ReadUInt16BigEndian(sir.AsSpan(33)), Is.EqualTo(340), "34 tracks × 10 data sectors"); + Assert.That(sir.AsSpan(35, 3).ToArray(), Is.EqualTo(new byte[] { 5, 6, 84 }), "date month-day-year"); + Assert.That(sir[38], Is.EqualTo(34), "highest track"); + Assert.That(sir[39], Is.EqualTo(10), "highest sector"); + }); + // Directory sectors 5..10 on track 0 linked, last link zero. + Assert.That(img.AsSpan(4 * 256, 2).ToArray(), Is.EqualTo(new byte[] { 0, 6 })); + Assert.That(img.AsSpan(9 * 256, 2).ToArray(), Is.EqualTo(new byte[] { 0, 0 })); + // Free chain: last sector of track 1 links to track 2 sector 1; the disk's last sector ends it. + Assert.That(img.AsSpan((1 * 10 + 9) * 256, 2).ToArray(), Is.EqualTo(new byte[] { 2, 1 })); + Assert.That(img.AsSpan((34 * 10 + 9) * 256, 2).ToArray(), Is.EqualTo(new byte[] { 0, 0 })); + } + + // ── round trips and edits ───────────────────────────────────────────── + + private static byte[] Padded(byte[] data) { + var sectors = Math.Max(1, (data.Length + 251) / 252); + var p = new byte[sectors * 252]; + data.CopyTo(p, 0); + return p; + } + + [TestCase(0)] + [TestCase(1)] + [TestCase(251)] + [TestCase(252)] + [TestCase(253)] + [TestCase(20_000)] + public void GivenFileOfBoundarySize_WhenWrittenAndRead_ThenBytesComeBackInWholeRecords(int size) { + var data = Enumerable.Range(0, size).Select(i => (byte)(i * 7 + 1)).ToArray(); + var img = FlexWriter.Format(35, 10); + FlexWriter.AddFile(img, "f.bin", data); + Assert.That(ReadAll(img)["F.BIN"], Is.EqualTo(Padded(data))); + } + + [Test] + public void GivenMoreFilesThanTrackZeroHolds_WhenAdded_ThenTheDirectoryGrowsOntoTheFreeChain() { + var img = FlexWriter.Format(35, 10); // 6 directory sectors = 60 entries + for (var i = 0; i < 65; ++i) FlexWriter.AddFile(img, $"F{i}.DAT", [(byte)i]); + var d = new FlexDisk(new MemoryStream(img)); + Assert.That(d.Entries, Has.Count.EqualTo(65)); + Assert.That(d.DirectorySectors, Has.Count.EqualTo(7)); + Assert.That(d.DirectorySectors[^1].Track, Is.Not.Zero); + Assert.That(d.FreeSectors, Is.EqualTo(340 - 65 - 1)); + } + + [Test] + public void GivenFile_WhenRemoved_ThenEntryIsDeletedSectorsZeroedAndReturnedToTheFreeChain() { + var img = FlexWriter.Format(35, 10); + FlexWriter.AddFile(img, "KEEP.TXT", "keep"u8); + FlexWriter.AddFile(img, "DROP.TXT", Encoding.ASCII.GetBytes(new string('W', 600))); + Assert.That(FlexWriter.RemoveFile(img, "drop.txt"), Is.True); + Assert.That(FlexWriter.RemoveFile(img, "drop.txt"), Is.False); + var d = new FlexDisk(new MemoryStream(img)); + Assert.That(d.Entries.Select(e => e.FullName), Is.EqualTo(new[] { "KEEP.TXT" })); + Assert.That(d.FreeSectors, Is.EqualTo(339)); + Assert.That(Encoding.ASCII.GetString(img).Contains("WWWW"), Is.False); + Assert.That(img[0x400 + 16 + 24], Is.EqualTo(0xFF), "deleted entry's first byte"); + // The freed sectors are reused by the next file once the chain head is exhausted: + // the free chain still adds up. + FlexWriter.AddFile(img, "NEW.TXT", new byte[300]); + Assert.That(new FlexDisk(new MemoryStream(img)).FreeSectors, Is.EqualTo(337)); + } + + [Test] + public void GivenFlexfloppyDisk_WhenWeAddAndRemove_ThenItsOwnFilesAreUntouched() { + var img = Resource("flexfloppy.dsk.gz"); + using var ms = new MemoryStream(); + ms.Write(img); + var d = new FlexFormatDescriptor(); + d.Add(ms, [ArchiveInputInfo.InMemory("ADDED.DAT", new byte[1000])]); + d.Remove(ms, ["HELLO.TXT"]); + Assert.That(ms.Length, Is.EqualTo(img.Length)); + var files = ReadAll(ms.ToArray()); + var manifest = Manifest("flexfloppy.manifest"); + Assert.That(files.Keys, Is.EquivalentTo(new[] { "NUMBERS.TXT", "EXACT.BIN", "RANDOM.BIN", "ADDED.DAT" })); + foreach (var n in new[] { "NUMBERS.TXT", "EXACT.BIN", "RANDOM.BIN" }) Assert.That(Sha(files[n]), Is.EqualTo(manifest[n]), n); + } + + [Test] + public void GivenDescriptorCreate_WhenTreeIsGiven_ThenFilesLandByLeafNameAndGeometryFits() { + var d = new FlexFormatDescriptor(); + using var ms = new MemoryStream(); + d.Create(ms, [ArchiveInputInfo.InMemory("HELLO.TXT", "hi"u8.ToArray()), new ArchiveInputInfo("", "SUB/", true), ArchiveInputInfo.InMemory("SUB/INNER.TXT", new byte[100_000])], + new FormatCreateOptions()); + var disk = new FlexDisk(new MemoryStream(ms.ToArray())); + Assert.That(disk.Entries.Select(e => e.FullName), Is.EquivalentTo(new[] { "HELLO.TXT", "INNER.TXT" })); + Assert.That((disk.Tracks, disk.SectorsPerTrack), Is.EqualTo((40, 18)), "100 KB (398 sectors) outgrows 35×10 but fits 40×18"); + } + + // ── refusals ────────────────────────────────────────────────────────── + + [TestCase("NOEXT", TestName = "Flex_NoExtension_Refused")] + [TestCase("1ABC.TXT", TestName = "Flex_NameStartsWithDigit_Refused")] + [TestCase("TOOLONGNAME.TXT", TestName = "Flex_NineCharName_Refused")] + [TestCase("A.TEXT", TestName = "Flex_FourCharExtension_Refused")] + [TestCase("A B.TXT", TestName = "Flex_Space_Refused")] + public void GivenUnstorableName_WhenAdded_ThenRefused(string name) + => Assert.Throws(() => FlexWriter.AddFile(FlexWriter.Format(35, 10), name, [1])); + + [Test] + public void GivenDiskTooFull_WhenAddingABatch_ThenRefusedAndImageUnchanged() { + using var ms = new MemoryStream(); + ms.Write(FlexWriter.Format(35, 10)); + var before = ms.ToArray(); + var d = new FlexFormatDescriptor(); + Assert.Throws(() => d.Add(ms, + [ArchiveInputInfo.InMemory("OK.DAT", [1]), ArchiveInputInfo.InMemory("HUGE.DAT", new byte[341 * 252])])); + Assert.That(ms.ToArray(), Is.EqualTo(before)); + } + + [Test] + public void GivenProtectedFile_WhenRemoved_ThenRefused() { + var img = FlexWriter.Format(35, 10); + FlexWriter.AddFile(img, "LOCK.SYS", [1]); + img[0x400 + 16 + 11] = 0x40; // delete protect + Assert.Throws(() => FlexWriter.RemoveFile(img, "LOCK.SYS")); + } + + [Test] + public void GivenFreeChainDisagreeingWithSir_WhenEdited_ThenRefused() { + var img = FlexWriter.Format(35, 10); + BinaryPrimitives.WriteUInt16BigEndian(img.AsSpan(0x200 + 33), 5); + Assert.Throws(() => FlexWriter.AddFile(img, "A.TXT", [1])); + } + + [Test] + public void GivenChainShorterThanTheDirectorySays_WhenExtracted_ThenRefused() { + var img = FlexWriter.Format(35, 10); + FlexWriter.AddFile(img, "A.TXT", new byte[1000]); + BinaryPrimitives.WriteUInt16BigEndian(img.AsSpan(0x400 + 16 + 17), 9); + var d = new FlexDisk(new MemoryStream(img)); + Assert.Throws(() => d.Extract(d.Entries.Single())); + } + + [Test] + public void GivenNonFlexBytes_WhenOpened_ThenInvalidData() { + var img = FlexWriter.Format(35, 10); + img[0x200] = 1; + Assert.Throws(() => new FlexDisk(new MemoryStream(img))); + } + + [Test] + public void GivenFlexDisk_WhenDetectedByMagic_ThenFlex() + => Assert.That(Compression.Lib.FormatDetector.DetectByMagic(Resource("flexfloppy.dsk.gz")).ToString(), Is.EqualTo("Flex")); +} diff --git a/Compression.Tests/Flex/ReferenceVectors/README.md b/Compression.Tests/Flex/ReferenceVectors/README.md new file mode 100644 index 000000000..5ce2bffeb --- /dev/null +++ b/Compression.Tests/Flex/ReferenceVectors/README.md @@ -0,0 +1,18 @@ +# FLEX reference vectors + +| File | What it is | +| --- | --- | +| `flexfloppy.dsk.gz` | 35 × 10 disk that `flexfloppy` (laurent-fr/flextools, commit `7cb69053`, built with gcc on Ubuntu 24.04) formatted, label `CWBFIXTURE`, volume 42, and filled with four files; recipe `generate.sh`; gzipped only to keep the repository small | +| `flexfloppy.manifest` | `sha256sum` of what the same `flexfloppy --extract` returns for that disk | +| `TSCFLX09.manifest` | `sha256sum` of what `flexfloppy --extract` returns for TSC's FLEX 9.1 distribution disk `TSCFLX09.DSK` | + +The TSC disk is not committed, because the software on it belongs to its publisher. Get it +from the FUFU archive +(; +89,600 bytes, sha256 +`cd61d5f398703822a3c064fcd3db8e004f0dd94736c08a5c24eaaa174ac6da5c`) and put it at +`test-corpus/flex/TSCFLX09.dsk`. That directory is git-ignored; the test that reads the +disk is skipped while it is absent. + +Quirk observed in `flexfloppy`: it stores a 252-byte file in two sectors, so `EXACT.BIN` +extracts as 504 bytes. Our reader reproduces exactly the sectors it wrote. diff --git a/Compression.Tests/Flex/ReferenceVectors/TSCFLX09.manifest b/Compression.Tests/Flex/ReferenceVectors/TSCFLX09.manifest new file mode 100644 index 000000000..a9c80e2bd --- /dev/null +++ b/Compression.Tests/Flex/ReferenceVectors/TSCFLX09.manifest @@ -0,0 +1,30 @@ +3b0b830916d93d5cb775c0455144f8b32b0faeb87fc7c064e6a5350d2b7b01ba APPEND.CMD +ed42276a28073ec14d1df65ef348fb73fc43162c4984493f5603762365c006ed ASMB.CMD +0e0dea13fb57c91c75913515fa9bc12c292fbcd12dc27b37f1f86f1a161577c3 ASN.CMD +284d9bd9984f299d4ee8e3dfb972d6a1ce1cfe47c8eec4e02131dbede8617a3f BUILD.CMD +ae72f404a8cebbe82f3f11ce93f0395346840168e8d45850ea2fda932ced7b37 CAT.CMD +fa2e48db4178eece4541593d94297574e499499f249797bd06e5cb00abc14e6f DATE.CMD +e40224fe5c2394d233e547a8ce261bf22f6449c0fccb7056d101e3494441439e DELETE.CMD +c73abd131ac386d37d90ce94edee24543ab8c6478a0907028a7f5008cdd03c1f EDIT.CMD +a63f7d84f0b495f78d67f13efdbc2ffb7b4779119908b8c2c5cfb6121cfd21d3 ERRORS.SYS +1d5b11374222a81afcafd004471cf29a85c019610cfda4caacce165ff5439795 EXEC.CMD +73f2ddbb54c74c54e92e16d9ef27583ad64932a6cfcb1826a2c34955e8c168cd FLEX.COR +e33e706f058bbc94de18f008345833c05aca5ba80b1601c06d6c02aae4aa9f92 FLEX.SYS +56758c31b702c752e771ec518e981c5f92319b0028d261e0b9f32e37136bf43f I.CMD +9ccdc538fc4db8b02c7997d37f268b9ea0d810cefa261fe36a4ef5e8a1b442b5 JUMP.CMD +47ec6eb9222b0f9e58b131b67c3b984bd3ece5733ec28750776c293aba451f0b LINK.CMD +e5f92a0b9d59cb885d73a5cb812bb9d7ea16a50facb07bc6b390194ec4365989 LIST.CMD +e62eeeffe5b8112cbf95ee74ab88e793635d815811030cc387400711e9bb81bc NEWDISK.CMD +09167c8cdcaac7a0e2ddcd5e0fe827464ecc850707b0f34642afa802ec939c58 O.CMD +9d053a14f9fe0163fa2e6cb61046048eaf634a1afe9b1c7906b697092d92ab5e P.CMD +0c2f0003998cbca9d1b48a505fb1f95705bac8f79f3d1b9b95ccda31462f119b PRINT.CMD +b7bd8b1a41226a4cc30379afa66362547aac03dbcf596ac3662965687709a4b5 PRINT.SYS +bf825c5fea29a4cf91e4d94bd7f38ef6aa612dcccba1f35209d8ac84ab4ba46a PROT.CMD +55d0e473deca9747eba31ab73e7a5be60c9ee4fe8fd537f065bf13c807ac2ab6 QCHECK.CMD +e5fa5afdc842f9bbea19880d51050ad502bd731e5082bd3f04e9d0cc83f07c5e RENAME.CMD +57791ae6d575711c29ceabd4f024f02149d12b3b03469fa733d6f38b9c20b7cf SAVE.CMD +a2c19e01e7195dfe34d69dc56d1d36fea45566faa4d2e0ae34e2f384036d4404 SAVE.LOW +99868920b5a1a521f1ecfdac92b97fccf80fe42cc35d12fb874068b0e20db7e7 TTYSET.CMD +fbbe36ac893a478b7f630734c38c223de1c74a158cff3830c21684fd3015824f VERIFY.CMD +6220b9d08052b47bb68b6b0bc8e33921a16542c8ce9732e174230e46a63f0950 VERSION.CMD +b2da73c6a7dd9c85f0be859a5b04c688d319358f3c444d8ccc6726f581bbcd38 XOUT.CMD diff --git a/Compression.Tests/Flex/ReferenceVectors/flexfloppy.dsk.gz b/Compression.Tests/Flex/ReferenceVectors/flexfloppy.dsk.gz new file mode 100644 index 000000000..386934603 Binary files /dev/null and b/Compression.Tests/Flex/ReferenceVectors/flexfloppy.dsk.gz differ diff --git a/Compression.Tests/Flex/ReferenceVectors/flexfloppy.manifest b/Compression.Tests/Flex/ReferenceVectors/flexfloppy.manifest new file mode 100644 index 000000000..94ce0b872 --- /dev/null +++ b/Compression.Tests/Flex/ReferenceVectors/flexfloppy.manifest @@ -0,0 +1,4 @@ +76bc1665ed746a9d55d5db76058116bb0b8ed5035bc879f1c167326b6792f898 EXACT.BIN +4c321efa2053bcd45e3a19c19ca45437cb1e3725f62c523dca187f7a3b96a3bc HELLO.TXT +64518aa99163ebc10368eb3f1fcd77733e6654859e97df5569a50b0928f4b292 NUMBERS.TXT +4c5e79b26a3d27742b1508afeff5d25e9f4f8167aedc909345d5bf16ddec8867 RANDOM.BIN diff --git a/Compression.Tests/Flex/ReferenceVectors/generate.sh b/Compression.Tests/Flex/ReferenceVectors/generate.sh new file mode 100644 index 000000000..cddcd40a6 --- /dev/null +++ b/Compression.Tests/Flex/ReferenceVectors/generate.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Regenerates flexfloppy.dsk.gz + flexfloppy.manifest: flexfloppy (laurent-fr/flextools) +# formats a 35x10 disk and adds every file; the manifest is the sha256 of what the same +# tool extracts. Usage: generate.sh [path-to-flexfloppy] +set -e +out="${1:?usage: generate.sh [flexfloppy]}"; ff="${2:-$HOME/flex/flextools/flexfloppy/flexfloppy}" +mkdir -p "$out"; d=$(mktemp -d) +$ff --new --tracks 35 --sectors 10 --label CWBFIXTURE --number 42 --out $d/flexfloppy.dsk +mkdir $d/in; cd $d/in +printf 'HELLO FROM FLEXFLOPPY\r' > HELLO.TXT +seq 1 400 | tr '\n' '\r' > NUMBERS.TXT # 1492 bytes: six sectors, the last one partial +head -c 252 /dev/urandom > EXACT.BIN # exactly one sector +head -c 2000 /dev/urandom > RANDOM.BIN +for f in HELLO.TXT NUMBERS.TXT EXACT.BIN RANDOM.BIN; do $ff --in $d/flexfloppy.dsk --add $f; done +$ff --in $d/flexfloppy.dsk --cat +mkdir $d/x; $ff --in $d/flexfloppy.dsk --extract $d/x >/dev/null +(cd $d/x && sha256sum *) > "$out/flexfloppy.manifest" +gzip -9 -n -c $d/flexfloppy.dsk > "$out/flexfloppy.dsk.gz" diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs index 6c7d4b452..1c9e17266 100644 --- a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs @@ -33,6 +33,7 @@ public static class FirmwareHexWriter { /// one the reader rendered. An input list with no payload describes an image /// with no data, which both formats can write. /// + /// More than one non-metadata payload is supplied. public static FirmwareImage ImageFrom(IReadOnlyList inputs, string sourceFormat) { ArgumentNullException.ThrowIfNull(inputs); @@ -53,7 +54,9 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st } // Any other single input is the payload, whatever it is called: a caller // converting a .bin into a .hex should not have to rename it first. - payload ??= input.ReadContent(); + if (payload is not null) + throw new InvalidDataException($"{sourceFormat} represents one logical firmware payload; multiple payload inputs are ambiguous."); + payload = input.ReadContent(); } List<(uint Address, byte[] Data)> segments; diff --git a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Flex/FlexDisk.cs b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Flex/FlexDisk.cs new file mode 100644 index 000000000..4904f7238 --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Flex/FlexDisk.cs @@ -0,0 +1,182 @@ +using System.Buffers.Binary; +using System.Text; + +namespace FileSystem.Flex; + +/// One directory entry of a FLEX disk. +/// File name (up to 8 characters). +/// Extension (up to 3 characters). +/// Protection bits: 0x80 write, 0x40 delete, 0x20 read, 0x10 catalog protect. +/// Track of the first sector. +/// Sector of the first sector. +/// Track of the last sector. +/// Sector of the last sector. +/// Sectors in the file. +/// File Sector Map indicator: a random file whose first two sectors are its sector map. +/// Creation month. +/// Creation day. +/// Creation year, tens and units only. +/// Byte offset of the entry in the image. +public sealed record FlexEntry(string Name, string Extension, byte Attributes, byte StartTrack, byte StartSector, + byte EndTrack, byte EndSector, ushort Sectors, bool IsRandom, byte Month, byte Day, byte Year, long EntryOffset) { + /// NAME.EXT as FLEX shows it. + public string FullName => this.Extension.Length == 0 ? this.Name : this.Name + "." + this.Extension; + + /// Bytes the file holds: every sector's 252 data bytes. + public long Size => (long)this.Sectors * FlexDisk.DataBytes; + + /// Creation date, when it is a valid one (years 00-74 read as 20xx, 75-99 as 19xx). + public DateTime? Created { + get { + if (this.Month is < 1 or > 12 || this.Day is < 1 or > 31 || this.Year > 99) return null; + var year = this.Year < 75 ? 2000 + this.Year : 1900 + this.Year; + return this.Day <= DateTime.DaysInMonth(year, this.Month) ? new DateTime(year, this.Month, this.Day, 0, 0, 0, DateTimeKind.Utc) : null; + } + } +} + +/// +/// The FLEX (Technical Systems Consultants, 6800/6809) disk layout over a raw .dsk +/// image: 256-byte sectors, every track holding the same number of sector slots, sector +/// numbers from 1. Track 0 holds the boot sectors (1, 2), the System Information Record +/// (3), a reserved sector (4) and the directory chain from sector 5; all other sectors are +/// file chains or the free chain. See docs/FLEX-ON-DISK.md. +/// +public sealed class FlexDisk { + /// Bytes per sector. + public const int SectorSize = 256; + + /// Data bytes per sector after the link (2) and record number (2). + public const int DataBytes = 252; + + /// Directory entries per directory sector, after its 16-byte header. + public const int EntriesPerSector = 10; + + /// Bytes per directory entry. + public const int EntrySize = 24; + + /// Byte offset of the System Information Record (track 0, sector 3). + public const int SirOffset = 2 * SectorSize; + + private readonly Stream _stream; + + /// Sectors per track (SIR "highest sector number"). + public int SectorsPerTrack { get; } + + /// Tracks (SIR "highest track number" + 1). + public int Tracks { get; } + + /// Volume label (11 characters). + public string Label { get; } + + /// Volume number. + public ushort VolumeNumber { get; } + + /// Head of the free chain. + public (byte Track, byte Sector) FirstFree { get; } + + /// Tail of the free chain. + public (byte Track, byte Sector) LastFree { get; } + + /// Sectors in the free chain, as the SIR records it. + public ushort FreeSectors { get; } + + /// Every live directory entry, in directory order. + public IReadOnlyList Entries { get; } + + /// Directory sectors in chain order. + public IReadOnlyList<(byte Track, byte Sector)> DirectorySectors { get; } + + /// Opens a FLEX image; throws when it is not one. + public FlexDisk(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + this._stream = stream; + if (stream.Length < 5 * SectorSize) throw new InvalidDataException("Image too short for a FLEX disk (needs track 0 sectors 1-5)."); + var sir = this.ReadAt(SirOffset, SectorSize); + if (sir.AsSpan(0, 16).IndexOfAnyExcept((byte)0) >= 0) + throw new InvalidDataException("Not a FLEX disk: the System Information Record's first 16 bytes are not zero."); + this.Tracks = sir[0x26] + 1; + this.SectorsPerTrack = sir[0x27]; + if (this.SectorsPerTrack < 5 || this.Tracks < 2) + throw new InvalidDataException($"Not a FLEX disk: geometry {this.Tracks} tracks × {this.SectorsPerTrack} sectors from the SIR."); + this.Label = Encoding.ASCII.GetString(sir, 0x10, 11).TrimEnd('\0', ' '); + this.VolumeNumber = BinaryPrimitives.ReadUInt16BigEndian(sir.AsSpan(0x1B)); + this.FirstFree = (sir[0x1D], sir[0x1E]); + this.LastFree = (sir[0x1F], sir[0x20]); + this.FreeSectors = BinaryPrimitives.ReadUInt16BigEndian(sir.AsSpan(0x21)); + + var dirs = new List<(byte, byte)>(); + var entries = new List(); + foreach (var (t, s, sector) in this.Chain(0, 5, limit: this.Tracks * this.SectorsPerTrack)) { + dirs.Add((t, s)); + var at = this.Offset(t, s); + for (var i = 0; i < EntriesPerSector; ++i) { + var e = sector.AsSpan(16 + i * EntrySize, EntrySize); + if (e[0] == 0 || (e[0] & 0x80) != 0) continue; // never used / deleted + entries.Add(new FlexEntry( + Text(e[..8]), Text(e.Slice(8, 3)), e[11], e[13], e[14], e[15], e[16], + BinaryPrimitives.ReadUInt16BigEndian(e[17..]), e[19] != 0, e[21], e[22], e[23], + at + 16 + i * EntrySize)); + } + } + this.DirectorySectors = dirs; + this.Entries = entries; + } + + private static string Text(ReadOnlySpan field) { + var end = field.IndexOf((byte)0); + return Encoding.ASCII.GetString(end < 0 ? field : field[..end]).TrimEnd(' '); + } + + /// Byte offset of a sector; throws when it is outside the geometry or the image. + public long Offset(int track, int sector) { + if (track >= this.Tracks || sector < 1 || sector > this.SectorsPerTrack) + throw new InvalidDataException($"FLEX sector {track},{sector} lies outside the disk's {this.Tracks}×{this.SectorsPerTrack} geometry."); + var at = ((long)track * this.SectorsPerTrack + sector - 1) * SectorSize; + if (at + SectorSize > this._stream.Length) + throw new InvalidDataException($"FLEX sector {track},{sector} lies past the end of the image."); + return at; + } + + /// Reads one sector. + public byte[] ReadSector(int track, int sector) => this.ReadAt(this.Offset(track, sector), SectorSize); + + /// Follows a link chain; throws on a loop or on more than sectors. + public IEnumerable<(byte Track, byte Sector, byte[] Data)> Chain(byte track, byte sector, int limit) { + var seen = new HashSet(); + while (track != 0 || sector != 0) { + if (!seen.Add(track << 8 | sector) || seen.Count > limit) + throw new InvalidDataException($"FLEX sector chain loops or overruns at {track},{sector}."); + var data = this.ReadSector(track, sector); + yield return (track, sector, data); + (track, sector) = (data[0], data[1]); + } + } + + /// + /// The file's bytes: the 252 data bytes of every sector in its chain — a random file's + /// two File Sector Map sectors included, exactly as they are stored. Refuses a chain that + /// does not end at the directory's end address after the directory's sector count. + /// + public byte[] Extract(FlexEntry entry) { + ArgumentNullException.ThrowIfNull(entry); + var result = new byte[entry.Size]; + var n = 0; + (byte T, byte S) last = (0, 0); + foreach (var (t, s, data) in this.Chain(entry.StartTrack, entry.StartSector, entry.Sectors)) { + data.AsSpan(4, DataBytes).CopyTo(result.AsSpan(n * DataBytes)); + ++n; + last = (t, s); + } + if (n != entry.Sectors || last != (entry.EndTrack, entry.EndSector)) + throw new InvalidDataException($"FLEX file {entry.FullName}: its chain holds {n} sectors ending at {last.T},{last.S}, the directory says {entry.Sectors} ending at {entry.EndTrack},{entry.EndSector}."); + return result; + } + + private byte[] ReadAt(long offset, int count) { + var buf = new byte[count]; + this._stream.Position = offset; + this._stream.ReadExactly(buf); + return buf; + } +} diff --git a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Flex/FlexFormatDescriptor.cs b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Flex/FlexFormatDescriptor.cs new file mode 100644 index 000000000..9447466fe --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Flex/FlexFormatDescriptor.cs @@ -0,0 +1,162 @@ +#pragma warning disable CS1591 +using Compression.Registry; +using Compression.Registry.Streaming; +using static Compression.Registry.FormatHelpers; + +namespace FileSystem.Flex; + +/// +/// FLEX disk images (Technical Systems Consultants' FLEX for the 6800/6809, 1976-): raw +/// .dsk sector dumps with 256-byte linked sectors, a System Information Record on +/// track 0 sector 3 and a directory chain from track 0 sector 5. +/// +/// +/// +/// Written from TSC's own manuals — the 6809 FLEX Adaptation Guide (disk format, +/// NEWDISK, SIR layout) and the FLEX Advanced Programmer's Guide (directory entry, +/// data sector, deleted entries, random files); see docs/FLEX-ON-DISK.md. +/// +/// +/// R/W in place: Add takes sectors from the head of the free chain and a free or deleted +/// directory slot (growing the directory chain when full); Remove marks the entry deleted, +/// zeroes the file's sectors and appends them to the free chain. Edits run on a copy and +/// are written back only when every step succeeded. FLEX keeps no byte length, so file +/// contents are whole 252-byte records (padded with zeros on write). Proof: the real TSC +/// FLEX 9 distribution disk reads byte-identical to what flexfloppy (laurent-fr/flextools) +/// extracts, and flexfloppy lists and extracts what we write and edit, while we read +/// what it formats and adds. +/// +/// +public sealed class FlexFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable, IFormatOptionsSchema { + + public IReadOnlyList OptionsSchema { get; } = [ + new FormatOptionDescriptor(Key: "VolumeLabel", DisplayName: "Volume Label", Kind: FormatOptionKind.String, Default: "", + Description: "SIR volume name (11 characters).", IsOptimizationAxis: false), + new FormatOptionDescriptor(Key: "Tracks", DisplayName: "Tracks", Kind: FormatOptionKind.Integer, Default: "0", + Description: "Tracks on the disk (0 = smallest standard geometry that fits: 35×10, 40×18, 77×15, 80×36, 255×255).", IsOptimizationAxis: false), + new FormatOptionDescriptor(Key: "SectorsPerTrack", DisplayName: "Sectors per track", Kind: FormatOptionKind.Integer, Default: "0", + Description: "256-byte sectors per track (0 = with Tracks, the standard geometry).", IsOptimizationAxis: false), + ]; + + public string Id => "Flex"; + public string DisplayName => "FLEX disk (6800/6809)"; + public FormatCategory Category => FormatCategory.Archive; + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | + FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.SupportsMultipleEntries; + + public string DefaultExtension => ".flx"; + public IReadOnlyList Extensions => [".flx", ".dsk"]; + public IReadOnlyList CompoundExtensions => []; + + /// + /// FLEX has no magic number. What every FLEX disk shares: the SIR (track 0 sector 3) + /// starts with 16 zero bytes, and the first directory sector (track 0 sector 5) links to + /// track 0 sector 6 and has twelve zero bytes after its link and record number. + /// + public IReadOnlyList MagicSignatures { get; } = [BuildSignature()]; + + private static MagicSignature BuildSignature() { + var bytes = new byte[0x410 - 0x200]; + var mask = new byte[bytes.Length]; + for (var i = 0; i < 16; ++i) mask[i] = 0xFF; // SIR bytes 0-15 zero + bytes[0x201] = 0x06; // dir sector link: track 0, sector 6 + for (var i = 0x200; i < 0x210; ++i) mask[i] = 0xFF; // link 00 06, then 14 zero bytes + return new(bytes, Offset: 0x200, Confidence: 0.40, Mask: mask); + } + + public IReadOnlyList Methods => [new("stored", "Stored")]; + public string? TarCompressionFormatId => null; + public AlgorithmFamily Family => AlgorithmFamily.Archive; + public string Description => + "FLEX (TSC 6800/6809) disk image: linked 256-byte sectors, SIR and directory on track 0; " + + "in-place R/W checked against flexfloppy and the TSC FLEX 9 distribution disk."; + + public List List(Stream stream, string? password) { + var d = new FlexDisk(stream); + return d.Entries.Select((e, i) => new ArchiveEntryInfo(i, e.FullName, e.Size, e.Size, "Stored", false, false, e.Created)).ToList(); + } + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + var d = new FlexDisk(stream); + foreach (var e in d.Entries) { + if (files != null && !MatchesFilter(e.FullName, files)) continue; + WriteFile(outputDir, e.FullName, d.Extract(e)); + } + } + + public Stream OpenEntry(Stream archive, string entryName, string? password) { + var bytes = this.ExtractEntryToMemory(archive, entryName, password); + return new BoundedEntryStream(new MemoryStream(bytes, writable: false), bytes.Length, leaveOpen: false); + } + + public byte[] ExtractEntryToMemory(Stream archive, string entryName, string? password) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryName); + if (archive.CanSeek) archive.Position = 0; + var d = new FlexDisk(archive); + var e = d.Entries.FirstOrDefault(x => string.Equals(x.FullName, entryName, StringComparison.OrdinalIgnoreCase)) + ?? throw new FileNotFoundException($"FLEX disk has no file '{entryName}'.", entryName); + return d.Extract(e); + } + + /// Formats a disk and writes the files by leaf name (FLEX has one flat directory). + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + ArgumentNullException.ThrowIfNull(options); + var files = inputs.Where(i => !i.IsDirectory) + .Select(i => (Name: Path.GetFileName(i.ArchiveName.Replace('\\', '/').TrimEnd('/')), Data: i.ReadContent(), When: i.Metadata?.LastWriteTimeUtc?.UtcDateTime)) + .ToList(); + foreach (var f in files) FlexWriter.EncodeName(f.Name); // refuse before formatting + var sectors = files.Sum(f => (long)Math.Max(1, (f.Data.Length + FlexDisk.DataBytes - 1) / FlexDisk.DataBytes)); + + var tracks = options.GetOptionInt("Tracks", 0); + var spt = options.GetOptionInt("SectorsPerTrack", 0); + if (tracks == 0 || spt == 0) (tracks, spt) = FlexWriter.ChooseGeometry(sectors, files.Count); + var img = FlexWriter.Format(tracks, spt, options.GetOption("VolumeLabel", "")); + foreach (var f in files) { + if (files.Count(x => string.Equals(x.Name, f.Name, StringComparison.OrdinalIgnoreCase)) > 1) + throw new NotSupportedException($"FLEX cannot hold two files named '{f.Name}'."); + FlexWriter.AddFile(img, f.Name, f.Data, f.When); + } + output.Write(img); + } + + /// Adds or replaces files in place; all inputs land or none do. + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(inputs); + Transact(archive, img => { + foreach (var input in inputs) { + if (input.IsDirectory || input.ArchiveName.Contains('/') || input.ArchiveName.Contains('\\')) + throw new NotSupportedException($"FLEX has no directories; '{input.ArchiveName}' cannot be stored."); + FlexWriter.AddFile(img, input.ArchiveName, input.ReadContent(), input.Metadata?.LastWriteTimeUtc?.UtcDateTime); + } + }); + } + + /// Deletes the named files in place; unknown names are ignored. + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(entryNames); + Transact(archive, img => { + foreach (var n in entryNames) FlexWriter.RemoveFile(img, n); + }); + } + + private static void Transact(Stream archive, Action edit) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanSeek || !archive.CanWrite) throw new NotSupportedException("FLEX edits need a seekable, writable image."); + var original = new byte[archive.Length]; + archive.Position = 0; + archive.ReadExactly(original); + var img = (byte[])original.Clone(); + edit(img); + for (var off = 0; off < img.Length; off += FlexDisk.SectorSize) { + var n = Math.Min(FlexDisk.SectorSize, img.Length - off); + if (img.AsSpan(off, n).SequenceEqual(original.AsSpan(off, n))) continue; + archive.Position = off; + archive.Write(img, off, n); + } + archive.Flush(); + } +} diff --git a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Flex/FlexWriter.cs b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Flex/FlexWriter.cs new file mode 100644 index 000000000..ec326f02a --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Flex/FlexWriter.cs @@ -0,0 +1,205 @@ +using System.Buffers.Binary; +using System.Text; + +namespace FileSystem.Flex; + +/// +/// Formats FLEX disks the way NEWDISK does and edits them the way the File Management +/// System does: files are written by taking sectors from the head of the free chain, +/// deleted files have their name's first byte set to $FF (negative) and their chain appended +/// to the tail of the free chain. The image is edited in memory; callers write it back. +/// +/// +/// FLEX stores whole sectors of 252 data bytes and keeps no byte length, so a file whose +/// length is not a multiple of 252 is padded with zeros — what FLEX itself reads back. A +/// zero-length file takes one zero sector, the least a FLEX file can hold. Names are +/// folded to capitals and must fit FLEX's rule: a letter, then letters, digits, hyphens +/// or underscores, at most 8 characters, with an extension of 1-3 such characters. +/// +public static class FlexWriter { + /// Standard geometries tried, smallest first, when none is given. + public static readonly (int Tracks, int Sectors)[] Geometries = [(35, 10), (40, 18), (77, 15), (80, 36), (255, 255)]; + + /// Formats an empty disk. + public static byte[] Format(int tracks, int sectorsPerTrack, string label = "", ushort volumeNumber = 0, DateTime? date = null) { + if (tracks is < 2 or > 256 || sectorsPerTrack is < 5 or > 255) + throw new NotSupportedException($"FLEX geometry {tracks}×{sectorsPerTrack} is outside 2-256 tracks × 5-255 sectors."); + var img = new byte[(long)tracks * sectorsPerTrack * FlexDisk.SectorSize]; + + // Directory: track 0 sectors 5..last, linked, zeroed, last link 0. + for (var s = 5; s < sectorsPerTrack; ++s) { + var at = Offset(sectorsPerTrack, 0, s); + img[at] = 0; img[at + 1] = (byte)(s + 1); + } + // Free chain: every sector of tracks 1.., in order, last link 0. + for (var t = 1; t < tracks; ++t) + for (var s = 1; s <= sectorsPerTrack; ++s) { + var at = Offset(sectorsPerTrack, t, s); + var (nt, ns) = s < sectorsPerTrack ? (t, s + 1) : (t + 1, 1); + if (nt >= tracks) (nt, ns) = (0, 0); + img[at] = (byte)nt; img[at + 1] = (byte)ns; + } + + var sir = img.AsSpan(FlexDisk.SirOffset, FlexDisk.SectorSize); + var name = Encoding.ASCII.GetBytes(label.ToUpperInvariant()); + name.AsSpan(0, Math.Min(name.Length, 11)).CopyTo(sir[0x10..]); + BinaryPrimitives.WriteUInt16BigEndian(sir[0x1B..], volumeNumber); + sir[0x1D] = 1; sir[0x1E] = 1; + sir[0x1F] = (byte)(tracks - 1); sir[0x20] = (byte)sectorsPerTrack; + BinaryPrimitives.WriteUInt16BigEndian(sir[0x21..], checked((ushort)((tracks - 1) * sectorsPerTrack))); + WriteDate(sir[0x23..], date ?? DateTime.UtcNow); + sir[0x26] = (byte)(tracks - 1); + sir[0x27] = (byte)sectorsPerTrack; + return img; + } + + /// Picks the smallest standard geometry whose data area holds sectors. + public static (int Tracks, int Sectors) ChooseGeometry(long sectorsNeeded, int directoryEntries) { + foreach (var (t, s) in Geometries) { + var dirSlots = (s - 4) * FlexDisk.EntriesPerSector; + var extraDir = directoryEntries > dirSlots ? (directoryEntries - dirSlots + FlexDisk.EntriesPerSector - 1) / FlexDisk.EntriesPerSector : 0; + if ((long)(t - 1) * s >= sectorsNeeded + extraDir && (long)(t - 1) * s <= ushort.MaxValue) return (t, s); + } + throw new NotSupportedException($"FLEX disks hold at most {ushort.MaxValue} data sectors; {sectorsNeeded} are needed."); + } + + /// Encodes a host name as FLEX name and extension, or throws . + public static (byte[] Name, byte[] Ext) EncodeName(string fileName) { + ArgumentNullException.ThrowIfNull(fileName); + var up = fileName.ToUpperInvariant(); + var dot = up.LastIndexOf('.'); + var name = dot < 0 ? up : up[..dot]; + var ext = dot < 0 ? "" : up[(dot + 1)..]; + static bool Valid(string s, int max) => s.Length is > 0 && s.Length <= max && s[0] is >= 'A' and <= 'Z' + && s.All(c => c is >= 'A' and <= 'Z' or >= '0' and <= '9' or '-' or '_'); + if (!Valid(name, 8) || !Valid(ext, 3)) + throw new NotSupportedException($"FLEX names are a letter then letters, digits, '-' or '_' (8 at most) with a 1-3 character extension; '{fileName}' is not."); + var n = new byte[8]; + var e = new byte[3]; + Encoding.ASCII.GetBytes(name, n); + Encoding.ASCII.GetBytes(ext, e); + return (n, e); + } + + /// Adds (or replaces) a file in the image. + public static void AddFile(byte[] img, string fileName, ReadOnlySpan data, DateTime? date = null) { + ArgumentNullException.ThrowIfNull(img); + var (name, ext) = EncodeName(fileName); + var disk = new FlexDisk(new MemoryStream(img, writable: false)); + var existing = disk.Entries.FirstOrDefault(e => Matches(img, e.EntryOffset, name, ext)); + if (existing is not null) { + if ((existing.Attributes & 0xC0) != 0) + throw new NotSupportedException($"FLEX file {existing.FullName} is write- or delete-protected."); + RemoveFile(img, fileName); + disk = new FlexDisk(new MemoryStream(img, writable: false)); + } + + var sectors = Math.Max(1, (data.Length + FlexDisk.DataBytes - 1) / FlexDisk.DataBytes); + var spt = disk.SectorsPerTrack; + var free = FreeChain(img, disk); + var slot = FindSlot(img, disk); + var needDirSector = slot is null ? 1 : 0; + if (free.Count < sectors + needDirSector) + throw new NotSupportedException($"FLEX disk has {free.Count} free sectors; '{fileName}' needs {sectors + needDirSector}."); + + var taken = 0; + if (slot is null) { + // Grow the directory by one sector from the head of the free chain, as FMS does. + var (dt, ds) = free[taken++]; + var dat = Offset(spt, dt, ds); + img.AsSpan((int)dat, FlexDisk.SectorSize).Clear(); + var (lt, ls) = disk.DirectorySectors[^1]; + var lat = Offset(spt, lt, ls); + img[lat] = dt; img[lat + 1] = ds; + slot = dat + 16; + } + + for (var i = 0; i < sectors; ++i) { + var (t, s) = free[taken + i]; + var at = (int)Offset(spt, t, s); + var sector = img.AsSpan(at, FlexDisk.SectorSize); + sector.Clear(); + if (i + 1 < sectors) { sector[0] = free[taken + i + 1].Track; sector[1] = free[taken + i + 1].Sector; } + BinaryPrimitives.WriteUInt16BigEndian(sector[2..], (ushort)(i + 1)); + var start = i * FlexDisk.DataBytes; + if (start < data.Length) data.Slice(start, Math.Min(FlexDisk.DataBytes, data.Length - start)).CopyTo(sector[4..]); + } + var first = free[taken]; + var last = free[taken + sectors - 1]; + taken += sectors; + + var entry = img.AsSpan((int)slot.Value, FlexDisk.EntrySize); + entry.Clear(); + name.CopyTo(entry); + ext.CopyTo(entry[8..]); + entry[13] = first.Track; entry[14] = first.Sector; + entry[15] = last.Track; entry[16] = last.Sector; + BinaryPrimitives.WriteUInt16BigEndian(entry[17..], (ushort)sectors); + WriteDate(entry[21..], date ?? DateTime.UtcNow); + + var sir = img.AsSpan(FlexDisk.SirOffset, FlexDisk.SectorSize); + if (taken < free.Count) { sir[0x1D] = free[taken].Track; sir[0x1E] = free[taken].Sector; } + else { sir[0x1D] = 0; sir[0x1E] = 0; sir[0x1F] = 0; sir[0x20] = 0; } + BinaryPrimitives.WriteUInt16BigEndian(sir[0x21..], (ushort)(free.Count - taken)); + } + + /// Deletes a file: name marked deleted, sectors zeroed and appended to the free chain. False when absent. + public static bool RemoveFile(byte[] img, string fileName) { + ArgumentNullException.ThrowIfNull(img); + var (name, ext) = EncodeName(fileName); + var disk = new FlexDisk(new MemoryStream(img, writable: false)); + var e = disk.Entries.FirstOrDefault(x => Matches(img, x.EntryOffset, name, ext)); + if (e is null) return false; + if ((e.Attributes & 0x40) != 0) throw new NotSupportedException($"FLEX file {e.FullName} is delete-protected."); + _ = disk.Extract(e); // the chain must be sound before it joins the free chain + var spt = disk.SectorsPerTrack; + var chain = disk.Chain(e.StartTrack, e.StartSector, e.Sectors).Select(c => (c.Track, c.Sector)).ToList(); + var free = FreeChain(img, disk); + + for (var i = 0; i < chain.Count; ++i) { + var at = (int)Offset(spt, chain[i].Track, chain[i].Sector); + img.AsSpan(at + 2, FlexDisk.SectorSize - 2).Clear(); // wipe record number and data, keep the link + } + var sir = img.AsSpan(FlexDisk.SirOffset, FlexDisk.SectorSize); + if (free.Count == 0) { sir[0x1D] = e.StartTrack; sir[0x1E] = e.StartSector; } + else { + var tail = (int)Offset(spt, free[^1].Track, free[^1].Sector); + img[tail] = e.StartTrack; img[tail + 1] = e.StartSector; + } + sir[0x1F] = e.EndTrack; sir[0x20] = e.EndSector; + BinaryPrimitives.WriteUInt16BigEndian(sir[0x21..], (ushort)(free.Count + chain.Count)); + img[e.EntryOffset] = 0xFF; // "negative" first byte (FPM); $FF is what real disks show and what flexfloppy skips + return true; + } + + private static bool Matches(byte[] img, long entryOffset, byte[] name, byte[] ext) + => img.AsSpan((int)entryOffset, 8).SequenceEqual(name) && img.AsSpan((int)entryOffset + 8, 3).SequenceEqual(ext); + + /// The free chain, checked against the SIR's head, tail and count; an inconsistent one is refused. + private static List<(byte Track, byte Sector)> FreeChain(byte[] img, FlexDisk disk) { + List<(byte Track, byte Sector)> list = disk.FirstFree == (0, 0) + ? [] + : [.. disk.Chain(disk.FirstFree.Track, disk.FirstFree.Sector, disk.Tracks * disk.SectorsPerTrack).Select(c => (c.Track, c.Sector))]; + if (list.Count != disk.FreeSectors || (list.Count > 0 && list[^1] != disk.LastFree)) + throw new NotSupportedException($"FLEX free chain ({list.Count} sectors) does not match the SIR ({disk.FreeSectors} sectors ending {disk.LastFree.Track},{disk.LastFree.Sector}); the disk is not edited."); + return list; + } + + /// First never-used or deleted directory entry, or null when every directory sector is full. + private static long? FindSlot(byte[] img, FlexDisk disk) { + foreach (var (t, s) in disk.DirectorySectors) { + var at = Offset(disk.SectorsPerTrack, t, s); + for (var i = 0; i < FlexDisk.EntriesPerSector; ++i) { + var e = at + 16 + i * FlexDisk.EntrySize; + if (img[e] == 0 || (img[e] & 0x80) != 0) return e; + } + } + return null; + } + + private static long Offset(int spt, int track, int sector) => ((long)track * spt + sector - 1) * FlexDisk.SectorSize; + + private static void WriteDate(Span d, DateTime t) { + d[0] = (byte)t.Month; d[1] = (byte)t.Day; d[2] = (byte)(t.Year % 100); + } +} diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index 6bee1a597..9e47d17d5 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -192,6 +192,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [D71](https://en.wikipedia.org/wiki/Commodore_DOS) | `D71` | R/W | ✅ | ✅ moving | — | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | — | [VICE disk images](https://vice-emu.sourceforge.io/vice_17.html) | | [D81](https://en.wikipedia.org/wiki/Commodore_DOS) | `D81` | R/W | ✅ | ✅ moving | — | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | — | [VICE disk images](https://vice-emu.sourceforge.io/vice_17.html) | | [DragonFS](https://en.wikipedia.org/wiki/Dragon_32/64) | `DragonFs` | R/W | ✅ | ✅ moving | — | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | DragonDOS | [DragonDOS](https://en.wikipedia.org/wiki/Dragon_32/64) | +| [FLEX disk (6800/6809)](https://en.wikipedia.org/wiki/FLEX_(operating_system)) | `Flex` | R/W | — | — | — | — | — | — | ✅ | [`flexfloppy`](https://github.com/laurent-fr/flextools) catalogues and extracts our created and in-place-edited disks, and we read what it formats and adds; the TSC FLEX 9.1 distribution disk reads byte-identical to `flexfloppy`'s extraction | Raw `.dsk` sector images: linked 256-byte sectors, SIR on track 0 sector 3, directory chain from sector 5 (grows past track 0); files are whole 252-byte records (FLEX keeps no byte length), random files keep their sector map; in-place add/delete as FLEX's FMS does | [6809 FLEX Adaptation Guide](http://flexusergroup.com/flexusergroup/pdfs/6809fadg.pdf), [FLEX Advanced Programmer's Guide](http://www.flexusergroup.com/flexusergroup/pdfs/flexapg.pdf), [on-disk notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/FLEX-ON-DISK.md) | | [G64 (Commodore GCR)](https://en.wikipedia.org/wiki/Commodore_DOS) | `G64` | R/W | ✅ | ✅ rebuild | — | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | Raw GCR tracks; the writer GCR-encodes a 1541 image | [G64 (VICE)](https://vice-emu.sourceforge.io/vice_17.html) | | [GEMDOS (Atari ST)](https://en.wikipedia.org/wiki/Atari_TOS) | `Gemdos` | R/W | ✅ | ✅ rebuild | — | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | Atari ST FAT variant | [GEMDOS](https://en.wikipedia.org/wiki/Atari_TOS) | | [MFS-1 (Acorn Master File System v1)](https://en.wikipedia.org/wiki/BBC_Master) | `Mfs1` | R/W | ✅ | ✅ moving | — | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | Acorn MFS-1; geometry-preserving catalog rebuild for edits | [BBC Master](https://en.wikipedia.org/wiki/BBC_Master) | @@ -284,7 +285,7 @@ ops.Remove(image, ["var/log/old.log"]); -Every public and protected member of all 886 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.FileSystems/REFERENCE.md). +Every public and protected member of all 890 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.FileSystems/REFERENCE.md). diff --git a/Hawkynt.FileFormats.FileSystems/REFERENCE.md b/Hawkynt.FileFormats.FileSystems/REFERENCE.md index 8c41326ac..c54074ea7 100644 --- a/Hawkynt.FileFormats.FileSystems/REFERENCE.md +++ b/Hawkynt.FileFormats.FileSystems/REFERENCE.md @@ -7080,6 +7080,105 @@ Builds Microsoft Xbox / Xbox 360 FATX filesystem images from scratch per the rev | `Build` | `byte[] Build(int sectorsPerCluster = 0, uint volumeId = 0)` | Builds a complete FATX image containing the previously-added files. | | `WriteTo` | `void WriteTo(Stream output, int sectorsPerCluster = 0, uint volumeId = 0)` | Writes the image into `output`: the superblock and FAT, then every cluster payload at its offset. Only a non-seekable target has to materialise the image, so a seekable one is bounded by the disk rather than by what a byte[] can address. | +### Namespace `FileSystem.Flex` + +[`FlexDisk`](#flexdisk) · [`FlexEntry`](#flexentry) · [`FlexFormatDescriptor`](#flexformatdescriptor) · [`FlexWriter`](#flexwriter) + +#### `FlexDisk` + +The FLEX (Technical Systems Consultants, 6800/6809) disk layout over a raw `.dsk` image: 256-byte sectors, every track holding the same number of sector slots, sector numbers from 1. Track 0 holds the boot sectors (1, 2), the System Information Record (3), a reserved sector (4) and the directory chain from sector 5; all other sectors are file chains or the free chain. See `docs/FLEX-ON-DISK.md`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FlexDisk` | `FlexDisk(Stream stream)` | Opens a FLEX image; throws `InvalidDataException` when it is not one. | +| `DataBytes` | `const int DataBytes` | Data bytes per sector after the link (2) and record number (2). | +| `EntriesPerSector` | `const int EntriesPerSector` | Directory entries per directory sector, after its 16-byte header. | +| `EntrySize` | `const int EntrySize` | Bytes per directory entry. | +| `SectorSize` | `const int SectorSize` | Bytes per sector. | +| `SirOffset` | `const int SirOffset` | Byte offset of the System Information Record (track 0, sector 3). | +| `DirectorySectors` | `IReadOnlyList> DirectorySectors { get; }` | Directory sectors in chain order. | +| `Entries` | `IReadOnlyList Entries { get; }` | Every live directory entry, in directory order. | +| `FirstFree` | `ValueTuple FirstFree { get; }` | Head of the free chain. | +| `FreeSectors` | `ushort FreeSectors { get; }` | Sectors in the free chain, as the SIR records it. | +| `Label` | `string Label { get; }` | Volume label (11 characters). | +| `LastFree` | `ValueTuple LastFree { get; }` | Tail of the free chain. | +| `SectorsPerTrack` | `int SectorsPerTrack { get; }` | Sectors per track (SIR "highest sector number"). | +| `Tracks` | `int Tracks { get; }` | Tracks (SIR "highest track number" + 1). | +| `VolumeNumber` | `ushort VolumeNumber { get; }` | Volume number. | +| `Chain` | `IEnumerable> Chain(byte track, byte sector, int limit)` | Follows a link chain; throws on a loop or on more than `limit` sectors. | +| `Extract` | `byte[] Extract(FlexEntry entry)` | The file's bytes: the 252 data bytes of every sector in its chain — a random file's two File Sector Map sectors included, exactly as they are stored. Refuses a chain that does not end at the directory's end address after the directory's sector count. | +| `Offset` | `long Offset(int track, int sector)` | Byte offset of a sector; throws when it is outside the geometry or the image. | +| `ReadSector` | `byte[] ReadSector(int track, int sector)` | Reads one sector. | + +#### `FlexEntry` + +One directory entry of a FLEX disk. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FlexEntry` | `FlexEntry(string Name, string Extension, byte Attributes, byte StartTrack, byte StartSector, byte EndTrack, byte EndSector, ushort Sectors, bool IsRandom, byte Month, byte Day, byte Year, long EntryOffset)` | One directory entry of a FLEX disk. | +| `Attributes` | `byte Attributes { get; init; }` | Protection bits: 0x80 write, 0x40 delete, 0x20 read, 0x10 catalog protect. | +| `Created` | `DateTime? Created { get; }` | Creation date, when it is a valid one (years 00-74 read as 20xx, 75-99 as 19xx). | +| `Day` | `byte Day { get; init; }` | Creation day. | +| `EndSector` | `byte EndSector { get; init; }` | Sector of the last sector. | +| `EndTrack` | `byte EndTrack { get; init; }` | Track of the last sector. | +| `EntryOffset` | `long EntryOffset { get; init; }` | Byte offset of the entry in the image. | +| `Extension` | `string Extension { get; init; }` | Extension (up to 3 characters). | +| `FullName` | `string FullName { get; }` | NAME.EXT as FLEX shows it. | +| `IsRandom` | `bool IsRandom { get; init; }` | File Sector Map indicator: a random file whose first two sectors are its sector map. | +| `Month` | `byte Month { get; init; }` | Creation month. | +| `Name` | `string Name { get; init; }` | File name (up to 8 characters). | +| `Sectors` | `ushort Sectors { get; init; }` | Sectors in the file. | +| `Size` | `long Size { get; }` | Bytes the file holds: every sector's 252 data bytes. | +| `StartSector` | `byte StartSector { get; init; }` | Sector of the first sector. | +| `StartTrack` | `byte StartTrack { get; init; }` | Track of the first sector. | +| `Year` | `byte Year { get; init; }` | Creation year, tens and units only. | + +#### `FlexFormatDescriptor` + +FLEX disk images (Technical Systems Consultants' FLEX for the 6800/6809, 1976-): raw `.dsk` sector dumps with 256-byte linked sectors, a System Information Record on track 0 sector 3 and a directory chain from track 0 sector 5. + +Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`, `IFormatOptionsSchema`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FlexFormatDescriptor` | `FlexFormatDescriptor()` | | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | FLEX has no magic number. What every FLEX disk shares: the SIR (track 0 sector 3) starts with 16 zero bytes, and the first directory sector (track 0 sector 5) links to track 0 sector 6 and has twelve zero bytes after its link and record number. | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `OptionsSchema` | `IReadOnlyList OptionsSchema { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Adds or replaces files in place; all inputs land or none do. | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Formats a disk and writes the files by leaf name (FLEX has one flat directory). | +| `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `OpenEntry` | `Stream OpenEntry(Stream archive, string entryName, string password)` | | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | Deletes the named files in place; unknown names are ignored. | + +#### `FlexWriter` + +Formats FLEX disks the way NEWDISK does and edits them the way the File Management System does: files are written by taking sectors from the head of the free chain, deleted files have their name's first byte set to $FF (negative) and their chain appended to the tail of the free chain. The image is edited in memory; callers write it back. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Geometries` | `static readonly ValueTuple[] Geometries` | Standard geometries tried, smallest first, when none is given. | +| `AddFile` | `static void AddFile(byte[] img, string fileName, ReadOnlySpan data, DateTime? date = null)` | Adds (or replaces) a file in the image. | +| `ChooseGeometry` | `static ValueTuple ChooseGeometry(long sectorsNeeded, int directoryEntries)` | Picks the smallest standard geometry whose data area holds `sectorsNeeded` sectors. | +| `EncodeName` | `static ValueTuple EncodeName(string fileName)` | Encodes a host name as FLEX name and extension, or throws `NotSupportedException`. | +| `Format` | `static byte[] Format(int tracks, int sectorsPerTrack, string label = "", ushort volumeNumber = 0, DateTime? date = null)` | Formats an empty disk. | +| `RemoveFile` | `static bool RemoveFile(byte[] img, string fileName)` | Deletes a file: name marked deleted, sectors zeroed and appended to the free chain. False when absent. | + ### Namespace `FileSystem.Gemdos` [`GemdosBpb`](#gemdosbpb) · [`GemdosExtentMap`](#gemdosextentmap) · [`GemdosFormatDescriptor`](#gemdosformatdescriptor) · [`GemdosInPlaceModifier`](#gemdosinplacemodifier) · [`GemdosOptimizer`](#gemdosoptimizer) · [`GemdosOptimizer.GemdosGeometry`](#gemdosoptimizergemdosgeometry) · [`GemdosReader`](#gemdosreader) · [`GemdosWriter`](#gemdoswriter) diff --git a/docs/FLEX-ON-DISK.md b/docs/FLEX-ON-DISK.md new file mode 100644 index 000000000..c9b7da367 --- /dev/null +++ b/docs/FLEX-ON-DISK.md @@ -0,0 +1,102 @@ +# FLEX disks on disk + +FLEX is the disk operating system Technical Systems Consultants wrote for the 6800 and 6809 +from 1976. Its disks are usually preserved as raw `.dsk` sector dumps. This page is the +specification `FileSystem.Flex` is written from. + +**Sources (rung 3, normative).** Both manuals are TSC's own, from the FLEX User Group archive: +* *6809 FLEX Adaptation Guide*, §2.3 (disk format) and §9.1 (NEWDISK, SIR table): + +* *FLEX Advanced Programmer's Guide*, the FCB directory-information bytes 4–27, + "Description of a directory sector", "Description of a data sector", "Random files" and + "Diskette Initialization": + +`flexfloppy` from laurent-fr/flextools (no licence, so used only as a black-box oracle) +and the TSC FLEX 9.1 distribution disk `TSCFLX09.DSK` (fufu archive, sha256 `cd61d5f3…ac5c`) +confirm the details. + +## Image geometry + +A `.dsk` is a raw dump of 256-byte sectors with no header. Every track has the same number +of sector slots, and sector numbers start at 1: +`offset = (track × sectors_per_track + sector − 1) × 256`. The geometry comes from the SIR. + +## Sectors + +Every sector except the two boot sectors is laid out as: + +| Bytes | Content | +| --- | --- | +| 0–1 | link: track and sector of the next sector in the chain; `00 00` ends it | +| 2–3 | logical record number, big-endian (1, 2, … in a file; 0 in the free chain) | +| 4–255 | 252 data bytes | + +FLEX records no byte length: a file is its sectors' data bytes, whole. + +## Track 0 + +| Sector | Use | +| --- | --- | +| 1–2 | boot loader | +| 3 | System Information Record (SIR) | +| 4 | reserved | +| 5–last | the directory chain, linked; its last sector's link is zero. The directory may grow onto other tracks | + +## SIR (track 0 sector 3; offsets decimal, from the Adaptation Guide §9.1.5) + +| Bytes | Content | +| --- | --- | +| 0–15 | zero (0–1 clear the link) | +| 16–26 | volume name, ASCII | +| 27–28 | volume number, binary (big-endian) | +| 29–30 | first sector of the free chain (track, sector) | +| 31–32 | last sector of the free chain | +| 33–34 | number of sectors in the free chain. NEWDISK writes the total data sectors, which excludes track 0 | +| 35–37 | date: month, day, year (binary) | +| 38 | highest track number | +| 39 | highest sector number (sectors per track) | + +Measured on `TSCFLX09.DSK`: `FLEX-SWT`, volume 6809, 35 tracks × 10 sectors, 161 free. +The free chain's tail is not at the end of the disk: deleted files are appended to the +tail. + +## Directory sector + +Bytes 0–3 hold the link and record number; bytes 4–15 are unused. Ten 24-byte entries +follow, from byte 16. An entry (FCB bytes 4–27): + +| Bytes | Content | +| --- | --- | +| 0–7 | name, NUL-padded; first byte `00` = never used, negative = deleted | +| 8–10 | extension, NUL-padded | +| 11 | attributes: bit 7 write-, 6 delete-, 5 read-, 4 catalog-protect | +| 12 | reserved | +| 13–14 | first sector (track, sector) | +| 15–16 | last sector | +| 17–18 | sector count, big-endian | +| 19 | File Sector Map indicator: non-zero (usually 2) = random file. Its first two sectors are the map, record 0 | +| 20 | reserved | +| 21–23 | creation date: month, day, year (tens and units) | + +Names start with a letter and contain letters, digits, `-` and `_`; FLEX stores them upper +case, and "files with null extensions should not be created". Deleted entries on real disks +carry `$FF` as the first byte, and that is the only marker `flexfloppy` skips. + +## What the implementation does with it + +* **Read**: the SIR, the directory chain (loops refused), and every live entry. File bytes + are the 252 data bytes of every sector in the chain. A random file's two map sectors are + included, as stored, which is what `flexfloppy` extracts. A chain whose length or end + sector disagrees with its entry is refused. +* **Create**: the NEWDISK layout — directory linked through track 0, every data sector in + one free chain, SIR filled in. The geometry is chosen from 35×10, 40×18, 77×15, 80×36 or + 255×255, or given. Files are written as FMS writes them. +* **Edit in place**, FMS style. Add takes sectors from the head of the free chain, uses the + first never-used or deleted directory slot (growing the directory chain by one sector when + none is free), numbers the records from 1, and updates the SIR. Remove refuses a + delete-protected file. Otherwise it sets the entry's first byte to `$FF`, zeroes the + file's record numbers and data, appends its chain to the tail of the free chain, and + updates the SIR. A free chain that disagrees with the SIR's head, tail or count is + refused, with the image unchanged. +* Contents are whole records. A file whose length is not a multiple of 252 comes back + zero-padded, and an empty file takes one sector. diff --git a/docs/catalogue/our-registry.csv b/docs/catalogue/our-registry.csv index bd9da9071..af095241a 100644 --- a/docs/catalogue/our-registry.csv +++ b/docs/catalogue/our-registry.csv @@ -184,6 +184,7 @@ format,Fits,FITS,Archive,,.fits;.fit;.fts,Y,Y,Y, format,Fla,Flash/Animate FLA,Archive,,.fla,Y,Y,Y, format,Flac,FLAC,Stream,,.flac;.fla,Y,Y,Y, format,FlacArchive,FLAC (archive view),Audio,,,Y,Y,, +format,Flex,FLEX disk (6800/6809),Archive,Y,.flx;.dsk,Y,Y,Y,Y format,Fli,FLI/FLC (Autodesk animation),Archive,,.fli;.flc,Y,Y,, format,Flv,Flash Video (FLV),Video,,.flv,Y,Y,Y, format,Fmod,FMOD Sample Bank,Archive,,.fsb,Y,Y,, diff --git a/docs/catalogue/reference-formats.csv b/docs/catalogue/reference-formats.csv index d168a2e19..029ec1a15 100644 --- a/docs/catalogue/reference-formats.csv +++ b/docs/catalogue/reference-formats.csv @@ -407,7 +407,7 @@ FATX (Xbox),fatx;xbox,filesystem,,,,,,,Y,1,,Fatx,Y,Y,Y,OK,, "UFS / FFS (BSD), UFS2",ufs;ufs1,filesystem,,,,,,,Y,1,,Ufs,Y,Y,Y,PARTIAL,,UFS1 only; UFS2 (magic 0x19540119) is not recognised Fossil (Plan 9),,filesystem,,,,,,,Y,1,,,,,,MISSING,, Files-11 (ODS-1/2/5),ods2;ods5;ods-1;ods-2;ods-5;openvms,filesystem,,,,,,,Y,1,,OpenVms;Ods1,Y,Y,Y,OK,, -FLEX file system,,filesystem,,,,,,,Y,1,,,,,,MISSING,, +FLEX file system,,filesystem,,,,,,,Y,1,http://www.flexusergroup.com/flexusergroup/pdfs/flexapg.pdf,Flex,Y,Y,Y,OK,flexfloppy, HAMMER,,filesystem,,,,,,,Y,1,,Hammer,Y,Y,Y,OK,, HAMMER2,,filesystem,,,,,,,Y,1,,Hammer2,Y,Y,Y,OK,, HFS (IBM MVS / z/OS),,filesystem,,,,,,,Y,1,,,,,,MISSING,,