+ PowerISO DAA images (format 0x100)

.github/workflows/ci.yml

@@ -243,6 +243,20 @@ jobs:
           --filter "FullyQualifiedName~Compression.Tests.Reiser4.Reiser4ProfileExternalTests|FullyQualifiedName~Compression.Tests.Reiser4.Reiser4MutationExternalTests|FullyQualifiedName~Compression.Tests.Reiser4.Reiser4NativeTreeExternalTests"
           --logger "trx;LogFileName=reiser4-oracle-tests.trx"
 
+      # Hard gate, both directions: PowerISO for Linux (freeware, checksum-pinned, used as a black
+      # box) converts our DAA images back to identical ISOs, and we read the DAA images it writes.
+      - name: DAA oracle
+        if: matrix.os == 'ubuntu-latest'
+        env:
+          CWB_REQUIRE_POWERISO: "1"
+        run: |
+          curl -sfL -o "$RUNNER_TEMP/poweriso.tgz" https://www.poweriso.com/poweriso-1.3.tar.gz
+          echo "605ae3fceb93952422b42d868f1317d1c6d48d2bdbaee700dcf17e99dbe65fc0  $RUNNER_TEMP/poweriso.tgz" | sha256sum -c -
+          tar xzf "$RUNNER_TEMP/poweriso.tgz" -C "$RUNNER_TEMP"
+          CWB_POWERISO="$RUNNER_TEMP/poweriso" dotnet test Compression.Tests --configuration Release --no-build \
+            --filter "FullyQualifiedName~Compression.Tests.Daa.DaaExternalConformanceTests" \
+            --logger "trx;LogFileName=daa-oracle-tests.trx"
+
       # bcachefs-tools' own checker judges every volume this package writes or
       # edits: the allocation information, the usage totals and the namespace are
       # cross-examined against each other, and bucket generations against the

Compression.Tests/Daa/DaaTests.cs

@@ -0,0 +1,207 @@
+using System.Buffers.Binary;
+using System.Security.Cryptography;
+using Compression.Registry;
+using FileFormat.Daa;
+
+namespace Compression.Tests.Daa;
+
+[TestFixture]
+public class DaaTests {
+  /// <summary>SHA-256 of the ISO PowerISO 1.3 (Linux) was given to produce <c>ReferenceVectors/small.daa</c>.</summary>
+  internal const string SmallIsoSha256 = "806dd11243038abdb3e987694cd7cc951d1789e70cc2282e5aa3045c99bf1497";
+
+  internal static string VectorPath(string name)
+    => Path.Combine(Documentation.FilesystemReadmeIsCurrentTests.RepositoryRoot(), "Compression.Tests", "Daa", "ReferenceVectors", name);
+
+  /// <summary>A real ISO 9660 image (PowerISO parses the filesystem) holding roughly <paramref name="payload"/> bytes.</summary>
+  internal static byte[] RealIso(int payload, int seed) {
+    var iso = new FileSystem.Iso.IsoWriter();
+    var random = new byte[payload / 2];
+    new Random(seed).NextBytes(random);
+    iso.AddFile("RANDOM.BIN", random);
+    iso.AddFile("TEXT.TXT", System.Text.Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat("compressible daa line ", payload / 44 + 1))));
+    return iso.Build();
+  }
+
+  internal static byte[] SampleIso(int size, int seed) {
+    var data = new byte[size];
+    new Random(seed).NextBytes(data.AsSpan(0, size / 3));
+    for (var i = 2 * size / 3; i < size; ++i) data[i] = (byte)"daa chunk text "[i % 15];
+    return data;
+  }
+
+  private static byte[] Write(byte[] iso, int chunk = DaaImage.DefaultChunkSize) {
+    using var ms = new MemoryStream();
+    DaaImage.Write(ms, iso, chunk);
+    return ms.ToArray();
+  }
+
+  private static byte[] Read(byte[] daa) {
+    using var s = new MemoryStream(daa);
+    using var o = new MemoryStream();
+    DaaImage.Read(s).CopyIsoTo(s, o);
+    return o.ToArray();
+  }
+
+  [TestCase(0)]
+  [TestCase(1)]
+  [TestCase(0x10000)]
+  [TestCase(0x10001)]
+  [TestCase(300_000)]
+  public void GivenAnIso_WhenRoundTripped_ThenIdentical(int size) {
+    var iso = SampleIso(size, size);
+    var daa = Write(iso);
+    Assert.That(daa.AsSpan(0, 4).SequenceEqual("DAA\0"u8), Is.True);
+    Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(daa.AsSpan(0x14)), Is.EqualTo(0x100));
+    Assert.That(BinaryPrimitives.ReadInt64LittleEndian(daa.AsSpan(0x30)), Is.EqualTo(daa.Length));
+    Assert.That(Read(daa), Is.EqualTo(iso));
+  }
+
+  [Test]
+  public void GivenThePowerIsoSample_WhenRead_ThenTheIsoMatchesWhatPowerIsoWasGiven() {
+    var daa = File.ReadAllBytes(VectorPath("small.daa"));
+    Assert.That(Convert.ToHexStringLower(SHA256.HashData(Read(daa))), Is.EqualTo(SmallIsoSha256));
+  }
+
+  [Test]
+  public void GivenTheDescriptor_WhenCreatingAndReplacing_ThenTheNewImageReads() {
+    var path = Path.GetTempFileName();
+    try {
+      File.WriteAllBytes(path, SampleIso(100_000, 1));
+      using var ms = new MemoryStream();
+      var d = new DaaFormatDescriptor();
+      d.Create(ms, [new ArchiveInputInfo(path, "image.iso", false)], new FormatCreateOptions());
+      var replacement = SampleIso(200_000, 2);
+      File.WriteAllBytes(path, replacement);
+      d.Add(ms, [new ArchiveInputInfo(path, "image.iso", false)]);
+      Assert.That(Read(ms.ToArray()), Is.EqualTo(replacement));
+    } finally {
+      File.Delete(path);
+    }
+  }
+
+  [Test]
+  public void GivenABadHeaderCrc_WhenRead_ThenInvalidDataAndListFallsBack() {
+    var daa = Write(SampleIso(5000, 3));
+    daa[0x20] ^= 1;
+    Assert.Throws<InvalidDataException>(() => DaaImage.Read(new MemoryStream(daa)));
+    Assert.That(new DaaFormatDescriptor().List(new MemoryStream(daa), null).Single().Name, Is.EqualTo("FULL.daa"));
+  }
+
+  [TestCase(0)]
+  [TestCase(4096)]
+  public void GivenAMandatoryImage_WhenPurged_ThenRefusedWithoutChangingTheArchive(int size) {
+    var iso = SampleIso(size, 11);
+    var original = Write(iso);
+    using var archive = new MemoryStream(original.ToArray());
+    IArchivePurgeable descriptor = new DaaFormatDescriptor();
+
+    Assert.Throws<NotSupportedException>(() => descriptor.Purge(archive));
+    Assert.That(descriptor.CanPurgeToEmpty, Is.False);
+    Assert.That(archive.ToArray(), Is.EqualTo(original));
+    Assert.That(Read(archive.ToArray()), Is.EqualTo(iso));
+  }
+
+  [Test]
+  public void GivenVersion110_WhenRead_ThenNotSupported() {
+    var daa = Write(SampleIso(5000, 4));
+    BinaryPrimitives.WriteUInt32LittleEndian(daa.AsSpan(0x14), 0x110);
+    BinaryPrimitives.WriteUInt32LittleEndian(daa.AsSpan(0x48), Crc(daa.AsSpan(0, 0x48)));
+    Assert.Throws<NotSupportedException>(() => DaaImage.Read(new MemoryStream(daa)));
+  }
+
+  [Test]
+  public void GivenATruncatedFile_WhenRead_ThenNotSupportedAsSplit()
+    => Assert.Throws<NotSupportedException>(() => DaaImage.Read(new MemoryStream(Write(SampleIso(5000, 5))[..^10])));
+
+  [Test]
+  public void GivenACorruptChunk_WhenRead_ThenInvalidData() {
+    var daa = Write(SampleIso(200_000, 6));
+    daa[^20] ^= 0x55;
+    Assert.Catch(() => Read(daa));
+  }
+
+  [Test]
+  public void GivenGbiOrVolumeSignatures_WhenRead_ThenNotSupported() {
+    var daa = Write(SampleIso(100, 7));
+    "GBI\0"u8.CopyTo(daa);
+    Assert.Throws<NotSupportedException>(() => DaaImage.Read(new MemoryStream(daa)));
+    "DAA VOL"u8.CopyTo(daa);
+    Assert.Throws<NotSupportedException>(() => DaaImage.Read(new MemoryStream(daa)));
+  }
+
+  private static uint Crc(ReadOnlySpan<byte> data) {
+    var crc = 0xFFFFFFFFu;
+    foreach (var b in data) {
+      crc ^= b;
+      for (var k = 0; k < 8; ++k) crc = (crc & 1) != 0 ? 0xEDB88320 ^ (crc >> 1) : crc >> 1;
+    }
+    return ~crc;
+  }
+}
+
+/// <summary>
+/// PowerISO for Linux 1.3 (freeware, used as a black box) as the oracle, both ways: it converts our
+/// DAA back to the identical ISO, and we read the DAA it writes. The binary comes from
+/// <c>CWB_POWERISO</c> (a WSL/Linux path); skipped when unset unless <c>CWB_REQUIRE_POWERISO=1</c>.
+/// </summary>
+[TestFixture]
+[Category("ExternalInterop")]
+public class DaaExternalConformanceTests {
+  private static string RequirePowerIso() {
+    var tool = Environment.GetEnvironmentVariable("CWB_POWERISO");
+    if (!string.IsNullOrEmpty(tool) && FsInteropToolbox.WslAvailable)
+      return tool;
+    const string hint = "PowerISO for Linux not configured: set CWB_POWERISO to the poweriso binary (https://www.poweriso.com/poweriso-1.3.tar.gz).";
+    if (Environment.GetEnvironmentVariable("CWB_REQUIRE_POWERISO") == "1") Assert.Fail(hint);
+    Assert.Ignore(hint);
+    return "";
+  }
+
+  private static (string StdOut, string StdErr, int ExitCode) Run(string command) {
+    var r = FsInteropToolbox.RunWsl(command);
+    TestContext.Out.WriteLine($"{command} -> {r.ExitCode}\n{r.StdOut}\n{r.StdErr}");
+    return r;
+  }
+
+  [TestCase(2048)]
+  [TestCase(147 * 2048)]
+  [TestCase(489 * 2048)]
+  public void GivenOurDaa_WhenPowerIsoConvertsItToIso_ThenTheIsoMatches(int size) {
+    var tool = RequirePowerIso();
+    var dir = Path.Combine(Path.GetTempPath(), "cwb-daax-" + Guid.NewGuid().ToString("N"));
+    Directory.CreateDirectory(dir);
+    try {
+      var iso = DaaTests.RealIso(size, size + 3);
+      var daa = Path.Combine(dir, "ours.daa");
+      using (var fs = File.Create(daa)) DaaImage.Write(fs, iso);
+      var outIso = Path.Combine(dir, "theirs.iso");
+      Run($"{tool} convert {FsInteropToolbox.WinToWsl(daa)} -o {FsInteropToolbox.WinToWsl(outIso)} -ot iso");
+      Assert.That(File.Exists(outIso), Is.True, "PowerISO produced no ISO from our DAA");
+      Assert.That(File.ReadAllBytes(outIso), Is.EqualTo(iso));
+    } finally {
+      try { Directory.Delete(dir, true); } catch { /* best effort */ }
+    }
+  }
+
+  [TestCase(4096)]
+  [TestCase(342 * 2048)]
+  public void GivenAPowerIsoDaa_WhenWeReadIt_ThenTheIsoMatches(int size) {
+    var tool = RequirePowerIso();
+    var dir = Path.Combine(Path.GetTempPath(), "cwb-daax-" + Guid.NewGuid().ToString("N"));
+    Directory.CreateDirectory(dir);
+    try {
+      var iso = DaaTests.RealIso(size, size + 4);
+      var isoPath = Path.Combine(dir, "source.iso");
+      File.WriteAllBytes(isoPath, iso);
+      var daa = Path.Combine(dir, "theirs.daa");
+      Run($"{tool} convert {FsInteropToolbox.WinToWsl(isoPath)} -o {FsInteropToolbox.WinToWsl(daa)} -ot daa");
+      using var s = File.OpenRead(daa);
+      using var o = new MemoryStream();
+      DaaImage.Read(s).CopyIsoTo(s, o);
+      Assert.That(o.ToArray(), Is.EqualTo(iso));
+    } finally {
+      try { Directory.Delete(dir, true); } catch { /* best effort */ }
+    }
+  }
+}

Compression.Tests/Daa/ReferenceVectors/README.md

@@ -0,0 +1,14 @@
+# DAA reference vector
+
+`small.daa` (2 237 bytes, SHA-256 `12611771ad4e57ff36677395421f13834f2378ec51761279a3bf15b6292be218`)
+was written by PowerISO for Linux 1.3 (`poweriso-1.3.tar.gz`, SHA-256
+`605ae3fc…65fc0`) with:
+
+    genisoimage -quiet -V DAAFIXTURE -o small.iso -J -R src
+    poweriso convert small.iso -o small.daa -ot daa
+
+Here `src` holds `readme.txt` ("PowerISO DAA reference vector\n") and
+`sub/fox.txt`, which is 150 000 bytes of repeated
+"the quick brown fox jumps over the lazy dog\n".
+`poweriso convert small.daa -ot iso` reproduces `small.iso` byte for byte. Its
+SHA-256 (`806dd112…1497`) is pinned in `DaaTests.SmallIsoSha256`.

Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.Daa/DaaFormatDescriptor.cs

@@ -0,0 +1,102 @@
+#pragma warning disable CS1591
+using System.Globalization;
+using System.Text;
+using Compression.Registry;
+using static Compression.Registry.FormatHelpers;
+
+namespace FileFormat.Daa;
+
+/// <summary>
+/// PowerISO DAA ("Direct Access Archive") compressed CD/DVD image, format version 0x100: exposes
+/// the ISO as <c>image.iso</c> plus a synthetic <c>metadata.ini</c>; creates and rewrites
+/// version 0x100 images PowerISO reads. Version 0x110, split, password-protected and gBurner GBI
+/// images are refused.
+/// </summary>
+/// <remarks>Specification: <c>docs/DAA-ON-DISK.md</c>.</remarks>
+public sealed class DaaFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations,
+    IArchiveCreatable, IArchiveModifiable, ISyntheticEntryNames {
+
+  public const string ImageName = "image.iso";
+  public const string MetadataName = "metadata.ini";
+  private static readonly IReadOnlySet<string> Synthetic = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { MetadataName };
+
+  public string Id => "Daa";
+  public string DisplayName => "DAA (PowerISO)";
+  public FormatCategory Category => FormatCategory.Archive;
+  public FormatCapabilities Capabilities =>
+    FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |
+    FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries;
+  public string DefaultExtension => ".daa";
+  public IReadOnlyList<string> Extensions => [".daa"];
+  public IReadOnlyList<string> CompoundExtensions => [];
+  public IReadOnlyList<MagicSignature> MagicSignatures => [new("DAA\0\0\0\0\0\0\0\0\0\0\0\0\0"u8.ToArray(), Confidence: 0.95)];
+  public IReadOnlyList<FormatMethodInfo> Methods => [new("deflate", "Deflate chunks")];
+  public string? TarCompressionFormatId => null;
+  public AlgorithmFamily Family => AlgorithmFamily.Archive;
+  public string Description => "PowerISO DAA compressed ISO image (format 0x100, deflate chunks)";
+  public IReadOnlySet<string> SyntheticEntryNames => Synthetic;
+
+  /// <summary>DAA requires an ISO image payload, so it has no empty container for purge to leave behind.</summary>
+  public bool CanPurgeToEmpty => false;
+
+  public List<ArchiveEntryInfo> List(Stream stream, string? password) {
+    try {
+      var image = DaaImage.Read(stream);
+      var packed = image.Chunks.Sum(c => (long)c.Length);
+      var metadata = Metadata(image);
+      return [
+        new(0, MetadataName, metadata.Length, metadata.Length, "stored", false, false, null),
+        new(1, ImageName, image.IsoSize, packed, "deflate", false, false, null),
+      ];
+    } catch (Exception e) when (e is InvalidDataException or NotSupportedException) {
+      var length = stream.CanSeek ? stream.Length : 0;
+      return [new(0, "FULL.daa", length, length, "stored", false, false, null)];
+    }
+  }
+
+  public void Extract(Stream stream, string outputDir, string? password, string[]? files) {
+    var image = DaaImage.Read(stream);
+    if (files is not { Length: > 0 } || MatchesFilter(MetadataName, files))
+      WriteFile(outputDir, MetadataName, Metadata(image));
+    if (files is not { Length: > 0 } || MatchesFilter(ImageName, files)) {
+      using var target = CreateEntryFile(outputDir, ImageName);
+      image.CopyIsoTo(stream, target);
+    }
+  }
+
+  public void Create(Stream output, IReadOnlyList<ArchiveInputInfo> inputs, FormatCreateOptions options) {
+    ArgumentNullException.ThrowIfNull(output);
+    DaaImage.Write(output, SingleImage(inputs) ?? throw new ArgumentException("DAA needs the ISO image to store.", nameof(inputs)));
+  }
+
+  /// <summary>Replaces <c>image.iso</c>, keeping the chunk size.</summary>
+  public void Add(Stream archive, IReadOnlyList<ArchiveInputInfo> inputs) {
+    ArgumentNullException.ThrowIfNull(archive);
+    var existing = DaaImage.Read(archive);
+    var replacement = SingleImage(inputs);
+    if (replacement is null) return;
+    WholeImageRebuildCommitter.Replace(archive,
+      candidate => DaaImage.Write(candidate, replacement, (int)existing.ChunkSize),
+      candidate => DaaImage.Read(candidate).CopyIsoTo(candidate, Stream.Null));
+  }
+
+  public void Remove(Stream archive, string[] entryNames) {
+    ArgumentNullException.ThrowIfNull(entryNames);
+    if (entryNames.Length > 0)
+      throw new NotSupportedException("DAA holds exactly one image; it can be replaced but not removed.");
+  }
+
+  private static byte[] Metadata(DaaImage i) => Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture,
+    $"[daa]\nversion=0x{i.Version:X}\nchunk_size={i.ChunkSize}\nchunks={i.Chunks.Count}\niso_size={i.IsoSize}\n"));
+
+  private static byte[]? SingleImage(IReadOnlyList<ArchiveInputInfo> inputs) {
+    byte[]? image = null;
+    foreach (var input in inputs) {
+      if (input.IsDirectory || string.IsNullOrEmpty(input.ArchiveName)) continue;
+      if (Path.GetFileName(input.ArchiveName.Replace('\\', '/')).Equals(MetadataName, StringComparison.OrdinalIgnoreCase)) continue;
+      if (image is not null) throw new ArgumentException("DAA holds exactly one image.", nameof(inputs));
+      image = input.ReadContent();
+    }
+    return image;
+  }
+}