diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d24cac9b9..3331f019d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,6 +243,20 @@ jobs: --filter "FullyQualifiedName~Compression.Tests.Reiser4.Reiser4ProfileExternalTests|FullyQualifiedName~Compression.Tests.Reiser4.Reiser4MutationExternalTests|FullyQualifiedName~Compression.Tests.Reiser4.Reiser4NativeTreeExternalTests" --logger "trx;LogFileName=reiser4-oracle-tests.trx" + # Hard gate, both directions: PowerISO for Linux (freeware, checksum-pinned, used as a black + # box) converts our DAA images back to identical ISOs, and we read the DAA images it writes. + - name: DAA oracle + if: matrix.os == 'ubuntu-latest' + env: + CWB_REQUIRE_POWERISO: "1" + run: | + curl -sfL -o "$RUNNER_TEMP/poweriso.tgz" https://www.poweriso.com/poweriso-1.3.tar.gz + echo "605ae3fceb93952422b42d868f1317d1c6d48d2bdbaee700dcf17e99dbe65fc0 $RUNNER_TEMP/poweriso.tgz" | sha256sum -c - + tar xzf "$RUNNER_TEMP/poweriso.tgz" -C "$RUNNER_TEMP" + CWB_POWERISO="$RUNNER_TEMP/poweriso" dotnet test Compression.Tests --configuration Release --no-build \ + --filter "FullyQualifiedName~Compression.Tests.Daa.DaaExternalConformanceTests" \ + --logger "trx;LogFileName=daa-oracle-tests.trx" + # bcachefs-tools' own checker judges every volume this package writes or # edits: the allocation information, the usage totals and the namespace are # cross-examined against each other, and bucket generations against the diff --git a/Compression.Tests/Daa/DaaTests.cs b/Compression.Tests/Daa/DaaTests.cs new file mode 100644 index 000000000..013ebe885 --- /dev/null +++ b/Compression.Tests/Daa/DaaTests.cs @@ -0,0 +1,207 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; +using Compression.Registry; +using FileFormat.Daa; + +namespace Compression.Tests.Daa; + +[TestFixture] +public class DaaTests { + /// SHA-256 of the ISO PowerISO 1.3 (Linux) was given to produce ReferenceVectors/small.daa. + internal const string SmallIsoSha256 = "806dd11243038abdb3e987694cd7cc951d1789e70cc2282e5aa3045c99bf1497"; + + internal static string VectorPath(string name) + => Path.Combine(Documentation.FilesystemReadmeIsCurrentTests.RepositoryRoot(), "Compression.Tests", "Daa", "ReferenceVectors", name); + + /// A real ISO 9660 image (PowerISO parses the filesystem) holding roughly bytes. + internal static byte[] RealIso(int payload, int seed) { + var iso = new FileSystem.Iso.IsoWriter(); + var random = new byte[payload / 2]; + new Random(seed).NextBytes(random); + iso.AddFile("RANDOM.BIN", random); + iso.AddFile("TEXT.TXT", System.Text.Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat("compressible daa line ", payload / 44 + 1)))); + return iso.Build(); + } + + internal static byte[] SampleIso(int size, int seed) { + var data = new byte[size]; + new Random(seed).NextBytes(data.AsSpan(0, size / 3)); + for (var i = 2 * size / 3; i < size; ++i) data[i] = (byte)"daa chunk text "[i % 15]; + return data; + } + + private static byte[] Write(byte[] iso, int chunk = DaaImage.DefaultChunkSize) { + using var ms = new MemoryStream(); + DaaImage.Write(ms, iso, chunk); + return ms.ToArray(); + } + + private static byte[] Read(byte[] daa) { + using var s = new MemoryStream(daa); + using var o = new MemoryStream(); + DaaImage.Read(s).CopyIsoTo(s, o); + return o.ToArray(); + } + + [TestCase(0)] + [TestCase(1)] + [TestCase(0x10000)] + [TestCase(0x10001)] + [TestCase(300_000)] + public void GivenAnIso_WhenRoundTripped_ThenIdentical(int size) { + var iso = SampleIso(size, size); + var daa = Write(iso); + Assert.That(daa.AsSpan(0, 4).SequenceEqual("DAA\0"u8), Is.True); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(daa.AsSpan(0x14)), Is.EqualTo(0x100)); + Assert.That(BinaryPrimitives.ReadInt64LittleEndian(daa.AsSpan(0x30)), Is.EqualTo(daa.Length)); + Assert.That(Read(daa), Is.EqualTo(iso)); + } + + [Test] + public void GivenThePowerIsoSample_WhenRead_ThenTheIsoMatchesWhatPowerIsoWasGiven() { + var daa = File.ReadAllBytes(VectorPath("small.daa")); + Assert.That(Convert.ToHexStringLower(SHA256.HashData(Read(daa))), Is.EqualTo(SmallIsoSha256)); + } + + [Test] + public void GivenTheDescriptor_WhenCreatingAndReplacing_ThenTheNewImageReads() { + var path = Path.GetTempFileName(); + try { + File.WriteAllBytes(path, SampleIso(100_000, 1)); + using var ms = new MemoryStream(); + var d = new DaaFormatDescriptor(); + d.Create(ms, [new ArchiveInputInfo(path, "image.iso", false)], new FormatCreateOptions()); + var replacement = SampleIso(200_000, 2); + File.WriteAllBytes(path, replacement); + d.Add(ms, [new ArchiveInputInfo(path, "image.iso", false)]); + Assert.That(Read(ms.ToArray()), Is.EqualTo(replacement)); + } finally { + File.Delete(path); + } + } + + [Test] + public void GivenABadHeaderCrc_WhenRead_ThenInvalidDataAndListFallsBack() { + var daa = Write(SampleIso(5000, 3)); + daa[0x20] ^= 1; + Assert.Throws(() => DaaImage.Read(new MemoryStream(daa))); + Assert.That(new DaaFormatDescriptor().List(new MemoryStream(daa), null).Single().Name, Is.EqualTo("FULL.daa")); + } + + [TestCase(0)] + [TestCase(4096)] + public void GivenAMandatoryImage_WhenPurged_ThenRefusedWithoutChangingTheArchive(int size) { + var iso = SampleIso(size, 11); + var original = Write(iso); + using var archive = new MemoryStream(original.ToArray()); + IArchivePurgeable descriptor = new DaaFormatDescriptor(); + + Assert.Throws(() => descriptor.Purge(archive)); + Assert.That(descriptor.CanPurgeToEmpty, Is.False); + Assert.That(archive.ToArray(), Is.EqualTo(original)); + Assert.That(Read(archive.ToArray()), Is.EqualTo(iso)); + } + + [Test] + public void GivenVersion110_WhenRead_ThenNotSupported() { + var daa = Write(SampleIso(5000, 4)); + BinaryPrimitives.WriteUInt32LittleEndian(daa.AsSpan(0x14), 0x110); + BinaryPrimitives.WriteUInt32LittleEndian(daa.AsSpan(0x48), Crc(daa.AsSpan(0, 0x48))); + Assert.Throws(() => DaaImage.Read(new MemoryStream(daa))); + } + + [Test] + public void GivenATruncatedFile_WhenRead_ThenNotSupportedAsSplit() + => Assert.Throws(() => DaaImage.Read(new MemoryStream(Write(SampleIso(5000, 5))[..^10]))); + + [Test] + public void GivenACorruptChunk_WhenRead_ThenInvalidData() { + var daa = Write(SampleIso(200_000, 6)); + daa[^20] ^= 0x55; + Assert.Catch(() => Read(daa)); + } + + [Test] + public void GivenGbiOrVolumeSignatures_WhenRead_ThenNotSupported() { + var daa = Write(SampleIso(100, 7)); + "GBI\0"u8.CopyTo(daa); + Assert.Throws(() => DaaImage.Read(new MemoryStream(daa))); + "DAA VOL"u8.CopyTo(daa); + Assert.Throws(() => DaaImage.Read(new MemoryStream(daa))); + } + + private static uint Crc(ReadOnlySpan data) { + var crc = 0xFFFFFFFFu; + foreach (var b in data) { + crc ^= b; + for (var k = 0; k < 8; ++k) crc = (crc & 1) != 0 ? 0xEDB88320 ^ (crc >> 1) : crc >> 1; + } + return ~crc; + } +} + +/// +/// PowerISO for Linux 1.3 (freeware, used as a black box) as the oracle, both ways: it converts our +/// DAA back to the identical ISO, and we read the DAA it writes. The binary comes from +/// CWB_POWERISO (a WSL/Linux path); skipped when unset unless CWB_REQUIRE_POWERISO=1. +/// +[TestFixture] +[Category("ExternalInterop")] +public class DaaExternalConformanceTests { + private static string RequirePowerIso() { + var tool = Environment.GetEnvironmentVariable("CWB_POWERISO"); + if (!string.IsNullOrEmpty(tool) && FsInteropToolbox.WslAvailable) + return tool; + const string hint = "PowerISO for Linux not configured: set CWB_POWERISO to the poweriso binary (https://www.poweriso.com/poweriso-1.3.tar.gz)."; + if (Environment.GetEnvironmentVariable("CWB_REQUIRE_POWERISO") == "1") Assert.Fail(hint); + Assert.Ignore(hint); + return ""; + } + + private static (string StdOut, string StdErr, int ExitCode) Run(string command) { + var r = FsInteropToolbox.RunWsl(command); + TestContext.Out.WriteLine($"{command} -> {r.ExitCode}\n{r.StdOut}\n{r.StdErr}"); + return r; + } + + [TestCase(2048)] + [TestCase(147 * 2048)] + [TestCase(489 * 2048)] + public void GivenOurDaa_WhenPowerIsoConvertsItToIso_ThenTheIsoMatches(int size) { + var tool = RequirePowerIso(); + var dir = Path.Combine(Path.GetTempPath(), "cwb-daax-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + try { + var iso = DaaTests.RealIso(size, size + 3); + var daa = Path.Combine(dir, "ours.daa"); + using (var fs = File.Create(daa)) DaaImage.Write(fs, iso); + var outIso = Path.Combine(dir, "theirs.iso"); + Run($"{tool} convert {FsInteropToolbox.WinToWsl(daa)} -o {FsInteropToolbox.WinToWsl(outIso)} -ot iso"); + Assert.That(File.Exists(outIso), Is.True, "PowerISO produced no ISO from our DAA"); + Assert.That(File.ReadAllBytes(outIso), Is.EqualTo(iso)); + } finally { + try { Directory.Delete(dir, true); } catch { /* best effort */ } + } + } + + [TestCase(4096)] + [TestCase(342 * 2048)] + public void GivenAPowerIsoDaa_WhenWeReadIt_ThenTheIsoMatches(int size) { + var tool = RequirePowerIso(); + var dir = Path.Combine(Path.GetTempPath(), "cwb-daax-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + try { + var iso = DaaTests.RealIso(size, size + 4); + var isoPath = Path.Combine(dir, "source.iso"); + File.WriteAllBytes(isoPath, iso); + var daa = Path.Combine(dir, "theirs.daa"); + Run($"{tool} convert {FsInteropToolbox.WinToWsl(isoPath)} -o {FsInteropToolbox.WinToWsl(daa)} -ot daa"); + using var s = File.OpenRead(daa); + using var o = new MemoryStream(); + DaaImage.Read(s).CopyIsoTo(s, o); + Assert.That(o.ToArray(), Is.EqualTo(iso)); + } finally { + try { Directory.Delete(dir, true); } catch { /* best effort */ } + } + } +} diff --git a/Compression.Tests/Daa/ReferenceVectors/README.md b/Compression.Tests/Daa/ReferenceVectors/README.md new file mode 100644 index 000000000..604974368 --- /dev/null +++ b/Compression.Tests/Daa/ReferenceVectors/README.md @@ -0,0 +1,14 @@ +# DAA reference vector + +`small.daa` (2 237 bytes, SHA-256 `12611771ad4e57ff36677395421f13834f2378ec51761279a3bf15b6292be218`) +was written by PowerISO for Linux 1.3 (`poweriso-1.3.tar.gz`, SHA-256 +`605ae3fc…65fc0`) with: + + genisoimage -quiet -V DAAFIXTURE -o small.iso -J -R src + poweriso convert small.iso -o small.daa -ot daa + +Here `src` holds `readme.txt` ("PowerISO DAA reference vector\n") and +`sub/fox.txt`, which is 150 000 bytes of repeated +"the quick brown fox jumps over the lazy dog\n". +`poweriso convert small.daa -ot iso` reproduces `small.iso` byte for byte. Its +SHA-256 (`806dd112…1497`) is pinned in `DaaTests.SmallIsoSha256`. diff --git a/Compression.Tests/Daa/ReferenceVectors/small.daa b/Compression.Tests/Daa/ReferenceVectors/small.daa new file mode 100644 index 000000000..e484937a7 Binary files /dev/null and b/Compression.Tests/Daa/ReferenceVectors/small.daa differ diff --git a/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs new file mode 100644 index 000000000..bbbc898af --- /dev/null +++ b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs @@ -0,0 +1,37 @@ +using Compression.Registry; +using FileFormat.FirmwareHex; + +namespace Compression.Tests.FirmwareHex; + +[TestFixture] +public sealed class FirmwareHexPayloadTests { + [TestCase(false)] + [TestCase(true)] + public void Create_RejectsMultiplePayloadsWithoutWritingOutput(bool emptyFirst) { + var descriptor = new IntelHexFormatDescriptor(); + using var output = new MemoryStream(); + var inputs = new[] { + ArchiveInputInfo.InMemory("first.bin", emptyFirst ? [] : [1, 2, 3]), + ArchiveInputInfo.InMemory("second.bin", [9, 8, 7]), + }; + + Assert.Throws(() => + ((IArchiveCreatable)descriptor).Create(output, inputs, new FormatCreateOptions())); + Assert.That(output.Length, Is.Zero); + } + + [Test] + public void Add_RejectsAnotherPayloadWithoutChangingTheArchive() { + var descriptor = new IntelHexFormatDescriptor(); + using var archive = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(archive, + [ArchiveInputInfo.InMemory("firmware.bin", [1, 2, 3])], new FormatCreateOptions()); + var original = archive.ToArray(); + archive.Position = 0; + + Assert.Throws(() => ((IArchiveModifiable)descriptor).Add(archive, + [ArchiveInputInfo.InMemory("ADD.BIN", [9, 8, 7])])); + Assert.That(archive.ToArray(), Is.EqualTo(original)); + } + +} diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.Daa/DaaFormatDescriptor.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.Daa/DaaFormatDescriptor.cs new file mode 100644 index 000000000..be7741c9b --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.Daa/DaaFormatDescriptor.cs @@ -0,0 +1,102 @@ +#pragma warning disable CS1591 +using System.Globalization; +using System.Text; +using Compression.Registry; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.Daa; + +/// +/// PowerISO DAA ("Direct Access Archive") compressed CD/DVD image, format version 0x100: exposes +/// the ISO as image.iso plus a synthetic metadata.ini; creates and rewrites +/// version 0x100 images PowerISO reads. Version 0x110, split, password-protected and gBurner GBI +/// images are refused. +/// +/// Specification: docs/DAA-ON-DISK.md. +public sealed class DaaFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, + IArchiveCreatable, IArchiveModifiable, ISyntheticEntryNames { + + public const string ImageName = "image.iso"; + public const string MetadataName = "metadata.ini"; + private static readonly IReadOnlySet Synthetic = new HashSet(StringComparer.OrdinalIgnoreCase) { MetadataName }; + + public string Id => "Daa"; + public string DisplayName => "DAA (PowerISO)"; + public FormatCategory Category => FormatCategory.Archive; + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | + FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; + public string DefaultExtension => ".daa"; + public IReadOnlyList Extensions => [".daa"]; + public IReadOnlyList CompoundExtensions => []; + public IReadOnlyList MagicSignatures => [new("DAA\0\0\0\0\0\0\0\0\0\0\0\0\0"u8.ToArray(), Confidence: 0.95)]; + public IReadOnlyList Methods => [new("deflate", "Deflate chunks")]; + public string? TarCompressionFormatId => null; + public AlgorithmFamily Family => AlgorithmFamily.Archive; + public string Description => "PowerISO DAA compressed ISO image (format 0x100, deflate chunks)"; + public IReadOnlySet SyntheticEntryNames => Synthetic; + + /// DAA requires an ISO image payload, so it has no empty container for purge to leave behind. + public bool CanPurgeToEmpty => false; + + public List List(Stream stream, string? password) { + try { + var image = DaaImage.Read(stream); + var packed = image.Chunks.Sum(c => (long)c.Length); + var metadata = Metadata(image); + return [ + new(0, MetadataName, metadata.Length, metadata.Length, "stored", false, false, null), + new(1, ImageName, image.IsoSize, packed, "deflate", false, false, null), + ]; + } catch (Exception e) when (e is InvalidDataException or NotSupportedException) { + var length = stream.CanSeek ? stream.Length : 0; + return [new(0, "FULL.daa", length, length, "stored", false, false, null)]; + } + } + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + var image = DaaImage.Read(stream); + if (files is not { Length: > 0 } || MatchesFilter(MetadataName, files)) + WriteFile(outputDir, MetadataName, Metadata(image)); + if (files is not { Length: > 0 } || MatchesFilter(ImageName, files)) { + using var target = CreateEntryFile(outputDir, ImageName); + image.CopyIsoTo(stream, target); + } + } + + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + DaaImage.Write(output, SingleImage(inputs) ?? throw new ArgumentException("DAA needs the ISO image to store.", nameof(inputs))); + } + + /// Replaces image.iso, keeping the chunk size. + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + var existing = DaaImage.Read(archive); + var replacement = SingleImage(inputs); + if (replacement is null) return; + WholeImageRebuildCommitter.Replace(archive, + candidate => DaaImage.Write(candidate, replacement, (int)existing.ChunkSize), + candidate => DaaImage.Read(candidate).CopyIsoTo(candidate, Stream.Null)); + } + + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(entryNames); + if (entryNames.Length > 0) + throw new NotSupportedException("DAA holds exactly one image; it can be replaced but not removed."); + } + + private static byte[] Metadata(DaaImage i) => Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, + $"[daa]\nversion=0x{i.Version:X}\nchunk_size={i.ChunkSize}\nchunks={i.Chunks.Count}\niso_size={i.IsoSize}\n")); + + private static byte[]? SingleImage(IReadOnlyList inputs) { + byte[]? image = null; + foreach (var input in inputs) { + if (input.IsDirectory || string.IsNullOrEmpty(input.ArchiveName)) continue; + if (Path.GetFileName(input.ArchiveName.Replace('\\', '/')).Equals(MetadataName, StringComparison.OrdinalIgnoreCase)) continue; + if (image is not null) throw new ArgumentException("DAA holds exactly one image.", nameof(inputs)); + image = input.ReadContent(); + } + return image; + } +} diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.Daa/DaaImage.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.Daa/DaaImage.cs new file mode 100644 index 000000000..66e2c1140 --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.Daa/DaaImage.cs @@ -0,0 +1,154 @@ +using System.Buffers.Binary; +using Compression.Core.Deflate; + +namespace FileFormat.Daa; + +/// +/// PowerISO DAA image, format version 0x100 (single volume, unencrypted): a 76-byte header, +/// a table of 3-byte compressed chunk lengths and raw-deflate chunks. Implemented from +/// docs/DAA-ON-DISK.md. +/// +public sealed class DaaImage { + public const int HeaderSize = 0x4C; + public const uint Version100 = 0x100; + public const uint Version110 = 0x110; + public const int DefaultChunkSize = 0x10000; + + public uint Version { get; private init; } + public uint ChunkSize { get; private init; } + public long IsoSize { get; private init; } + public IReadOnlyList<(long Offset, int Length)> Chunks { get; private init; } = []; + + /// Not a DAA image, or inconsistent. + /// Version 0x110, split, encrypted or gBurner images. + public static DaaImage Read(Stream stream) { + stream.Position = 0; + var h = new byte[HeaderSize]; + if (stream.ReadAtLeast(h, HeaderSize, throwOnEndOfStream: false) != HeaderSize) + throw new InvalidDataException("DAA: file shorter than the 76-byte header."); + var signature = h.AsSpan(0, 16); + if (signature.StartsWith("DAA VOL"u8) || signature.StartsWith("GBI VOL"u8)) + throw new NotSupportedException("DAA: this is a later volume of a split image; open the first part."); + if (signature.StartsWith("GBI\0"u8)) + throw new NotSupportedException("DAA: gBurner GBI images obfuscate their chunk table and are not supported."); + if (!signature.SequenceEqual("DAA\0\0\0\0\0\0\0\0\0\0\0\0\0"u8)) + throw new InvalidDataException("DAA: \"DAA\" signature missing."); + if (Crc32(h.AsSpan(0, 0x48)) != BinaryPrimitives.ReadUInt32LittleEndian(h.AsSpan(0x48))) + throw new InvalidDataException("DAA: header CRC mismatch."); + + var sizeOffset = BinaryPrimitives.ReadUInt32LittleEndian(h.AsSpan(0x10)); + var version = BinaryPrimitives.ReadUInt32LittleEndian(h.AsSpan(0x14)); + var dataOffset = BinaryPrimitives.ReadUInt32LittleEndian(h.AsSpan(0x18)); + var chunkSize = BinaryPrimitives.ReadUInt32LittleEndian(h.AsSpan(0x24)); + var isoSize = BinaryPrimitives.ReadInt64LittleEndian(h.AsSpan(0x28)); + var daaSize = BinaryPrimitives.ReadInt64LittleEndian(h.AsSpan(0x30)); + if (version == Version110) + throw new NotSupportedException("DAA: format version 0x110 (bit-packed, optionally LZMA chunk table) is not supported yet."); + if (version != Version100) + throw new InvalidDataException($"DAA: unknown format version 0x{version:X}."); + if (daaSize != stream.Length) + throw new NotSupportedException("DAA: the header size differs from the file size — a split (multi-volume) image."); + if (sizeOffset < HeaderSize || dataOffset < sizeOffset || dataOffset > stream.Length || chunkSize == 0 || chunkSize > 1 << 24 || isoSize < 0) + throw new InvalidDataException("DAA: header offsets or sizes are invalid."); + + // Descriptors between the header and the chunk table: type 1 = multi-volume, 3 = password. + var position = (long)HeaderSize; + Span d = stackalloc byte[8]; + while (position < sizeOffset) { + stream.Position = position; + stream.ReadExactly(d); + var type = BinaryPrimitives.ReadUInt32LittleEndian(d); + var length = BinaryPrimitives.ReadUInt32LittleEndian(d[4..]); + if (type == 3) throw new NotSupportedException("DAA: password-protected images are not supported."); + if (type == 1) throw new NotSupportedException("DAA: multi-volume images are not supported."); + if (length < 8) throw new InvalidDataException("DAA: descriptor length is invalid."); + position += length; + } + + var table = new byte[dataOffset - sizeOffset]; + stream.Position = sizeOffset; + stream.ReadExactly(table); + var count = table.Length / 3; + if ((long)count * chunkSize < isoSize) + throw new InvalidDataException("DAA: the chunk table is too short for the ISO size."); + var chunks = new List<(long, int)>(count); + long offset = dataOffset; + for (var i = 0; i < count; ++i) { + var length = table[3 * i] << 16 | table[3 * i + 1] | table[3 * i + 2] << 8; + chunks.Add((offset, length)); + offset += length; + } + if (offset > stream.Length) + throw new InvalidDataException("DAA: chunk data runs past the end of the file."); + return new DaaImage { Version = version, ChunkSize = chunkSize, IsoSize = isoSize, Chunks = chunks }; + } + + public void CopyIsoTo(Stream source, Stream output) { + var written = 0L; + for (var i = 0; i < this.Chunks.Count && written < this.IsoSize; ++i) { + var (offset, length) = this.Chunks[i]; + var packed = new byte[length]; + source.Position = offset; + source.ReadExactly(packed); + if (packed.Length < 4 || Crc32(packed.AsSpan(0, packed.Length - 4)) != BinaryPrimitives.ReadUInt32LittleEndian(packed.AsSpan(packed.Length - 4))) + throw new InvalidDataException($"DAA: chunk {i} fails its CRC-32."); + var data = DeflateDecompressor.Decompress(packed.AsSpan(0, packed.Length - 4)); + var want = (int)Math.Min(this.ChunkSize, this.IsoSize - written); + if (data.Length < want || data.Length > this.ChunkSize) + throw new InvalidDataException($"DAA: chunk {i} decodes to {data.Length} bytes, expected {want}."); + output.Write(data, 0, want); + written += want; + } + if (written != this.IsoSize) + throw new InvalidDataException("DAA: chunks end before the ISO size."); + } + + /// Writes a version 0x100 image: header, 3-byte length table, raw-deflate chunks each followed by its CRC-32. + public static void Write(Stream output, ReadOnlySpan iso, int chunkSize = DefaultChunkSize, DeflateCompressionLevel level = DeflateCompressionLevel.Default) { + if (chunkSize <= 0 || chunkSize > 1 << 24) + throw new ArgumentOutOfRangeException(nameof(chunkSize)); + var count = Math.Max(1, (iso.Length + chunkSize - 1) / chunkSize); + var packed = new List(count); + for (var i = 0; i < count; ++i) { + var slice = iso.Slice(Math.Min(i * chunkSize, iso.Length), Math.Clamp(iso.Length - i * chunkSize, 0, chunkSize)); + var deflated = DeflateCompressor.Compress(slice, level); + var chunk = new byte[deflated.Length + 4]; + deflated.CopyTo(chunk, 0); + BinaryPrimitives.WriteUInt32LittleEndian(chunk.AsSpan(deflated.Length), Crc32(deflated)); + if (chunk.Length > 0xFFFFFF) throw new InvalidDataException("DAA: a compressed chunk exceeds the 24-bit length field."); + packed.Add(chunk); + } + var sizeOffset = HeaderSize; + var dataOffset = sizeOffset + 3 * count; + var total = dataOffset + packed.Sum(c => (long)c.Length); + + var h = new byte[HeaderSize]; + "DAA"u8.CopyTo(h); + BinaryPrimitives.WriteUInt32LittleEndian(h.AsSpan(0x10), (uint)sizeOffset); + BinaryPrimitives.WriteUInt32LittleEndian(h.AsSpan(0x14), Version100); + BinaryPrimitives.WriteUInt32LittleEndian(h.AsSpan(0x18), (uint)dataOffset); + BinaryPrimitives.WriteUInt32LittleEndian(h.AsSpan(0x1C), 1); + BinaryPrimitives.WriteUInt32LittleEndian(h.AsSpan(0x24), (uint)chunkSize); + BinaryPrimitives.WriteInt64LittleEndian(h.AsSpan(0x28), iso.Length); + BinaryPrimitives.WriteInt64LittleEndian(h.AsSpan(0x30), total); + BinaryPrimitives.WriteUInt32LittleEndian(h.AsSpan(0x48), Crc32(h.AsSpan(0, 0x48))); + output.Write(h); + Span entry = stackalloc byte[3]; + foreach (var c in packed) { + entry[0] = (byte)(c.Length >> 16); + entry[1] = (byte)c.Length; + entry[2] = (byte)(c.Length >> 8); + output.Write(entry); + } + foreach (var c in packed) output.Write(c); + } + + private static uint Crc32(ReadOnlySpan data) { + var crc = 0xFFFFFFFFu; + foreach (var b in data) { + crc ^= b; + for (var k = 0; k < 8; ++k) crc = (crc & 1) != 0 ? 0xEDB88320 ^ (crc >> 1) : crc >> 1; + } + return ~crc; + } +} diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs index 6c7d4b452..1c9e17266 100644 --- a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs @@ -33,6 +33,7 @@ public static class FirmwareHexWriter { /// one the reader rendered. An input list with no payload describes an image /// with no data, which both formats can write. /// + /// More than one non-metadata payload is supplied. public static FirmwareImage ImageFrom(IReadOnlyList inputs, string sourceFormat) { ArgumentNullException.ThrowIfNull(inputs); @@ -53,7 +54,9 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st } // Any other single input is the payload, whatever it is called: a caller // converting a .bin into a .hex should not have to rename it first. - payload ??= input.ReadContent(); + if (payload is not null) + throw new InvalidDataException($"{sourceFormat} represents one logical firmware payload; multiple payload inputs are ambiguous."); + payload = input.ReadContent(); } List<(uint Address, byte[] Data)> segments; diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index 4ee792651..73a52cf7e 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -222,6 +222,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | --- | --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | --- | --- | --- | | [BeeGFS](https://en.wikipedia.org/wiki/BeeGFS) | `BeeGfs` | N/A | — | — | — | — | — | — | — | V3/V6 decoder + RAID0 stripe arithmetic against documented layout, over real ext/XFS backing images and their native `user.fhgfs` xattrs — no BeeGFS-produced target set serves as an oracle | Read-only mounting of offline metadata/storage target sets through `FilesystemStreamSet`; V3 dentries + V6 inline/separate regular-file inodes; non-mirrored non-sparse RAID0. Nothing is created or modified: BeeGFS-level mutation would be a transaction across every target plus management state | [BeeGFS](https://www.beegfs.io/) | | [CephFS / RADOS pool export](https://en.wikipedia.org/wiki/Ceph_(software)) | `CephFs` | R/W | ✅ | — | — | ✅ | ✅ | — | ✅ | detection of the on-disk signature | RADOS objects only | [Ceph](https://docs.ceph.com/) | +| [DAA (PowerISO)](https://en.wikipedia.org/wiki/PowerISO#DAA) | `Daa` | R/W | — | — | — | — | — | — | — | PowerISO for Linux `convert` both ways | Format 0x100: raw-deflate chunks with per-chunk CRC-32; 0x110, split, password and gBurner GBI refused. Layout: [DAA-ON-DISK.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/DAA-ON-DISK.md) | [PowerISO](https://www.poweriso.com) | | [Dell EMC Isilon OneFS](https://en.wikipedia.org/wiki/OneFS_distributed_file_system) | `OneFs` | R | — | — | — | — | — | — | — | Dell architecture/diagnostics; raw serialization not public | Isilon OneFS | [OneFS](https://www.dell.com/en-us/dt/storage/powerscale.htm) | | [eCryptfs](https://en.wikipedia.org/wiki/ECryptfs) | `Ecryptfs` | R/W | ✅ | — | — | ✅ | ✅ | — | ✅ | [ecryptfs-utils passphrase vector](https://github.com/dustinkirkland/ecryptfs-utils/blob/master/tests/userspace/verify-passphrase-sig.sh) + Linux kernel layout | Passphrase lower-file AES-128/192/256 read/create; private-key auth and xattr-only metadata are unsupported | [eCryptfs](https://www.kernel.org/doc/html/latest/filesystems/ecryptfs.html) | | [GlusterFS brick](https://en.wikipedia.org/wiki/Gluster) | `GlusterFs` | R | ✅ | — | — | ✅ | ✅ | — | — | detection of the on-disk signature | Brick metadata only | [GlusterFS](https://docs.gluster.org/) | @@ -285,7 +286,7 @@ ops.Remove(image, ["var/log/old.log"]); -Every public and protected member of all 890 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.FileSystems/REFERENCE.md). +Every public and protected member of all 892 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.FileSystems/REFERENCE.md). diff --git a/Hawkynt.FileFormats.FileSystems/REFERENCE.md b/Hawkynt.FileFormats.FileSystems/REFERENCE.md index 21439cab7..5dccf216d 100644 --- a/Hawkynt.FileFormats.FileSystems/REFERENCE.md +++ b/Hawkynt.FileFormats.FileSystems/REFERENCE.md @@ -547,6 +547,60 @@ Writes PSP CSO v1/v2 and ZSO compressed-ISO containers. | `DefaultBlockSize` | `const int DefaultBlockSize` | Default block size — one cooked ISO 9660 sector. | | `Build` | `static byte[] Build(ReadOnlySpan uncompressedData, int blockSize = 2048)` | Builds a canonical CSO v1 stream around `uncompressedData`. | +### Namespace `FileFormat.Daa` + +[`DaaFormatDescriptor`](#daaformatdescriptor) · [`DaaImage`](#daaimage) + +#### `DaaFormatDescriptor` + +PowerISO DAA ("Direct Access Archive") compressed CD/DVD image, format version 0x100: exposes the ISO as `image.iso` plus a synthetic `metadata.ini`; creates and rewrites version 0x100 images PowerISO reads. Version 0x110, split, password-protected and gBurner GBI images are refused. + +Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`, `ISyntheticEntryNames`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `DaaFormatDescriptor` | `DaaFormatDescriptor()` | | +| `ImageName` | `const string ImageName` | | +| `MetadataName` | `const string MetadataName` | | +| `CanPurgeToEmpty` | `bool CanPurgeToEmpty { get; }` | DAA requires an ISO image payload, so it has no empty container for purge to leave behind. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `SyntheticEntryNames` | `IReadOnlySet SyntheticEntryNames { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Replaces `image.iso`, keeping the chunk size. | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | | + +#### `DaaImage` + +PowerISO DAA image, format version 0x100 (single volume, unencrypted): a 76-byte header, a table of 3-byte compressed chunk lengths and raw-deflate chunks. Implemented from `docs/DAA-ON-DISK.md`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `DaaImage` | `DaaImage()` | | +| `DefaultChunkSize` | `const int DefaultChunkSize` | | +| `HeaderSize` | `const int HeaderSize` | | +| `Version100` | `const uint Version100` | | +| `Version110` | `const uint Version110` | | +| `ChunkSize` | `uint ChunkSize { get; }` | | +| `Chunks` | `IReadOnlyList> Chunks { get; }` | | +| `IsoSize` | `long IsoSize { get; }` | | +| `Version` | `uint Version { get; }` | | +| `CopyIsoTo` | `void CopyIsoTo(Stream source, Stream output)` | | +| `Read` | `static DaaImage Read(Stream stream)` | | +| `Write` | `static void Write(Stream output, ReadOnlySpan iso, int chunkSize = 65536, DeflateCompressionLevel level = 6)` | Writes a version 0x100 image: header, 3-byte length table, raw-deflate chunks each followed by its CRC-32. | + ### Namespace `FileFormat.Dmg` [`DmgEntry`](#dmgentry) · [`DmgFormatDescriptor`](#dmgformatdescriptor) · [`DmgReader`](#dmgreader) · [`DmgWriter`](#dmgwriter) diff --git a/docs/DAA-ON-DISK.md b/docs/DAA-ON-DISK.md new file mode 100644 index 000000000..32126819e --- /dev/null +++ b/docs/DAA-ON-DISK.md @@ -0,0 +1,64 @@ +# DAA (PowerISO "Direct Access Archive") — on-disk notes + +`FileFormat.Daa` is implemented from this page. Two sources fed it: + +- Images written by PowerISO for Linux 1.3 (`poweriso-1.3.tar.gz`, freeware, + run as a black box), read field by field. +- Luigi Auriemma's daa2iso 0.1.7e (GPL-2.0). It was read as a description of + behaviour only; no code was taken from it. + +PowerISO is also the oracle in `Compression.Tests/Daa/DaaTests.cs`. It +converts our images back to identical ISOs, and we read the images it writes. + +All integers are little-endian. + +## Header (76 bytes) + +| Offset | Size | Field | +| --- | --- | --- | +| 0x00 | 16 | signature: `"DAA"` + 13 NULs. `"DAA VOL"` marks a later volume of a split image; `"GBI"` is gBurner's variant | +| 0x10 | 4 | offset of the chunk table (`0x4C` when there are no descriptors) | +| 0x14 | 4 | format version: `0x100`, or `0x110` for newer PowerISO | +| 0x18 | 4 | offset of the chunk data | +| 0x1C | 4 | always 1 | +| 0x20 | 4 | always 0 | +| 0x24 | 4 | chunk size, the decoded bytes per chunk (PowerISO 1.3: `0x10000`) | +| 0x28 | 8 | ISO size | +| 0x30 | 8 | DAA file size; if it differs from the file length, the image is split | +| 0x38 | 16 | header data: zero for 0x100; carries the table-bit, LZMA and obfuscation parameters for 0x110 | +| 0x48 | 4 | CRC-32 (zlib polynomial, standard init and final inversion) of bytes 0x00–0x47 | + +## Descriptors + +From `0x4C` up to the chunk table there are `{type, length}` records (8 bytes +plus a body, where `length` includes the 8). Type 1 marks a multi-volume image. +Type 3 marks a password-protected image: its body holds the password type, the +password CRC and a 128-byte key. Both are refused. + +## Version 0x100 chunk table + +There is one 3-byte entry per chunk. The compressed length is +`b0 << 16 | b1 | b2 << 8`. Note the order: the high byte comes first, then the +low byte, then the middle byte. Chunks follow each other from the data offset. + +Each chunk is a **raw deflate** stream (no zlib header) that decodes to the +chunk size, followed by **4 bytes: the CRC-32** (zlib polynomial, little-endian like every other field) +of the deflate bytes. This trailer is measured, not taken from daa2iso, which +never reads it. Every chunk PowerISO 1.3 wrote carries it, and PowerISO +rejects images whose chunks lack it ("The file format is invalid or +unsupported"). The last chunk decodes to the remainder, or more; only up to the +ISO size is kept. PowerISO 1.3 writes nothing else: random data is stored in +deflate's own stored blocks, not as an uncompressed chunk. + +## Not supported (refused with a message) + +- **Version 0x110.** The chunk table is a bit-packed stream of + `(length − 5, type)` pairs. It can be zlib-compressed and XOR-obfuscated, the + deflate block-type numbering is permuted, chunks can be LZMA with an x86 BCJ + filter, and a chunk whose length reaches the chunk size is stored. No + PowerISO build available here writes 0x110, so none of this could be + verified. It stays refused rather than guessed. +- **gBurner GBI.** It has the same layout, but the chunk table is additionally + offset by the low byte of the header CRC and XORed with `table length / 4`. + There is no sample. +- **Split and password-protected images.** diff --git a/docs/catalogue/our-registry.csv b/docs/catalogue/our-registry.csv index 8b6512c45..48aa78564 100644 --- a/docs/catalogue/our-registry.csv +++ b/docs/catalogue/our-registry.csv @@ -129,6 +129,7 @@ format,Cxfs,SGI CXFS (Cluster XFS),Archive,Y,.cxfs,Y,Y,Y,Y format,D64,D64,Archive,Y,.d64,Y,Y,Y,Y format,D71,D71,Archive,Y,.d71,Y,Y,Y,Y format,D81,D81,Archive,Y,.d81,Y,Y,Y,Y +format,Daa,DAA (PowerISO),Archive,,.daa,Y,Y,Y,Y format,Dar,Disk ARchive (DAR),Archive,,.dar,Y,Y,Y, format,Dbm,DBM (DigiBooster Pro),Audio,,.dbm,Y,Y,, format,Dcx,DCX (multi-image PCX),Archive,,.dcx,Y,Y,, diff --git a/docs/catalogue/reference-formats.csv b/docs/catalogue/reference-formats.csv index 7c10a338d..97fb9cd81 100644 --- a/docs/catalogue/reference-formats.csv +++ b/docs/catalogue/reference-formats.csv @@ -270,7 +270,7 @@ Dreamcast GDI,gdi,disk-image,,,,,R,,,1,,,,,,MISSING,, ECM (Error Code Modeler),ecm,disk-image,,,,,R,,,1,,,,,,MISSING,, gBurner image,gbi,disk-image,,,,,R,,,1,,,,,,MISSING,, ISZ (compressed ISO),isz,disk-image,,,,,R,,Y,2,,,,,,MISSING,, -PowerISO DAA,daa,disk-image,,,,,R,,Y,2,,,,,,MISSING,, +PowerISO DAA,daa,disk-image,,,,,R,,Y,2,https://www.poweriso.com,Daa,Y,Y,Y,PARTIAL,,"format 0x100 read/write verified both ways against PowerISO for Linux 1.3; 0x110, split, password and GBI refused (no verifiable samples); spec docs/DAA-ON-DISK.md" MagicISO UIF,uif,disk-image,,,,,R,,Y,2,,,,,,MISSING,, Compressed ISO (CSO/ZSO),cso;zso,disk-image,,,,,R,,,1,,Cso,Y,Y,Y,OK,, HTC NBH ROM image,nbh,firmware,,,,,R,,,1,,,,,,MISSING,,