+ Mozilla ARchive (MAR)

Compression.Tests/Compression.Tests.csproj

@@ -154,6 +154,10 @@
     <EmbeddedResource Include="Zoo\ReferenceVectors\*.zoo">
       <LogicalName>ZooVectors.%(Filename)%(Extension)</LogicalName>
     </EmbeddedResource>
+    <!-- MARs from mardor's test data and written by mardor; see Mar/ReferenceVectors/README.md. -->
+    <EmbeddedResource Include="Mar\ReferenceVectors\*.mar;Mar\ReferenceVectors\mardor-listing.txt">
+      <LogicalName>MarVectors.%(Filename)%(Extension)</LogicalName>
+    </EmbeddedResource>
     <!-- VDF volumes from GothicVDFS, VDFSTool and ZenKit, with ZenKit's listings; see Vdfs/ReferenceVectors/README.md. -->
     <EmbeddedResource Include="Vdfs\ReferenceVectors\*.vdf;Vdfs\ReferenceVectors\*.tsv">
       <LogicalName>VdfsVectors.%(Filename)%(Extension)</LogicalName>

Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs

@@ -0,0 +1,37 @@
+using Compression.Registry;
+using FileFormat.FirmwareHex;
+
+namespace Compression.Tests.FirmwareHex;
+
+[TestFixture]
+public sealed class FirmwareHexPayloadTests {
+  [TestCase(false)]
+  [TestCase(true)]
+  public void Create_RejectsMultiplePayloadsWithoutWritingOutput(bool emptyFirst) {
+    var descriptor = new IntelHexFormatDescriptor();
+    using var output = new MemoryStream();
+    var inputs = new[] {
+      ArchiveInputInfo.InMemory("first.bin", emptyFirst ? [] : [1, 2, 3]),
+      ArchiveInputInfo.InMemory("second.bin", [9, 8, 7]),
+    };
+
+    Assert.Throws<InvalidDataException>(() =>
+      ((IArchiveCreatable)descriptor).Create(output, inputs, new FormatCreateOptions()));
+    Assert.That(output.Length, Is.Zero);
+  }
+
+  [Test]
+  public void Add_RejectsAnotherPayloadWithoutChangingTheArchive() {
+    var descriptor = new IntelHexFormatDescriptor();
+    using var archive = new MemoryStream();
+    ((IArchiveCreatable)descriptor).Create(archive,
+      [ArchiveInputInfo.InMemory("firmware.bin", [1, 2, 3])], new FormatCreateOptions());
+    var original = archive.ToArray();
+    archive.Position = 0;
+
+    Assert.Throws<InvalidDataException>(() => ((IArchiveModifiable)descriptor).Add(archive,
+      [ArchiveInputInfo.InMemory("ADD.BIN", [9, 8, 7])]));
+    Assert.That(archive.ToArray(), Is.EqualTo(original));
+  }
+
+}

Compression.Tests/Mar/MarTests.cs

@@ -0,0 +1,162 @@
+using System.Reflection;
+using System.Security.Cryptography;
+using Compression.Registry;
+using FileFormat.Mar;
+
+namespace Compression.Tests.Mar;
+
+/// <summary>
+/// Mozilla ARchive. Oracle: mardor (Mozilla release engineering's MAR library). The two
+/// <c>test-*.mar</c> fixtures are mardor's own Thunderbird test packages (one signed); the
+/// <c>mardor-*.mar</c> ones mardor wrote for this suite. <c>mardor-listing.txt</c> is mardor's view of
+/// every fixture. See ReferenceVectors/README.md.
+/// </summary>
+[TestFixture]
+public class MarTests {
+
+  private static byte[] Vector(string name) {
+    using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream($"MarVectors.{name}")
+      ?? throw new FileNotFoundException($"Embedded MAR vector '{name}' is missing from the test assembly.");
+    using var ms = new MemoryStream();
+    stream.CopyTo(ms);
+    return ms.ToArray();
+  }
+
+  private static readonly byte[] ATxt = "hello mar\n"u8.ToArray();
+  private static readonly byte[] BBin = [.. Enumerable.Range(0, 1024).Select(i => (byte)i)];
+
+  private static IEnumerable<string[]> Listing()
+    => System.Text.Encoding.ASCII.GetString(Vector("mardor-listing.txt"))
+      .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+      .Select(l => l.Split(' '));
+
+  [TestCase("test-xz.mar")]
+  [TestCase("test-bz2.mar")]
+  [TestCase("mardor-old.mar")]
+  [TestCase("mardor-product.mar")]
+  [TestCase("mardor-xz.mar")]
+  [Category("ThirdPartyVector")]
+  public void Given_MardorFixture_When_Read_Then_MatchesMardorListing(string vector) {
+    var lines = Listing().Where(l => l[0] == vector || (l[0] == "#" && l[1] == vector)).ToList();
+    var header = lines.Single(l => l[0] == "#");
+    var expectedEntries = lines.Where(l => l[0] != "#").ToList();
+
+    var r = new MarReader(new MemoryStream(Vector(vector)));
+    var sigs = int.Parse(header[2]["sigs=".Length..]);
+    Assert.That(r.HasSignatureBlock, Is.EqualTo(sigs >= 0));
+    Assert.That(r.Signatures, Has.Count.EqualTo(Math.Max(sigs, 0)));
+    var product = header[3]["product=".Length..];
+    Assert.That(r.ProductInformation is { } pi ? $"{pi.Channel}|{pi.ProductVersion}" : "-", Is.EqualTo(product));
+
+    Assert.That(r.Entries.Select(e => e.Name), Is.EqualTo(expectedEntries.Select(l => l[1])));
+    for (var i = 0; i < r.Entries.Count; ++i) {
+      var e = r.Entries[i];
+      Assert.That(e.Size, Is.EqualTo(uint.Parse(expectedEntries[i][2])), e.Name);
+      Assert.That(Convert.ToString(e.Flags, 8), Is.EqualTo(expectedEntries[i][3]), e.Name);
+      Assert.That(Convert.ToHexStringLower(SHA256.HashData(r.Extract(e))), Is.EqualTo(expectedEntries[i][4]), e.Name);
+    }
+  }
+
+  [TestCase("test-xz.mar", "xz")]
+  [TestCase("test-bz2.mar", "bzip2")]
+  [TestCase("mardor-product.mar", "stored")]
+  [Category("ThirdPartyVector")]
+  public void Given_UpdatePackage_When_Listed_Then_MethodNamesTheMemberCompression(string vector, string method) {
+    var list = new MarFormatDescriptor().List(new MemoryStream(Vector(vector)), null);
+    Assert.That(list.Select(e => e.Method).Distinct(), Is.EqualTo(new[] { method }));
+  }
+
+  [TestCase("mardor-old.mar", false)]
+  [TestCase("mardor-product.mar", true)]
+  [Category("ThirdPartyVector")]
+  public void Given_MardorInputs_When_WeWrite_Then_ByteIdenticalToMardor(string vector, bool product) {
+    using var ms = new MemoryStream();
+    MarWriter.Write(ms, [("a.txt", ATxt, 0x1B6u), ("dir/b.bin", BBin, 0x1B6u)],
+      product ? [MarWriter.ProductInformation("release", "130.0")] : null);
+    Assert.That(ms.ToArray(), Is.EqualTo(Vector(vector)));
+  }
+
+  [Test, Category("RoundTrip")]
+  public void Given_ProductOptions_When_Created_Then_ReadBackWithProductBlock() {
+    var d = new MarFormatDescriptor();
+    using var ms = new MemoryStream();
+    var options = new FormatCreateOptions { FormatSpecific = new() { ["channel"] = "beta", ["productversion"] = "131.0b1" } };
+    d.Create(ms, [ArchiveInputInfo.InMemory("a.txt", ATxt), ArchiveInputInfo.InMemory("dir/b.bin", BBin)], options);
+    ms.Position = 0;
+    var r = new MarReader(ms);
+    Assert.That(r.ProductInformation, Is.EqualTo(("beta", "131.0b1")));
+    Assert.That(r.Extract(r.Entries[1]), Is.EqualTo(BBin));
+    Assert.That(r.DeclaredFileSize, Is.EqualTo((ulong)ms.Length));
+  }
+
+  [Test, Category("EdgeCase")]
+  public void Given_OnlyChannel_When_Created_Then_Throws() {
+    var options = new FormatCreateOptions { FormatSpecific = new() { ["channel"] = "beta" } };
+    Assert.Throws<ArgumentException>(() => new MarFormatDescriptor().Create(new MemoryStream(), [], options));
+  }
+
+  [Test, Category("EdgeCase")]
+  public void Given_NoFiles_When_Created_Then_EmptyListing() {
+    var d = new MarFormatDescriptor();
+    using var ms = new MemoryStream();
+    d.Create(ms, [], new FormatCreateOptions());
+    Assert.That(ms.Length, Is.EqualTo(12));
+    ms.Position = 0;
+    Assert.That(d.List(ms, null), Is.Empty);
+  }
+
+  [Test, Category("RoundTrip")]
+  public void Given_UnsignedProductMar_When_AddedAndRemoved_Then_ProductBlockAndFlagsSurvive() {
+    var d = new MarFormatDescriptor();
+    using var ms = new MemoryStream();
+    ms.Write(Vector("mardor-product.mar"));
+    ((IArchiveModifiable)d).Add(ms, [ArchiveInputInfo.InMemory("c.txt", "added"u8.ToArray())]);
+    ((IArchiveModifiable)d).Remove(ms, ["a.txt"]);
+    ms.Position = 0;
+    var r = new MarReader(ms);
+    Assert.That(r.ProductInformation, Is.EqualTo(("release", "130.0")));
+    Assert.That(r.Entries.Select(e => e.Name), Is.EqualTo(new[] { "dir/b.bin", "c.txt" }));
+    Assert.That(r.Entries[0].Flags, Is.EqualTo(0x1B6u));
+    Assert.That(r.Extract(r.Entries[0]), Is.EqualTo(BBin));
+    Assert.That(r.Extract(r.Entries[1]), Is.EqualTo("added"u8.ToArray()));
+  }
+
+  [Test, Category("EdgeCase")]
+  public void Given_SignedMar_When_Modified_Then_RefusedAndUntouched() {
+    var original = Vector("test-bz2.mar");
+    using var ms = new MemoryStream();
+    ms.Write(original);
+    var d = (IArchiveModifiable)new MarFormatDescriptor();
+    Assert.Throws<NotSupportedException>(() => d.Remove(ms, ["update.manifest"]));
+    Assert.Throws<NotSupportedException>(() => d.Add(ms, [ArchiveInputInfo.InMemory("x", [1])]));
+    Assert.That(ms.ToArray(), Is.EqualTo(original));
+  }
+
+  [Test, Category("EdgeCase")]
+  public void Given_WrongMagic_When_Read_Then_InvalidData() {
+    var bytes = Vector("mardor-old.mar");
+    bytes[3] = (byte)'2';
+    Assert.Throws<InvalidDataException>(() => _ = new MarReader(new MemoryStream(bytes)));
+  }
+
+  [Test, Category("EdgeCase")]
+  public void Given_IndexOffsetPastTheEnd_When_Read_Then_InvalidData() {
+    var bytes = Vector("mardor-old.mar");
+    bytes[4] = 0x7F;
+    Assert.Throws<InvalidDataException>(() => _ = new MarReader(new MemoryStream(bytes)));
+  }
+
+  [Test, Category("EdgeCase")]
+  public void Given_EntryPointingIntoTheIndex_When_Read_Then_InvalidData() {
+    var bytes = Vector("mardor-old.mar");
+    // first index entry's size field (index at 0x412: UInt32 indexSize, then offset, size)
+    bytes[0x412 + 4 + 4 + 2] = 0xFF;
+    Assert.Throws<InvalidDataException>(() => _ = new MarReader(new MemoryStream(bytes)));
+  }
+
+  [Test, Category("EdgeCase")]
+  public void Given_TruncatedFile_When_Read_Then_InvalidData() {
+    var bytes = Vector("mardor-product.mar")[..20];
+    Assert.Catch<Exception>(() => _ = new MarReader(new MemoryStream(bytes)));
+  }
+}

Compression.Tests/Mar/ReferenceVectors/.gitattributes

@@ -0,0 +1,2 @@
+# Fixture bytes are the test: never let a checkout rewrite line endings.
+*.mar -text

Compression.Tests/Mar/ReferenceVectors/README.md

@@ -0,0 +1,16 @@
+# MAR reference vectors
+
+| File | Origin |
+| --- | --- |
+| `test-xz.mar` | mardor's test data (`tests/test-xz.mar`, mozilla-releng/build-mar master, fetched 2026-10-19): a Thunderbird `thunderbird-comm-esr` 100.0 update package, xz-compressed members, signature block with no signatures. |
+| `test-bz2.mar` | mardor's test data (`tests/test-bz2.mar`, same revision): the same package with bzip2-compressed members and one RSA signature. |
+| `mardor-old.mar` | written by mardor (build-mar master, Python 3.13, Windows 11) with `mar -c` from `a.txt` (`hello mar\n`) and `dir/b.bin` (bytes 0..255 four times): the original layout without signature block. |
+| `mardor-product.mar` | same inputs, `mar -H release -V 130.0 -c`: signed-era layout, zero signatures, product-information block. |
+| `mardor-xz.mar` | same as `mardor-product.mar` plus `-J` (members xz-compressed). |
+| `mardor-listing.txt` | mardor's `MarReader` view of every file above: signature count (-1 = no block), product information, and each member's size, mode and the SHA-256 of its stored bytes. |
+
+The two `test-*.mar` files are © Mozilla, distributed under the Mozilla Public License 2.0
+(<https://mozilla.org/MPL/2.0/>), as part of mardor; they are unmodified.
+
+Our writer must reproduce `mardor-old.mar` and `mardor-product.mar` byte for byte from the same inputs
+(mardor recorded Windows' `0666` mode for both members).

Compression.Tests/Mar/ReferenceVectors/mardor-listing.txt

@@ -0,0 +1,21 @@
+# test-xz.mar sigs=0 product=thunderbird-comm-esr|100.0
+test-xz.mar Contents/MacOS/defaults/pref/channel-prefs.js 96 644 7ba5d20884bb59191e5788122495bd2226d28751e96dbdc886c22946e806da83
+test-xz.mar Contents/MacOS/update-settings.ini 180 644 9c29428fe4c81b8d541c8364de8a4109ee5321dd40e0d29368c7421ecda5ef3c
+test-xz.mar defaults/pref/channel-prefs.js 96 644 7ba5d20884bb59191e5788122495bd2226d28751e96dbdc886c22946e806da83
+test-xz.mar update.manifest 164 644 a96c569df98462269f2bf04b842bf7f7a0a3461844d3181dc2561a2b9b6558fa
+test-xz.mar update-settings.ini 180 644 9c29428fe4c81b8d541c8364de8a4109ee5321dd40e0d29368c7421ecda5ef3c
+# test-bz2.mar sigs=1 product=thunderbird-comm-esr|100.0
+test-bz2.mar update.manifest 141 664 c717714abe8bb1cf173837f4592f8bf229c9884d71621bf8e5709e968ac929f3
+test-bz2.mar defaults/pref/channel-prefs.js 76 664 cdcb59e3add3e34b239316f365970b666383a07fca4e4595652b843dda53dc07
+test-bz2.mar Contents/MacOS/defaults/pref/channel-prefs.js 76 664 cdcb59e3add3e34b239316f365970b666383a07fca4e4595652b843dda53dc07
+test-bz2.mar update-settings.ini 152 664 6c4d6d9824a6846ce95d4fddcee9e177c0f90ad052fa683bffe3167abc358bb9
+test-bz2.mar Contents/MacOS/update-settings.ini 152 664 6c4d6d9824a6846ce95d4fddcee9e177c0f90ad052fa683bffe3167abc358bb9
+# mardor-old.mar sigs=-1 product=-
+mardor-old.mar a.txt 10 666 4ea8280daa0d618b98c4b85e6db2fff92b19b99318603d0037b426342fbb1e13
+mardor-old.mar dir/b.bin 1024 666 785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9
+# mardor-product.mar sigs=0 product=release|130.0
+mardor-product.mar a.txt 10 666 4ea8280daa0d618b98c4b85e6db2fff92b19b99318603d0037b426342fbb1e13
+mardor-product.mar dir/b.bin 1024 666 785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9
+# mardor-xz.mar sigs=0 product=release|130.0
+mardor-xz.mar a.txt 68 666 2ae9565f7893f85690c7c067e260c0b6051e46a07a3f9b11d9e48933a26e7b27
+mardor-xz.mar dir/b.bin 308 666 24ab21bc489b17266f9f380ce392136d380e05d12725f2e1eddab587ef73ba2c

Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Mar/MarFormatDescriptor.cs

@@ -0,0 +1,108 @@
+#pragma warning disable CS1591
+using Compression.Registry;
+using static Compression.Registry.FormatHelpers;
+
+namespace FileFormat.Mar;
+
+/// <summary>
+/// Mozilla ARchive (MAR) — the container of Firefox / Thunderbird update packages.
+///
+/// References:
+/// <list type="bullet">
+///   <item><description><c>https://wiki.mozilla.org/Software_Update:MAR</c> — Mozilla's format description</description></item>
+///   <item><description><c>https://github.com/mozilla-releng/build-mar</c> — mardor, Mozilla release engineering's MAR reader/writer (the oracle)</description></item>
+/// </list>
+/// </summary>
+public sealed class MarFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable {
+
+  public string Id => "Mar";
+  public string DisplayName => "MAR";
+  public FormatCategory Category => FormatCategory.Archive;
+  // R/W: unsigned archives are rebuilt with their product-information / additional sections and
+  // member permission bits intact; signed archives refuse edits (the signatures would no longer verify).
+  public FormatCapabilities Capabilities =>
+    FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |
+    FormatCapabilities.CanModify | FormatCapabilities.CanTest |
+    FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories;
+  public string DefaultExtension => ".mar";
+  public IReadOnlyList<string> Extensions => [".mar"];
+  public IReadOnlyList<string> CompoundExtensions => [];
+  public IReadOnlyList<MagicSignature> MagicSignatures => [new("MAR1"u8.ToArray(), Confidence: 0.9)];
+  public IReadOnlyList<FormatMethodInfo> Methods => [new("store", "Stored")];
+  public string? TarCompressionFormatId => null;
+  public AlgorithmFamily Family => AlgorithmFamily.Archive;
+  public string Description =>
+    "Mozilla update archive. Members are listed and extracted exactly as stored (update packages hold them " +
+    "bzip2- or xz-compressed; the Method column names which). Create options: channel + productversion " +
+    "write a product-information block.";
+
+  public List<ArchiveEntryInfo> List(Stream stream, string? password) {
+    var r = new MarReader(stream);
+    var list = new List<ArchiveEntryInfo>(r.Entries.Count);
+    for (var i = 0; i < r.Entries.Count; ++i) {
+      var e = r.Entries[i];
+      list.Add(new ArchiveEntryInfo(i, e.Name, e.Size, e.Size, MarReader.DescribeStoredData(Head(stream, e)), false, false, null));
+    }
+    return list;
+  }
+
+  public void Extract(Stream stream, string outputDir, string? password, string[]? files) {
+    var r = new MarReader(stream);
+    foreach (var e in r.Entries) {
+      if (files != null && !MatchesFilter(e.Name, files)) continue;
+      WriteFile(outputDir, e.Name, r.Extract(e));
+    }
+  }
+
+  public void Create(Stream output, IReadOnlyList<ArchiveInputInfo> inputs, FormatCreateOptions options) {
+    var channel = options.GetString("channel");
+    var version = options.GetString("productversion");
+    if ((channel == null) != (version == null))
+      throw new ArgumentException("MAR: channel and productversion must be given together.");
+    IReadOnlyList<MarAdditionalSection>? sections = channel != null ? [MarWriter.ProductInformation(channel, version!)] : null;
+    var files = FilesOnly(inputs).Select(f => (f.Name, f.Data, MarWriter.DefaultFlags)).ToList();
+    MarWriter.Write(output, files, sections);
+  }
+
+  public void Add(Stream archive, IReadOnlyList<ArchiveInputInfo> inputs) {
+    ArgumentNullException.ThrowIfNull(inputs);
+    Rewrite(archive, files => {
+      foreach (var (name, data) in FilesOnly(inputs)) {
+        var existing = files.FindIndex(f => string.Equals(f.Name, name, StringComparison.Ordinal));
+        if (existing >= 0) files[existing] = (name, data, files[existing].Flags);
+        else files.Add((name, data, MarWriter.DefaultFlags));
+      }
+    });
+  }
+
+  public void Remove(Stream archive, string[] entryNames) {
+    ArgumentNullException.ThrowIfNull(entryNames);
+    var drop = new HashSet<string>(entryNames, StringComparer.Ordinal);
+    Rewrite(archive, files => files.RemoveAll(f => drop.Contains(f.Name)));
+  }
+
+  private static void Rewrite(Stream archive, Action<List<(string Name, byte[] Data, uint Flags)>> edit) {
+    ArgumentNullException.ThrowIfNull(archive);
+    archive.Position = 0;
+    var r = new MarReader(archive);
+    if (r.Signatures.Count > 0)
+      throw new NotSupportedException("MAR: the archive is signed; editing it would invalidate its signatures.");
+    var files = r.Entries.Select(e => (e.Name, r.Extract(e), e.Flags)).ToList();
+    edit(files);
+    using var rebuilt = new MemoryStream();
+    MarWriter.Write(rebuilt, files, r.HasSignatureBlock ? r.AdditionalSections : null);
+    archive.Position = 0;
+    archive.SetLength(0);
+    rebuilt.Position = 0;
+    rebuilt.CopyTo(archive);
+    archive.Flush();
+  }
+
+  private static byte[] Head(Stream stream, MarEntry e) {
+    var n = (int)Math.Min(6u, e.Size);
+    var buf = new byte[n];
+    stream.Position = e.Offset;
+    stream.ReadExactly(buf);
+    return buf;
+  }
+}