diff --git a/Compression.Tests/Compression.Tests.csproj b/Compression.Tests/Compression.Tests.csproj index 023dbbe39..0c44fe69a 100644 --- a/Compression.Tests/Compression.Tests.csproj +++ b/Compression.Tests/Compression.Tests.csproj @@ -154,6 +154,10 @@ ZooVectors.%(Filename)%(Extension) + + + MarVectors.%(Filename)%(Extension) + VdfsVectors.%(Filename)%(Extension) diff --git a/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs new file mode 100644 index 000000000..bbbc898af --- /dev/null +++ b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs @@ -0,0 +1,37 @@ +using Compression.Registry; +using FileFormat.FirmwareHex; + +namespace Compression.Tests.FirmwareHex; + +[TestFixture] +public sealed class FirmwareHexPayloadTests { + [TestCase(false)] + [TestCase(true)] + public void Create_RejectsMultiplePayloadsWithoutWritingOutput(bool emptyFirst) { + var descriptor = new IntelHexFormatDescriptor(); + using var output = new MemoryStream(); + var inputs = new[] { + ArchiveInputInfo.InMemory("first.bin", emptyFirst ? [] : [1, 2, 3]), + ArchiveInputInfo.InMemory("second.bin", [9, 8, 7]), + }; + + Assert.Throws(() => + ((IArchiveCreatable)descriptor).Create(output, inputs, new FormatCreateOptions())); + Assert.That(output.Length, Is.Zero); + } + + [Test] + public void Add_RejectsAnotherPayloadWithoutChangingTheArchive() { + var descriptor = new IntelHexFormatDescriptor(); + using var archive = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(archive, + [ArchiveInputInfo.InMemory("firmware.bin", [1, 2, 3])], new FormatCreateOptions()); + var original = archive.ToArray(); + archive.Position = 0; + + Assert.Throws(() => ((IArchiveModifiable)descriptor).Add(archive, + [ArchiveInputInfo.InMemory("ADD.BIN", [9, 8, 7])])); + Assert.That(archive.ToArray(), Is.EqualTo(original)); + } + +} diff --git a/Compression.Tests/Mar/MarTests.cs b/Compression.Tests/Mar/MarTests.cs new file mode 100644 index 000000000..00527d499 --- /dev/null +++ b/Compression.Tests/Mar/MarTests.cs @@ -0,0 +1,162 @@ +using System.Reflection; +using System.Security.Cryptography; +using Compression.Registry; +using FileFormat.Mar; + +namespace Compression.Tests.Mar; + +/// +/// Mozilla ARchive. Oracle: mardor (Mozilla release engineering's MAR library). The two +/// test-*.mar fixtures are mardor's own Thunderbird test packages (one signed); the +/// mardor-*.mar ones mardor wrote for this suite. mardor-listing.txt is mardor's view of +/// every fixture. See ReferenceVectors/README.md. +/// +[TestFixture] +public class MarTests { + + private static byte[] Vector(string name) { + using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream($"MarVectors.{name}") + ?? throw new FileNotFoundException($"Embedded MAR vector '{name}' is missing from the test assembly."); + using var ms = new MemoryStream(); + stream.CopyTo(ms); + return ms.ToArray(); + } + + private static readonly byte[] ATxt = "hello mar\n"u8.ToArray(); + private static readonly byte[] BBin = [.. Enumerable.Range(0, 1024).Select(i => (byte)i)]; + + private static IEnumerable Listing() + => System.Text.Encoding.ASCII.GetString(Vector("mardor-listing.txt")) + .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(l => l.Split(' ')); + + [TestCase("test-xz.mar")] + [TestCase("test-bz2.mar")] + [TestCase("mardor-old.mar")] + [TestCase("mardor-product.mar")] + [TestCase("mardor-xz.mar")] + [Category("ThirdPartyVector")] + public void Given_MardorFixture_When_Read_Then_MatchesMardorListing(string vector) { + var lines = Listing().Where(l => l[0] == vector || (l[0] == "#" && l[1] == vector)).ToList(); + var header = lines.Single(l => l[0] == "#"); + var expectedEntries = lines.Where(l => l[0] != "#").ToList(); + + var r = new MarReader(new MemoryStream(Vector(vector))); + var sigs = int.Parse(header[2]["sigs=".Length..]); + Assert.That(r.HasSignatureBlock, Is.EqualTo(sigs >= 0)); + Assert.That(r.Signatures, Has.Count.EqualTo(Math.Max(sigs, 0))); + var product = header[3]["product=".Length..]; + Assert.That(r.ProductInformation is { } pi ? $"{pi.Channel}|{pi.ProductVersion}" : "-", Is.EqualTo(product)); + + Assert.That(r.Entries.Select(e => e.Name), Is.EqualTo(expectedEntries.Select(l => l[1]))); + for (var i = 0; i < r.Entries.Count; ++i) { + var e = r.Entries[i]; + Assert.That(e.Size, Is.EqualTo(uint.Parse(expectedEntries[i][2])), e.Name); + Assert.That(Convert.ToString(e.Flags, 8), Is.EqualTo(expectedEntries[i][3]), e.Name); + Assert.That(Convert.ToHexStringLower(SHA256.HashData(r.Extract(e))), Is.EqualTo(expectedEntries[i][4]), e.Name); + } + } + + [TestCase("test-xz.mar", "xz")] + [TestCase("test-bz2.mar", "bzip2")] + [TestCase("mardor-product.mar", "stored")] + [Category("ThirdPartyVector")] + public void Given_UpdatePackage_When_Listed_Then_MethodNamesTheMemberCompression(string vector, string method) { + var list = new MarFormatDescriptor().List(new MemoryStream(Vector(vector)), null); + Assert.That(list.Select(e => e.Method).Distinct(), Is.EqualTo(new[] { method })); + } + + [TestCase("mardor-old.mar", false)] + [TestCase("mardor-product.mar", true)] + [Category("ThirdPartyVector")] + public void Given_MardorInputs_When_WeWrite_Then_ByteIdenticalToMardor(string vector, bool product) { + using var ms = new MemoryStream(); + MarWriter.Write(ms, [("a.txt", ATxt, 0x1B6u), ("dir/b.bin", BBin, 0x1B6u)], + product ? [MarWriter.ProductInformation("release", "130.0")] : null); + Assert.That(ms.ToArray(), Is.EqualTo(Vector(vector))); + } + + [Test, Category("RoundTrip")] + public void Given_ProductOptions_When_Created_Then_ReadBackWithProductBlock() { + var d = new MarFormatDescriptor(); + using var ms = new MemoryStream(); + var options = new FormatCreateOptions { FormatSpecific = new() { ["channel"] = "beta", ["productversion"] = "131.0b1" } }; + d.Create(ms, [ArchiveInputInfo.InMemory("a.txt", ATxt), ArchiveInputInfo.InMemory("dir/b.bin", BBin)], options); + ms.Position = 0; + var r = new MarReader(ms); + Assert.That(r.ProductInformation, Is.EqualTo(("beta", "131.0b1"))); + Assert.That(r.Extract(r.Entries[1]), Is.EqualTo(BBin)); + Assert.That(r.DeclaredFileSize, Is.EqualTo((ulong)ms.Length)); + } + + [Test, Category("EdgeCase")] + public void Given_OnlyChannel_When_Created_Then_Throws() { + var options = new FormatCreateOptions { FormatSpecific = new() { ["channel"] = "beta" } }; + Assert.Throws(() => new MarFormatDescriptor().Create(new MemoryStream(), [], options)); + } + + [Test, Category("EdgeCase")] + public void Given_NoFiles_When_Created_Then_EmptyListing() { + var d = new MarFormatDescriptor(); + using var ms = new MemoryStream(); + d.Create(ms, [], new FormatCreateOptions()); + Assert.That(ms.Length, Is.EqualTo(12)); + ms.Position = 0; + Assert.That(d.List(ms, null), Is.Empty); + } + + [Test, Category("RoundTrip")] + public void Given_UnsignedProductMar_When_AddedAndRemoved_Then_ProductBlockAndFlagsSurvive() { + var d = new MarFormatDescriptor(); + using var ms = new MemoryStream(); + ms.Write(Vector("mardor-product.mar")); + ((IArchiveModifiable)d).Add(ms, [ArchiveInputInfo.InMemory("c.txt", "added"u8.ToArray())]); + ((IArchiveModifiable)d).Remove(ms, ["a.txt"]); + ms.Position = 0; + var r = new MarReader(ms); + Assert.That(r.ProductInformation, Is.EqualTo(("release", "130.0"))); + Assert.That(r.Entries.Select(e => e.Name), Is.EqualTo(new[] { "dir/b.bin", "c.txt" })); + Assert.That(r.Entries[0].Flags, Is.EqualTo(0x1B6u)); + Assert.That(r.Extract(r.Entries[0]), Is.EqualTo(BBin)); + Assert.That(r.Extract(r.Entries[1]), Is.EqualTo("added"u8.ToArray())); + } + + [Test, Category("EdgeCase")] + public void Given_SignedMar_When_Modified_Then_RefusedAndUntouched() { + var original = Vector("test-bz2.mar"); + using var ms = new MemoryStream(); + ms.Write(original); + var d = (IArchiveModifiable)new MarFormatDescriptor(); + Assert.Throws(() => d.Remove(ms, ["update.manifest"])); + Assert.Throws(() => d.Add(ms, [ArchiveInputInfo.InMemory("x", [1])])); + Assert.That(ms.ToArray(), Is.EqualTo(original)); + } + + [Test, Category("EdgeCase")] + public void Given_WrongMagic_When_Read_Then_InvalidData() { + var bytes = Vector("mardor-old.mar"); + bytes[3] = (byte)'2'; + Assert.Throws(() => _ = new MarReader(new MemoryStream(bytes))); + } + + [Test, Category("EdgeCase")] + public void Given_IndexOffsetPastTheEnd_When_Read_Then_InvalidData() { + var bytes = Vector("mardor-old.mar"); + bytes[4] = 0x7F; + Assert.Throws(() => _ = new MarReader(new MemoryStream(bytes))); + } + + [Test, Category("EdgeCase")] + public void Given_EntryPointingIntoTheIndex_When_Read_Then_InvalidData() { + var bytes = Vector("mardor-old.mar"); + // first index entry's size field (index at 0x412: UInt32 indexSize, then offset, size) + bytes[0x412 + 4 + 4 + 2] = 0xFF; + Assert.Throws(() => _ = new MarReader(new MemoryStream(bytes))); + } + + [Test, Category("EdgeCase")] + public void Given_TruncatedFile_When_Read_Then_InvalidData() { + var bytes = Vector("mardor-product.mar")[..20]; + Assert.Catch(() => _ = new MarReader(new MemoryStream(bytes))); + } +} diff --git a/Compression.Tests/Mar/ReferenceVectors/.gitattributes b/Compression.Tests/Mar/ReferenceVectors/.gitattributes new file mode 100644 index 000000000..bb6720576 --- /dev/null +++ b/Compression.Tests/Mar/ReferenceVectors/.gitattributes @@ -0,0 +1,2 @@ +# Fixture bytes are the test: never let a checkout rewrite line endings. +*.mar -text diff --git a/Compression.Tests/Mar/ReferenceVectors/README.md b/Compression.Tests/Mar/ReferenceVectors/README.md new file mode 100644 index 000000000..94ab52814 --- /dev/null +++ b/Compression.Tests/Mar/ReferenceVectors/README.md @@ -0,0 +1,16 @@ +# MAR reference vectors + +| File | Origin | +| --- | --- | +| `test-xz.mar` | mardor's test data (`tests/test-xz.mar`, mozilla-releng/build-mar master, fetched 2026-10-19): a Thunderbird `thunderbird-comm-esr` 100.0 update package, xz-compressed members, signature block with no signatures. | +| `test-bz2.mar` | mardor's test data (`tests/test-bz2.mar`, same revision): the same package with bzip2-compressed members and one RSA signature. | +| `mardor-old.mar` | written by mardor (build-mar master, Python 3.13, Windows 11) with `mar -c` from `a.txt` (`hello mar\n`) and `dir/b.bin` (bytes 0..255 four times): the original layout without signature block. | +| `mardor-product.mar` | same inputs, `mar -H release -V 130.0 -c`: signed-era layout, zero signatures, product-information block. | +| `mardor-xz.mar` | same as `mardor-product.mar` plus `-J` (members xz-compressed). | +| `mardor-listing.txt` | mardor's `MarReader` view of every file above: signature count (-1 = no block), product information, and each member's size, mode and the SHA-256 of its stored bytes. | + +The two `test-*.mar` files are © Mozilla, distributed under the Mozilla Public License 2.0 +(), as part of mardor; they are unmodified. + +Our writer must reproduce `mardor-old.mar` and `mardor-product.mar` byte for byte from the same inputs +(mardor recorded Windows' `0666` mode for both members). diff --git a/Compression.Tests/Mar/ReferenceVectors/mardor-listing.txt b/Compression.Tests/Mar/ReferenceVectors/mardor-listing.txt new file mode 100644 index 000000000..22798f56a --- /dev/null +++ b/Compression.Tests/Mar/ReferenceVectors/mardor-listing.txt @@ -0,0 +1,21 @@ +# test-xz.mar sigs=0 product=thunderbird-comm-esr|100.0 +test-xz.mar Contents/MacOS/defaults/pref/channel-prefs.js 96 644 7ba5d20884bb59191e5788122495bd2226d28751e96dbdc886c22946e806da83 +test-xz.mar Contents/MacOS/update-settings.ini 180 644 9c29428fe4c81b8d541c8364de8a4109ee5321dd40e0d29368c7421ecda5ef3c +test-xz.mar defaults/pref/channel-prefs.js 96 644 7ba5d20884bb59191e5788122495bd2226d28751e96dbdc886c22946e806da83 +test-xz.mar update.manifest 164 644 a96c569df98462269f2bf04b842bf7f7a0a3461844d3181dc2561a2b9b6558fa +test-xz.mar update-settings.ini 180 644 9c29428fe4c81b8d541c8364de8a4109ee5321dd40e0d29368c7421ecda5ef3c +# test-bz2.mar sigs=1 product=thunderbird-comm-esr|100.0 +test-bz2.mar update.manifest 141 664 c717714abe8bb1cf173837f4592f8bf229c9884d71621bf8e5709e968ac929f3 +test-bz2.mar defaults/pref/channel-prefs.js 76 664 cdcb59e3add3e34b239316f365970b666383a07fca4e4595652b843dda53dc07 +test-bz2.mar Contents/MacOS/defaults/pref/channel-prefs.js 76 664 cdcb59e3add3e34b239316f365970b666383a07fca4e4595652b843dda53dc07 +test-bz2.mar update-settings.ini 152 664 6c4d6d9824a6846ce95d4fddcee9e177c0f90ad052fa683bffe3167abc358bb9 +test-bz2.mar Contents/MacOS/update-settings.ini 152 664 6c4d6d9824a6846ce95d4fddcee9e177c0f90ad052fa683bffe3167abc358bb9 +# mardor-old.mar sigs=-1 product=- +mardor-old.mar a.txt 10 666 4ea8280daa0d618b98c4b85e6db2fff92b19b99318603d0037b426342fbb1e13 +mardor-old.mar dir/b.bin 1024 666 785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9 +# mardor-product.mar sigs=0 product=release|130.0 +mardor-product.mar a.txt 10 666 4ea8280daa0d618b98c4b85e6db2fff92b19b99318603d0037b426342fbb1e13 +mardor-product.mar dir/b.bin 1024 666 785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9 +# mardor-xz.mar sigs=0 product=release|130.0 +mardor-xz.mar a.txt 68 666 2ae9565f7893f85690c7c067e260c0b6051e46a07a3f9b11d9e48933a26e7b27 +mardor-xz.mar dir/b.bin 308 666 24ab21bc489b17266f9f380ce392136d380e05d12725f2e1eddab587ef73ba2c diff --git a/Compression.Tests/Mar/ReferenceVectors/mardor-old.mar b/Compression.Tests/Mar/ReferenceVectors/mardor-old.mar new file mode 100644 index 000000000..c3abb6f82 Binary files /dev/null and b/Compression.Tests/Mar/ReferenceVectors/mardor-old.mar differ diff --git a/Compression.Tests/Mar/ReferenceVectors/mardor-product.mar b/Compression.Tests/Mar/ReferenceVectors/mardor-product.mar new file mode 100644 index 000000000..2eee69ec7 Binary files /dev/null and b/Compression.Tests/Mar/ReferenceVectors/mardor-product.mar differ diff --git a/Compression.Tests/Mar/ReferenceVectors/mardor-xz.mar b/Compression.Tests/Mar/ReferenceVectors/mardor-xz.mar new file mode 100644 index 000000000..f95ce477f Binary files /dev/null and b/Compression.Tests/Mar/ReferenceVectors/mardor-xz.mar differ diff --git a/Compression.Tests/Mar/ReferenceVectors/test-bz2.mar b/Compression.Tests/Mar/ReferenceVectors/test-bz2.mar new file mode 100644 index 000000000..77fac2813 Binary files /dev/null and b/Compression.Tests/Mar/ReferenceVectors/test-bz2.mar differ diff --git a/Compression.Tests/Mar/ReferenceVectors/test-xz.mar b/Compression.Tests/Mar/ReferenceVectors/test-xz.mar new file mode 100644 index 000000000..fec035283 Binary files /dev/null and b/Compression.Tests/Mar/ReferenceVectors/test-xz.mar differ diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Mar/MarFormatDescriptor.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Mar/MarFormatDescriptor.cs new file mode 100644 index 000000000..f9ceb5015 --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Mar/MarFormatDescriptor.cs @@ -0,0 +1,108 @@ +#pragma warning disable CS1591 +using Compression.Registry; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.Mar; + +/// +/// Mozilla ARchive (MAR) — the container of Firefox / Thunderbird update packages. +/// +/// References: +/// +/// https://wiki.mozilla.org/Software_Update:MAR — Mozilla's format description +/// https://github.com/mozilla-releng/build-mar — mardor, Mozilla release engineering's MAR reader/writer (the oracle) +/// +/// +public sealed class MarFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable { + + public string Id => "Mar"; + public string DisplayName => "MAR"; + public FormatCategory Category => FormatCategory.Archive; + // R/W: unsigned archives are rebuilt with their product-information / additional sections and + // member permission bits intact; signed archives refuse edits (the signatures would no longer verify). + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | + FormatCapabilities.CanModify | FormatCapabilities.CanTest | + FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; + public string DefaultExtension => ".mar"; + public IReadOnlyList Extensions => [".mar"]; + public IReadOnlyList CompoundExtensions => []; + public IReadOnlyList MagicSignatures => [new("MAR1"u8.ToArray(), Confidence: 0.9)]; + public IReadOnlyList Methods => [new("store", "Stored")]; + public string? TarCompressionFormatId => null; + public AlgorithmFamily Family => AlgorithmFamily.Archive; + public string Description => + "Mozilla update archive. Members are listed and extracted exactly as stored (update packages hold them " + + "bzip2- or xz-compressed; the Method column names which). Create options: channel + productversion " + + "write a product-information block."; + + public List List(Stream stream, string? password) { + var r = new MarReader(stream); + var list = new List(r.Entries.Count); + for (var i = 0; i < r.Entries.Count; ++i) { + var e = r.Entries[i]; + list.Add(new ArchiveEntryInfo(i, e.Name, e.Size, e.Size, MarReader.DescribeStoredData(Head(stream, e)), false, false, null)); + } + return list; + } + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + var r = new MarReader(stream); + foreach (var e in r.Entries) { + if (files != null && !MatchesFilter(e.Name, files)) continue; + WriteFile(outputDir, e.Name, r.Extract(e)); + } + } + + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + var channel = options.GetString("channel"); + var version = options.GetString("productversion"); + if ((channel == null) != (version == null)) + throw new ArgumentException("MAR: channel and productversion must be given together."); + IReadOnlyList? sections = channel != null ? [MarWriter.ProductInformation(channel, version!)] : null; + var files = FilesOnly(inputs).Select(f => (f.Name, f.Data, MarWriter.DefaultFlags)).ToList(); + MarWriter.Write(output, files, sections); + } + + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(inputs); + Rewrite(archive, files => { + foreach (var (name, data) in FilesOnly(inputs)) { + var existing = files.FindIndex(f => string.Equals(f.Name, name, StringComparison.Ordinal)); + if (existing >= 0) files[existing] = (name, data, files[existing].Flags); + else files.Add((name, data, MarWriter.DefaultFlags)); + } + }); + } + + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(entryNames); + var drop = new HashSet(entryNames, StringComparer.Ordinal); + Rewrite(archive, files => files.RemoveAll(f => drop.Contains(f.Name))); + } + + private static void Rewrite(Stream archive, Action> edit) { + ArgumentNullException.ThrowIfNull(archive); + archive.Position = 0; + var r = new MarReader(archive); + if (r.Signatures.Count > 0) + throw new NotSupportedException("MAR: the archive is signed; editing it would invalidate its signatures."); + var files = r.Entries.Select(e => (e.Name, r.Extract(e), e.Flags)).ToList(); + edit(files); + using var rebuilt = new MemoryStream(); + MarWriter.Write(rebuilt, files, r.HasSignatureBlock ? r.AdditionalSections : null); + archive.Position = 0; + archive.SetLength(0); + rebuilt.Position = 0; + rebuilt.CopyTo(archive); + archive.Flush(); + } + + private static byte[] Head(Stream stream, MarEntry e) { + var n = (int)Math.Min(6u, e.Size); + var buf = new byte[n]; + stream.Position = e.Offset; + stream.ReadExactly(buf); + return buf; + } +} diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Mar/MarReader.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Mar/MarReader.cs new file mode 100644 index 000000000..51c2134b0 --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Mar/MarReader.cs @@ -0,0 +1,182 @@ +using System.Buffers.Binary; +using System.Text; + +namespace FileFormat.Mar; + +/// One file of a Mozilla ARchive: where its bytes sit, how many there are, and its permission bits. +public sealed record MarEntry(string Name, uint Offset, uint Size, uint Flags); + +/// A signature block entry (algorithm 1 = RSA-PKCS1-SHA1, 2 = RSA-PKCS1-SHA384). +public sealed record MarSignature(uint AlgorithmId, byte[] Data); + +/// An additional-section block entry; id 1 is the product-information block. +public sealed record MarAdditionalSection(uint Id, byte[] Data) { + /// Product-information block id. + public const uint ProductInformationId = 1; +} + +/// +/// Reads a Mozilla ARchive (MAR), the container of Firefox/Thunderbird update packages. +/// +/// +/// Layout (all integers big-endian), per https://wiki.mozilla.org/Software_Update:MAR and +/// Mozilla's modules/libmar: +/// +/// "MAR1" UInt32 indexOffset +/// [UInt64 fileSize UInt32 sigCount {UInt32 algorithm UInt32 size byte[size]}*] (signed-era MARs) +/// [UInt32 sectionCount {UInt32 size(incl. these 8) UInt32 id byte[size-8]}*] (signed-era MARs) +/// file data … +/// index: UInt32 indexSize {UInt32 offset UInt32 size UInt32 flags name\0}* +/// +/// Old MARs omit both optional blocks; as Mozilla's own readers do, their presence is inferred from +/// where the first file's data starts. Member bytes are returned exactly as stored — update +/// packages hold each file bzip2- or xz-compressed, and like mar -x we do not unwrap that. +/// +public sealed class MarReader { + + /// The "MAR1" magic. + public static ReadOnlySpan Magic => "MAR1"u8; + + private readonly Stream _stream; + + public MarReader(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + this._stream = stream; + var length = stream.Length; + Span head = stackalloc byte[8]; + stream.Position = 0; + stream.ReadExactly(head); + if (!head[..4].SequenceEqual(Magic)) + throw new InvalidDataException("Not a MAR archive (missing MAR1 magic)."); + this.IndexOffset = BinaryPrimitives.ReadUInt32BigEndian(head[4..]); + if (this.IndexOffset < 8 || this.IndexOffset > length - 4) + throw new InvalidDataException($"MAR index offset {this.IndexOffset} lies outside the file."); + + stream.Position = this.IndexOffset; + var indexSize = ReadUInt32(); + if (indexSize > length - this.IndexOffset - 4) + throw new InvalidDataException("MAR index overruns the file."); + var index = new byte[indexSize]; + stream.ReadExactly(index); + + var entries = new List(); + var pos = 0; + while (pos < index.Length) { + if (index.Length - pos < 13) + throw new InvalidDataException("Truncated MAR index entry."); + var offset = BinaryPrimitives.ReadUInt32BigEndian(index.AsSpan(pos)); + var size = BinaryPrimitives.ReadUInt32BigEndian(index.AsSpan(pos + 4)); + var flags = BinaryPrimitives.ReadUInt32BigEndian(index.AsSpan(pos + 8)); + pos += 12; + var end = Array.IndexOf(index, (byte)0, pos); + if (end < 0) + throw new InvalidDataException("Unterminated MAR index entry name."); + var name = Encoding.UTF8.GetString(index, pos, end - pos); + pos = end + 1; + if (offset < 8 || (ulong)offset + size > this.IndexOffset) + throw new InvalidDataException($"MAR entry '{name}' points outside the data area."); + entries.Add(new MarEntry(name, offset, size, flags)); + } + this.Entries = entries; + + // Optional blocks: present when the earliest file data starts beyond the 8-byte header. + var dataOffset = entries.Count == 0 ? this.IndexOffset : Math.Max(8u, entries.Min(e => e.Offset)); + if (dataOffset <= 8) + return; + + stream.Position = 8; + if (dataOffset < 20) + throw new InvalidDataException("MAR data starts inside the signature block header."); + this.DeclaredFileSize = ReadUInt64(); + var sigCount = ReadUInt32(); + var sigs = new List(); + for (var i = 0u; i < sigCount; ++i) { + if (stream.Position + 8 > dataOffset) + throw new InvalidDataException("MAR signature block overruns the file data."); + var algorithm = ReadUInt32(); + var size = ReadUInt32(); + if (stream.Position + size > dataOffset) + throw new InvalidDataException("MAR signature overruns the file data."); + var data = new byte[size]; + stream.ReadExactly(data); + sigs.Add(new MarSignature(algorithm, data)); + } + this.Signatures = sigs; + + if (stream.Position + 4 > dataOffset) + return; + var sectionCount = ReadUInt32(); + var sections = new List(); + for (var i = 0u; i < sectionCount; ++i) { + if (stream.Position + 8 > dataOffset) + throw new InvalidDataException("MAR additional-section block overruns the file data."); + var size = ReadUInt32(); + var id = ReadUInt32(); + if (size < 8 || stream.Position + size - 8 > dataOffset) + throw new InvalidDataException("MAR additional section overruns the file data."); + var data = new byte[size - 8]; + stream.ReadExactly(data); + sections.Add(new MarAdditionalSection(id, data)); + } + this.AdditionalSections = sections; + } + + /// Offset of the index. + public uint IndexOffset { get; } + + /// Files in index order. + public IReadOnlyList Entries { get; } + + /// Whether the archive carries the signature/additional-section header blocks. + public bool HasSignatureBlock => this.DeclaredFileSize.HasValue; + + /// The total file size the signature block declares, when present. + public ulong? DeclaredFileSize { get; } + + /// Signatures; empty for unsigned or old-style archives. + public IReadOnlyList Signatures { get; } = []; + + /// Additional sections (product information and any unknown ones), in file order. + public IReadOnlyList AdditionalSections { get; } = []; + + /// Channel and product version of the product-information block, when present. + public (string Channel, string ProductVersion)? ProductInformation { + get { + var block = this.AdditionalSections.FirstOrDefault(s => s.Id == MarAdditionalSection.ProductInformationId); + if (block == null) + return null; + var parts = Encoding.ASCII.GetString(block.Data).Split('\0'); + return parts.Length >= 2 ? (parts[0], parts[1]) : null; + } + } + + /// Returns a member's stored bytes. + public byte[] Extract(MarEntry entry) { + ArgumentNullException.ThrowIfNull(entry); + var data = new byte[entry.Size]; + this._stream.Position = entry.Offset; + this._stream.ReadExactly(data); + return data; + } + + /// + /// Names the compression the update tooling applied to a member ("xz", "bzip2") by its magic, + /// or "stored" — informational only: extraction always returns the stored bytes. + /// + public static string DescribeStoredData(ReadOnlySpan head) + => head.StartsWith((ReadOnlySpan)[0xFD, (byte)'7', (byte)'z', (byte)'X', (byte)'Z', 0x00]) ? "xz" + : head.StartsWith("BZh"u8) ? "bzip2" + : "stored"; + + private uint ReadUInt32() { + Span b = stackalloc byte[4]; + this._stream.ReadExactly(b); + return BinaryPrimitives.ReadUInt32BigEndian(b); + } + + private ulong ReadUInt64() { + Span b = stackalloc byte[8]; + this._stream.ReadExactly(b); + return BinaryPrimitives.ReadUInt64BigEndian(b); + } +} diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Mar/MarWriter.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Mar/MarWriter.cs new file mode 100644 index 000000000..1d05e78d4 --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Mar/MarWriter.cs @@ -0,0 +1,78 @@ +using System.Buffers.Binary; +using System.Text; + +namespace FileFormat.Mar; + +/// +/// Writes an unsigned Mozilla ARchive. Without an additional-section list it writes the original +/// layout (header, data, index); with one (even empty) — e.g. a product-information block — it writes +/// the signed-era layout with an empty signature block, exactly as Mozilla's mar -H channel -V version -c +/// and mardor do for unsigned packages. +/// +public static class MarWriter { + + /// Default permission bits for a member with none supplied (rw-r--r--). + public const uint DefaultFlags = 0x1A4; + + /// Builds the product-information section payload (channel and version, each NUL-terminated). + public static MarAdditionalSection ProductInformation(string channel, string productVersion) + => new(MarAdditionalSection.ProductInformationId, Encoding.ASCII.GetBytes(channel + "\0" + productVersion + "\0")); + + public static void Write(Stream output, IReadOnlyList<(string Name, byte[] Data, uint Flags)> files, + IReadOnlyList? additionalSections = null) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(files); + var signedEra = additionalSections != null; + var sections = additionalSections ?? []; + + using var ms = new MemoryStream(); + ms.Write("MAR1"u8); + WriteUInt32(ms, 0); // index offset, patched below + if (signedEra) { + WriteUInt64(ms, 0); // total file size, patched below + WriteUInt32(ms, 0); // no signatures + WriteUInt32(ms, (uint)sections.Count); + foreach (var s in sections) { + WriteUInt32(ms, checked((uint)s.Data.Length + 8)); + WriteUInt32(ms, s.Id); + ms.Write(s.Data); + } + } + + var index = new MemoryStream(); + foreach (var (name, data, flags) in files) { + if (string.IsNullOrEmpty(name) || name.Contains('\0')) + throw new ArgumentException($"Invalid MAR member name '{name}'.", nameof(files)); + var offset = checked((uint)ms.Position); + ms.Write(data); + WriteUInt32(index, offset); + WriteUInt32(index, checked((uint)data.Length)); + WriteUInt32(index, flags); + index.Write(Encoding.UTF8.GetBytes(name)); + index.WriteByte(0); + } + + var indexOffset = checked((uint)ms.Position); + WriteUInt32(ms, checked((uint)index.Length)); + index.Position = 0; + index.CopyTo(ms); + + var image = ms.GetBuffer().AsSpan(0, (int)ms.Length); + BinaryPrimitives.WriteUInt32BigEndian(image[4..], indexOffset); + if (signedEra) + BinaryPrimitives.WriteUInt64BigEndian(image[8..], (ulong)image.Length); + output.Write(image); + } + + private static void WriteUInt32(Stream s, uint v) { + Span b = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(b, v); + s.Write(b); + } + + private static void WriteUInt64(Stream s, ulong v) { + Span b = stackalloc byte[8]; + BinaryPrimitives.WriteUInt64BigEndian(b, v); + s.Write(b); + } +} diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md index 9357f828e..d3f6d7f96 100644 --- a/Hawkynt.FileFormats.Archives/README.md +++ b/Hawkynt.FileFormats.Archives/README.md @@ -215,6 +215,7 @@ Every State, Test, Maintenance, Compress/Decompress and Demux/Mux/Remux cell is | [KMZ](https://en.wikipedia.org/wiki/Keyhole_Markup_Language) | `Kmz` | `.kmz` | R/W | ✅ | defrag · wipe | | [developers.google.com](https://developers.google.com/kml/documentation) | | [LIT (Microsoft Reader)](https://en.wikipedia.org/wiki/Microsoft_Reader) | `Lit` | `.lit` | R | ✅ | — | | [Archive Team](http://fileformats.archiveteam.org/wiki/Microsoft_Reader) | | [MAFF](https://en.wikipedia.org/wiki/Mozilla_Archive_Format) | `Maff` | `.maff` | R/W | ✅ | defrag · wipe | | [maf.mozdev.org](http://maf.mozdev.org/maff-specification.html) | +| [MAR](https://wiki.mozilla.org/Software_Update:MAR) | `Mar` | `.mar` | R/W | ✅ | — | Firefox/Thunderbird update packages; members are listed and extracted as stored (bzip2/xz named in Method); writes mardor-identical archives; signed archives refuse edits | [wiki.mozilla.org](https://wiki.mozilla.org/Software_Update:MAR) | | [mbox (Unix mailbox)](https://en.wikipedia.org/wiki/Mbox) | `Mbox` | `.mbox` `.mbx` | R/W | ✅ | — | Entries are message_NN.eml | [RFC](https://www.rfc-editor.org/rfc/rfc4155) | | [MOBI / AZW](https://en.wikipedia.org/wiki/Mobipocket) | `Mobi` | `.mobi` `.prc` `.azw` `.azw3` | WORM | ✅ | — | Creates MOBI 7 books from one UTF-8 HTML document (stored or PalmDOC), accepted by KindleUnpack; reads stored and PalmDOC text with trailing entries; HUFF/CDIC and DRM text stay raw records | [wiki.mobileread.com](https://wiki.mobileread.com/wiki/MOBI) | | [MSG](https://en.wikipedia.org/wiki/MSG_(file_format)) | `Msg` | `.msg` | R/W | ✅ | wipe | CFB envelope; MAPI properties are not synthesised | [Microsoft Learn](https://learn.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxmsg/) | @@ -544,7 +545,7 @@ Use it when a .NET process needs to enumerate, extract, test, create, edit or in -Every public and protected member of all 2571 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). +Every public and protected member of all 2577 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). diff --git a/Hawkynt.FileFormats.Archives/REFERENCE.md b/Hawkynt.FileFormats.Archives/REFERENCE.md index 464eb0ba6..525cf9bfb 100644 --- a/Hawkynt.FileFormats.Archives/REFERENCE.md +++ b/Hawkynt.FileFormats.Archives/REFERENCE.md @@ -20525,6 +20525,104 @@ Inherits `ModifiableZipContainerFormatDescriptor`. Implements `IArchiveCreatable | `DisplayName` | `override string DisplayName { get; }` | Gets the display name. | | `Extensions` | `override IReadOnlyList Extensions { get; }` | Gets the extensions. The first entry doubles as `DefaultExtension`. | +### Namespace `FileFormat.Mar` + +[`MarAdditionalSection`](#maradditionalsection) · [`MarEntry`](#marentry) · [`MarFormatDescriptor`](#marformatdescriptor) · [`MarReader`](#marreader) · [`MarSignature`](#marsignature) · [`MarWriter`](#marwriter) + +#### `MarAdditionalSection` + +An additional-section block entry; id 1 is the product-information block. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `MarAdditionalSection` | `MarAdditionalSection(uint Id, byte[] Data)` | An additional-section block entry; id 1 is the product-information block. | +| `ProductInformationId` | `const uint ProductInformationId` | Product-information block id. | +| `Data` | `byte[] Data { get; init; }` | | +| `Id` | `uint Id { get; init; }` | | + +#### `MarEntry` + +One file of a Mozilla ARchive: where its bytes sit, how many there are, and its permission bits. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `MarEntry` | `MarEntry(string Name, uint Offset, uint Size, uint Flags)` | One file of a Mozilla ARchive: where its bytes sit, how many there are, and its permission bits. | +| `Flags` | `uint Flags { get; init; }` | | +| `Name` | `string Name { get; init; }` | | +| `Offset` | `uint Offset { get; init; }` | | +| `Size` | `uint Size { get; init; }` | | + +#### `MarFormatDescriptor` + +Mozilla ARchive (MAR) — the container of Firefox / Thunderbird update packages. References: `https://wiki.mozilla.org/Software_Update:MAR` — Mozilla's format description`https://github.com/mozilla-releng/build-mar` — mardor, Mozilla release engineering's MAR reader/writer (the oracle) + +Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `MarFormatDescriptor` | `MarFormatDescriptor()` | | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | | + +#### `MarReader` + +Reads a Mozilla ARchive (MAR), the container of Firefox/Thunderbird update packages. + +| Member | Signature | Summary | +| --- | --- | --- | +| `MarReader` | `MarReader(Stream stream)` | | +| `AdditionalSections` | `IReadOnlyList AdditionalSections { get; }` | Additional sections (product information and any unknown ones), in file order. | +| `DeclaredFileSize` | `ulong? DeclaredFileSize { get; }` | The total file size the signature block declares, when present. | +| `Entries` | `IReadOnlyList Entries { get; }` | Files in index order. | +| `HasSignatureBlock` | `bool HasSignatureBlock { get; }` | Whether the archive carries the signature/additional-section header blocks. | +| `IndexOffset` | `uint IndexOffset { get; }` | Offset of the index. | +| `Magic` | `static ReadOnlySpan Magic { get; }` | The "MAR1" magic. | +| `ProductInformation` | `ValueTuple? ProductInformation { get; }` | Channel and product version of the product-information block, when present. | +| `Signatures` | `IReadOnlyList Signatures { get; }` | Signatures; empty for unsigned or old-style archives. | +| `DescribeStoredData` | `static string DescribeStoredData(ReadOnlySpan head)` | Names the compression the update tooling applied to a member ("xz", "bzip2") by its magic, or "stored" — informational only: extraction always returns the stored bytes. | +| `Extract` | `byte[] Extract(MarEntry entry)` | Returns a member's stored bytes. | + +#### `MarSignature` + +A signature block entry (algorithm 1 = RSA-PKCS1-SHA1, 2 = RSA-PKCS1-SHA384). + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `MarSignature` | `MarSignature(uint AlgorithmId, byte[] Data)` | A signature block entry (algorithm 1 = RSA-PKCS1-SHA1, 2 = RSA-PKCS1-SHA384). | +| `AlgorithmId` | `uint AlgorithmId { get; init; }` | | +| `Data` | `byte[] Data { get; init; }` | | + +#### `MarWriter` + +Writes an unsigned Mozilla ARchive. Without an additional-section list it writes the original layout (header, data, index); with one (even empty) — e.g. a product-information block — it writes the signed-era layout with an empty signature block, exactly as Mozilla's `mar -H channel -V version -c` and mardor do for unsigned packages. + +| Member | Signature | Summary | +| --- | --- | --- | +| `DefaultFlags` | `const uint DefaultFlags` | Default permission bits for a member with none supplied (rw-r--r--). | +| `ProductInformation` | `static MarAdditionalSection ProductInformation(string channel, string productVersion)` | Builds the product-information section payload (channel and version, each NUL-terminated). | +| `Write` | `static void Write(Stream output, IReadOnlyList> files, IReadOnlyList additionalSections = null)` | | + ### Namespace `FileFormat.Matlab` [`MatlabArrayInfo`](#matlabarrayinfo) · [`MatlabConstants`](#matlabconstants) · [`MatlabFormatDescriptor`](#matlabformatdescriptor) · [`MatlabReader`](#matlabreader) diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs index 6c7d4b452..1c9e17266 100644 --- a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs @@ -33,6 +33,7 @@ public static class FirmwareHexWriter { /// one the reader rendered. An input list with no payload describes an image /// with no data, which both formats can write. /// + /// More than one non-metadata payload is supplied. public static FirmwareImage ImageFrom(IReadOnlyList inputs, string sourceFormat) { ArgumentNullException.ThrowIfNull(inputs); @@ -53,7 +54,9 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st } // Any other single input is the payload, whatever it is called: a caller // converting a .bin into a .hex should not have to rename it first. - payload ??= input.ReadContent(); + if (payload is not null) + throw new InvalidDataException($"{sourceFormat} represents one logical firmware payload; multiple payload inputs are ambiguous."); + payload = input.ReadContent(); } List<(uint Address, byte[] Data)> segments; diff --git a/docs/catalogue/reference-formats.csv b/docs/catalogue/reference-formats.csv index d168a2e19..3748bff24 100644 --- a/docs/catalogue/reference-formats.csv +++ b/docs/catalogue/reference-formats.csv @@ -68,7 +68,7 @@ Electron asar,asar,archive,,,,,R,,,1,https://github.com/electron/asar,Asar,Y,Y,, BCM,bcm,compression,,,,,R,,,1,https://github.com/encode84/bcm,Bcm,Y,Y,,OK,, Split file (.001),001;002,archive,R,,R,,R,,,3,,SplitFile,Y,Y,,OK,, Apple Archive (.aar),aar,archive,,,,,,,Y,1,,,,,,MISSING,,https://developer.apple.com/documentation/applearchive -Mozilla ARchive (MAR),mar;mac,archive,,,,,,fmt/1483,Y,2,,,,,,MISSING,,https://wiki.mozilla.org/Software_Update:MAR +Mozilla ARchive (MAR),mar;mac,archive,,,,,,fmt/1483,Y,2,https://wiki.mozilla.org/Software_Update:MAR,Mar,Y,Y,Y,OK,, SeqBox,sbx,archive,,,,,,,Y,1,,,,,,MISSING,,https://github.com/MarcoPon/SeqBox AFA,afa,archive,,,,,,,Y,1,,,,,,MISSING,, B1,b1,archive,,,,,,,Y,1,,,,,,MISSING,,