+ IExpress and MSU; CAB LZX/MSZIP across frames

Compression.Core/Deflate/DeflateDecompressor.cs

@@ -139,6 +139,23 @@ public static byte[] Decompress(ReadOnlySpan<byte> compressedData) {
     return new DeflateDecompressor(compressedData.ToArray(), capacity).DecompressAll();
   }
 
+  /// <summary>
+  /// Decompresses DEFLATE data whose matches may reach back into <paramref name="dictionary"/>,
+  /// the bytes that precede it (MSZIP keeps the previous block's output as history).
+  /// </summary>
+  /// <param name="compressedData">The DEFLATE compressed data.</param>
+  /// <param name="dictionary">Preceding output; only the last 32 KiB can be referenced.</param>
+  /// <returns>The decompressed data, without the dictionary.</returns>
+  public static byte[] Decompress(ReadOnlySpan<byte> compressedData, ReadOnlySpan<byte> dictionary) {
+    if (dictionary.Length > WindowSize)
+      dictionary = dictionary[^WindowSize..];
+    var capacity = (int)Math.Clamp(compressedData.Length * 4L + dictionary.Length, 4096 + dictionary.Length, Array.MaxLength);
+    var d = new DeflateDecompressor(compressedData.ToArray(), capacity);
+    dictionary.CopyTo(d._buf);
+    d._wpos = d._rpos = dictionary.Length;
+    return d.DecompressAll();
+  }
+
   /// <summary>
   /// Decompresses data from the input stream into the provided buffer.
   /// Returns the number of bytes written. Returns 0 when decompression is complete.

Compression.Core/Deflate/MsZipDecompressor.cs

@@ -108,4 +108,18 @@ public static byte[] DecompressBlock(ReadOnlySpan<byte> block) {
 
     return DeflateDecompressor.Decompress(block[2..]);
   }
+
+  /// <summary>
+  /// Decompresses one MSZIP block of a folder: the Deflate data may refer back up to 32 KiB
+  /// into <paramref name="history"/>, the output of the folder's previous blocks ([MS-MCI] 2.2:
+  /// the history buffer is not reset between blocks).
+  /// </summary>
+  /// <param name="block">The block, including its "CK" prefix.</param>
+  /// <param name="history">The folder's output so far (only the last 32 KiB matter).</param>
+  /// <returns>The block's uncompressed bytes.</returns>
+  public static byte[] DecompressBlock(ReadOnlySpan<byte> block, ReadOnlySpan<byte> history) {
+    if (block.Length < 2 || block[0] != MsZipCompressor.SignatureByte0 || block[1] != MsZipCompressor.SignatureByte1)
+      throw new InvalidDataException("MSZIP: invalid block signature.");
+    return DeflateDecompressor.Decompress(block[2..], history);
+  }
 }

Compression.Core/Dictionary/Lzx/LzxDecompressor.cs

@@ -60,6 +60,14 @@
   private readonly LzxStreamFormat _format;
   private bool _readStreamHeader;
 
+  // A block may span several calls (cabinet frames are 32 KiB, blocks are not bounded by them),
+  // and so may the match that straddles a frame boundary.
+  private int _blockType;
+  private int _blockSize;
+  private int _blockRemaining;
+  private int _pendingMatchLength;
+  private int _pendingMatchOffset;
+
   // E8 call-instruction translation state
   private readonly bool _e8Translation;
   private long _totalDecompressedBytes;
@@ -80,7 +88,7 @@
   /// Thrown when <paramref name="windowBits"/> is outside [15, 21].
   /// </exception>
   public LzxDecompressor(Stream input, int windowBits = 15, bool e8Translation = false,
       LzxStreamFormat format = LzxStreamFormat.Wim) {
     ArgumentNullException.ThrowIfNull(input);
     ArgumentOutOfRangeException.ThrowIfLessThan(windowBits, LzxConstants.MinWindowBits);
     ArgumentOutOfRangeException.ThrowIfGreaterThan(windowBits, LzxConstants.MaxWindowBits);
@@ -128,41 +136,40 @@
     }
 
     while (outPos < uncompressedSize) {
-      var blockType = (int)this.ReadBits(3);
-
-      // Block size: twenty-four bits in a cabinet; in a WIM, one bit meaning
-      // "the usual 32 768" or sixteen bits otherwise.
-      int blockSize;
-      if (this._format == LzxStreamFormat.Cab)
-        blockSize = (int)this.ReadBits(24);
-      else if (this.ReadBits(1) == 1)
-        blockSize = LzxConstants.DefaultBlockSize;
-      else
-        blockSize = (int)this.ReadBits(16);
-
-      blockSize = Math.Min(blockSize, uncompressedSize - outPos);
-
-      switch (blockType) {
-        case LzxConstants.BlockTypeVerbatim:
-          this.ReadVerbatimBlockHeader();
-          this.DecodeBlock(false, output, outPos, blockSize);
-          break;
-
-        case LzxConstants.BlockTypeAligned:
-          this.ReadAlignedBlockHeader();
-          this.DecodeBlock(true, output, outPos, blockSize);
-          break;
-
-        case LzxConstants.BlockTypeUncompressed:
-          this.DecodeUncompressedBlock(output, outPos, blockSize);
-          break;
-
-        default:
-          ThrowInvalidBlockType(blockType);
-          break;
+      var starting = this._blockRemaining == 0;
+      if (starting) {
+        this._blockType = (int)this.ReadBits(3);
+
+        // Block size: twenty-four bits in a cabinet; in a WIM, one bit meaning
+        // "the usual 32 768" or sixteen bits otherwise.
+        if (this._format == LzxStreamFormat.Cab)
+          this._blockSize = (int)this.ReadBits(24);
+        else if (this.ReadBits(1) == 1)
+          this._blockSize = LzxConstants.DefaultBlockSize;
+        else
+          this._blockSize = (int)this.ReadBits(16);
+        if (this._blockSize == 0)
+          throw new InvalidDataException("LZX block of length zero.");
+        this._blockRemaining = this._blockSize;
+
+        switch (this._blockType) {
+          case LzxConstants.BlockTypeVerbatim: this.ReadVerbatimBlockHeader(); break;
+          case LzxConstants.BlockTypeAligned: this.ReadAlignedBlockHeader(); break;
+          case LzxConstants.BlockTypeUncompressed: break;
+          default: ThrowInvalidBlockType(this._blockType); break;
+        }
       }
 
-      outPos += blockSize;
+      // The block may run past what this call is asked for; the rest of it follows in the
+      // next call (the next cabinet frame).
+      var count = Math.Min(this._blockRemaining, uncompressedSize - outPos);
+      if (this._blockType == LzxConstants.BlockTypeUncompressed)
+        this.DecodeUncompressedBlock(output, outPos, count, starting, count == this._blockRemaining);
+      else
+        this.DecodeBlock(this._blockType == LzxConstants.BlockTypeAligned, output, outPos, count);
+
+      outPos += count;
+      this._blockRemaining -= count;
     }
 
     if (this._e8Translation && uncompressedSize > 10)
@@ -197,22 +204,24 @@
     this.ReadVerbatimBlockHeader();
   }
 
-  private void DecodeUncompressedBlock(byte[] output, int outPos, int blockSize) {
-    // Discard partial bits to align to the next 16-bit word boundary
-    this.AlignTo16Bits();
+  private void DecodeUncompressedBlock(byte[] output, int outPos, int blockSize, bool starting, bool ending) {
+    if (starting) {
+      // Discard partial bits to align to the next 16-bit word boundary
+      this.AlignTo16Bits();
 
-    // R0/R1/R2 are stored as three LE 32-bit integers
-    this._r0 = this.ReadRawInt32LE();
-    this._r1 = this.ReadRawInt32LE();
-    this._r2 = this.ReadRawInt32LE();
+      // R0/R1/R2 are stored as three LE 32-bit integers
+      this._r0 = this.ReadRawInt32LE();
+      this._r1 = this.ReadRawInt32LE();
+      this._r2 = this.ReadRawInt32LE();
+    }
 
     // Copy raw bytes
     var bytesRead = this._input.Read(output, outPos, blockSize);
     if (bytesRead < blockSize)
       ThrowUnexpectedEndOfStream();
 
     // If block length is odd, consume one padding byte so stream stays word-aligned
-    if ((blockSize & 1) != 0)
+    if (ending && (this._blockSize & 1) != 0)
       this._input.ReadByte();
 
     // Mirror into the circular window
@@ -291,6 +300,20 @@
 
   private void DecodeBlock(bool isAligned, byte[] output, int outPos, int blockSize) {
     var end = outPos + blockSize;
+
+    // The tail of a match that the previous call's end cut off.
+    if (this._pendingMatchLength > 0) {
+      var src = (this._windowPos - this._pendingMatchOffset + this._windowSize) & (this._windowSize - 1);
+      while (this._pendingMatchLength > 0 && outPos < end) {
+        var b = this._window[src];
+        output[outPos++] = b;
+        this._window[this._windowPos] = b;
+        this._windowPos = (this._windowPos + 1) & (this._windowSize - 1);
+        src = (src + 1) & (this._windowSize - 1);
+        --this._pendingMatchLength;
+      }
+    }
+
     while (outPos < end) {
       var mainSym = this.DecodeSymbol(this._mainTable!, LzxDecompressor.MainTableBits, this._mainLengths, this._numMainSymbols);
 
@@ -318,13 +341,18 @@
 
         // Copy from sliding window (handles run-length / overlapping copies)
         var srcPos = (this._windowPos - matchOffset + this._windowSize) & (this._windowSize - 1);
-        for (var i = 0; i < matchLength && outPos < end; ++i) {
+        var i = 0;
+        for (; i < matchLength && outPos < end; ++i) {
           var b = this._window[srcPos];
           output[outPos++] = b;
           this._window[this._windowPos] = b;
           this._windowPos = (this._windowPos + 1) & (this._windowSize - 1);
           srcPos = (srcPos + 1) & (this._windowSize - 1);
         }
+        if (i < matchLength) {
+          this._pendingMatchLength = matchLength - i;
+          this._pendingMatchOffset = matchOffset;
+        }
       }
     }
   }

Compression.Core/REFERENCE.md

@@ -390,6 +390,7 @@ Decompresses data in the DEFLATE format (RFC 1951).
 | `Decompress` | `int Decompress(Span<byte> output)` | Decompresses data into `output`, filling it unless the DEFLATE stream ends first. |
 | `Decompress` | `int Decompress(byte[] output, int offset, int count)` | Decompresses data from the input stream into the provided buffer. Returns the number of bytes written. Returns 0 when decompression is complete. |
 | `Decompress` | `static byte[] Decompress(ReadOnlySpan<byte> compressedData)` | Decompresses DEFLATE data in one shot. |
+| `Decompress` | `static byte[] Decompress(ReadOnlySpan<byte> compressedData, ReadOnlySpan<byte> dictionary)` | Decompresses DEFLATE data whose matches may reach back into `dictionary`, the bytes that precede it (MSZIP keeps the previous block's output as history). |
 | `ReadRemainder` | `int ReadRemainder(Span<byte> destination)` | Reads bytes that follow the end of the DEFLATE data — a container's trailer, or the next member — taking the decompressor's read-ahead first and then the underlying stream. |
 
 #### `DeflateHuffmanTable`
@@ -433,6 +434,7 @@ Decompresses data in the MSZIP format: a sequence of "CK"-prefixed Deflate block
 | --- | --- | --- |
 | `BlockSize` | `const int BlockSize` | The uncompressed block size limit: 32 768 bytes. |
 | `DecompressBlock` | `static byte[] DecompressBlock(ReadOnlySpan<byte> block)` | Decompresses a single MSZIP block (including its "CK" prefix) and returns the uncompressed bytes. |
+| `DecompressBlock` | `static byte[] DecompressBlock(ReadOnlySpan<byte> block, ReadOnlySpan<byte> history)` | Decompresses one MSZIP block of a folder: the Deflate data may refer back up to 32 KiB into `history`, the output of the folder's previous blocks ([MS-MCI] 2.2: the history buffer is not reset between blocks). |
 | `Decompress` | `static byte[] Decompress(ReadOnlySpan<byte> data, int uncompressedSize = -1)` | Decompresses an MSZIP byte stream. |
 
 #### `RawDeflateStream`

Compression.Lib/FormatDetector.cs

@@ -504,6 +504,9 @@ private static Format DetectInstaller(string path) {
       var buf = new byte[2];
       var isPe = fs.Read(buf, 0, 2) >= 2 && buf[0] == 'M' && buf[1] == 'Z';
 
+      // IExpress / wextract: the cabinet lives in the RT_RCDATA resource "CABINET".
+      if (isPe && FileFormat.IExpress.IExpressFormatDescriptor.IsIExpress(fs)) return Format.IExpress;
+
       // An NSIS first header sits right after the exe stub, at a multiple of 512 bytes (the
       // exehead itself only looks there), so a large installer keeps it far from the tail.
       if (HasNsisFirstHeader(fs)) return Format.Nsis;

Compression.Tests/Compression.Tests.csproj

@@ -75,6 +75,10 @@
     <EmbeddedResource Include="BcacheFs\ReferenceVectors\*.gz;BcacheFs\ReferenceVectors\manifest.txt">
       <LogicalName>BcacheFsVectors.%(Filename)%(Extension)</LogicalName>
     </EmbeddedResource>
+    <!-- A package built by Windows iexpress.exe, plus expand.exe's extraction; see IExpress/ReferenceVectors/README.md. -->
+    <EmbeddedResource Include="IExpress\ReferenceVectors\*.gz;IExpress\ReferenceVectors\oracle.tsv">
+      <LogicalName>IExpressVectors.%(Filename)%(Extension)</LogicalName>
+    </EmbeddedResource>
     <!-- Installers compiled by makensis 2.51 / 3.10, plus 7-Zip's extraction; see Nsis/ReferenceVectors/README.md. -->
     <EmbeddedResource Include="Nsis\ReferenceVectors\*.gz;Nsis\ReferenceVectors\oracle.tsv">
       <LogicalName>NsisVectors.%(Filename)%(Extension)</LogicalName>

Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs

@@ -0,0 +1,37 @@
+using Compression.Registry;
+using FileFormat.FirmwareHex;
+
+namespace Compression.Tests.FirmwareHex;
+
+[TestFixture]
+public sealed class FirmwareHexPayloadTests {
+  [TestCase(false)]
+  [TestCase(true)]
+  public void Create_RejectsMultiplePayloadsWithoutWritingOutput(bool emptyFirst) {
+    var descriptor = new IntelHexFormatDescriptor();
+    using var output = new MemoryStream();
+    var inputs = new[] {
+      ArchiveInputInfo.InMemory("first.bin", emptyFirst ? [] : [1, 2, 3]),
+      ArchiveInputInfo.InMemory("second.bin", [9, 8, 7]),
+    };
+
+    Assert.Throws<InvalidDataException>(() =>
+      ((IArchiveCreatable)descriptor).Create(output, inputs, new FormatCreateOptions()));
+    Assert.That(output.Length, Is.Zero);
+  }
+
+  [Test]
+  public void Add_RejectsAnotherPayloadWithoutChangingTheArchive() {
+    var descriptor = new IntelHexFormatDescriptor();
+    using var archive = new MemoryStream();
+    ((IArchiveCreatable)descriptor).Create(archive,
+      [ArchiveInputInfo.InMemory("firmware.bin", [1, 2, 3])], new FormatCreateOptions());
+    var original = archive.ToArray();
+    archive.Position = 0;
+
+    Assert.Throws<InvalidDataException>(() => ((IArchiveModifiable)descriptor).Add(archive,
+      [ArchiveInputInfo.InMemory("ADD.BIN", [9, 8, 7])]));
+    Assert.That(archive.ToArray(), Is.EqualTo(original));
+  }
+
+}

Compression.Tests/IExpress/IExpressReferenceVectorTests.cs

@@ -0,0 +1,88 @@
+using System.IO.Compression;
+using System.Security.Cryptography;
+using System.Text;
+using Compression.Lib;
+using FileFormat.IExpress;
+
+namespace Compression.Tests.IExpress;
+
+/// <summary>
+/// A package built by Windows iexpress.exe (see ReferenceVectors/README.md), checked against what
+/// Windows expand.exe extracts from its CABINET resource (oracle.tsv).
+/// </summary>
+[TestFixture]
+public class IExpressReferenceVectorTests {
+
+  private const string Fixture = "iexpress-lzx.exe";
+
+  private static byte[] Load() {
+    using var resource = typeof(IExpressReferenceVectorTests).Assembly.GetManifestResourceStream("IExpressVectors." + Fixture + ".gz")!;
+    using var gz = new GZipStream(resource, CompressionMode.Decompress); // BCL as the test-side oracle
+    using var ms = new MemoryStream();
+    gz.CopyTo(ms);
+    return ms.ToArray();
+  }
+
+  private static Dictionary<string, (long Size, string Sha)> Oracle() {
+    using var resource = typeof(IExpressReferenceVectorTests).Assembly.GetManifestResourceStream("IExpressVectors.oracle.tsv")!;
+    using var reader = new StreamReader(resource);
+    var result = new Dictionary<string, (long, string)>();
+    while (reader.ReadLine() is { } line)
+      if (line.Length > 0 && !line.StartsWith('#') && line.Split('\t') is var f && f[0] == Fixture)
+        result[f[1]] = (long.Parse(f[2], System.Globalization.CultureInfo.InvariantCulture), f[3]);
+    return result;
+  }
+
+  [Test]
+  public void GivenIExpressPackage_WhenListed_ThenTheCabinetFilesMatchExpand() {
+    var oracle = Oracle();
+    var entries = new IExpressFormatDescriptor().List(new MemoryStream(Load()), null).Where(e => e.Kind != "metadata").ToList();
+    Assert.That(entries.Select(e => e.Name), Is.EquivalentTo(oracle.Keys));
+    foreach (var e in entries)
+      Assert.That(e.OriginalSize, Is.EqualTo(oracle[e.Name].Size), e.Name);
+  }
+
+  [Test]
+  public void GivenIExpressPackage_WhenExtracted_ThenEveryFileMatchesExpand() {
+    var oracle = Oracle();
+    var dir = Path.Combine(Path.GetTempPath(), "cwb-iex-" + Guid.NewGuid().ToString("N"));
+    try {
+      new IExpressFormatDescriptor().Extract(new MemoryStream(Load()), dir, null, null);
+      foreach (var (name, (_, sha)) in oracle)
+        Assert.That(Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(Path.Combine(dir, name)))), Is.EqualTo(sha), name);
+      var ini = File.ReadAllText(Path.Combine(dir, "iexpress.ini"), Encoding.UTF8);
+      Assert.That(ini, Does.Contain("title=Fixture").And.Contain("parse_status=ok"));
+    } finally {
+      if (Directory.Exists(dir))
+        Directory.Delete(dir, recursive: true);
+    }
+  }
+
+  [Test]
+  public void GivenIExpressPackageOnDisk_WhenDetected_ThenIExpressIsChosen() {
+    var path = Path.Combine(Path.GetTempPath(), "cwb-iex-" + Guid.NewGuid().ToString("N") + ".exe");
+    try {
+      File.WriteAllBytes(path, Load());
+      Assert.That(FormatDetector.Detect(path), Is.EqualTo(FormatDetector.Format.IExpress));
+    } finally {
+      File.Delete(path);
+    }
+  }
+
+  [Test]
+  public void GivenPeWithoutCabinet_WhenProbed_ThenItIsNotIExpress() {
+    var data = Load();
+    // Rename the resource: "CABINET" (UTF-16) becomes "CABINEX".
+    var name = Encoding.Unicode.GetBytes("CABINET");
+    var at = data.AsSpan().IndexOf(name);
+    Assert.That(at, Is.GreaterThan(0));
+    data[at + 12] = (byte)'X';
+    Assert.That(IExpressFormatDescriptor.IsIExpress(new MemoryStream(data)), Is.False);
+    var entries = new IExpressFormatDescriptor().List(new MemoryStream(data), null);
+    Assert.That(entries.Select(e => e.Name), Is.EqualTo(new[] { "iexpress.ini" }));
+  }
+
+  [Test]
+  public void GivenNonPeBytes_WhenProbed_ThenItIsNotIExpress()
+    => Assert.That(IExpressFormatDescriptor.IsIExpress(new MemoryStream(new byte[100])), Is.False);
+}

Compression.Tests/IExpress/ReferenceVectors/README.md

@@ -0,0 +1,10 @@
+# IExpress reference package
+
+`iexpress-lzx.exe` was built by Windows' own `C:\Windows\System32\iexpress.exe` (Windows 11 Pro
+10.0.26200), never by CompressionWorkbench, from the SED file that `mkiexpress.py` writes
+(`python mkiexpress.py <workdir> iexpress-lzx`): an extract-only package of three files (a short
+text, 3000 seeded random bytes, 32000 bytes of a repeated pattern). It is stored gzip-compressed and
+otherwise unchanged.
+
+`oracle.tsv` is what Windows `expand.exe -F:*` writes from the package's `CABINET` resource (taken
+out with 7-Zip 26.03): name, size and SHA-256 per file.