diff --git a/Compression.Core/Deflate/DeflateDecompressor.cs b/Compression.Core/Deflate/DeflateDecompressor.cs index ca2d93897..8f82d507d 100644 --- a/Compression.Core/Deflate/DeflateDecompressor.cs +++ b/Compression.Core/Deflate/DeflateDecompressor.cs @@ -139,6 +139,23 @@ public static byte[] Decompress(ReadOnlySpan compressedData) { return new DeflateDecompressor(compressedData.ToArray(), capacity).DecompressAll(); } + /// + /// Decompresses DEFLATE data whose matches may reach back into , + /// the bytes that precede it (MSZIP keeps the previous block's output as history). + /// + /// The DEFLATE compressed data. + /// Preceding output; only the last 32 KiB can be referenced. + /// The decompressed data, without the dictionary. + public static byte[] Decompress(ReadOnlySpan compressedData, ReadOnlySpan dictionary) { + if (dictionary.Length > WindowSize) + dictionary = dictionary[^WindowSize..]; + var capacity = (int)Math.Clamp(compressedData.Length * 4L + dictionary.Length, 4096 + dictionary.Length, Array.MaxLength); + var d = new DeflateDecompressor(compressedData.ToArray(), capacity); + dictionary.CopyTo(d._buf); + d._wpos = d._rpos = dictionary.Length; + return d.DecompressAll(); + } + /// /// Decompresses data from the input stream into the provided buffer. /// Returns the number of bytes written. Returns 0 when decompression is complete. diff --git a/Compression.Core/Deflate/MsZipDecompressor.cs b/Compression.Core/Deflate/MsZipDecompressor.cs index 1c6c97ff2..8a21b3d85 100644 --- a/Compression.Core/Deflate/MsZipDecompressor.cs +++ b/Compression.Core/Deflate/MsZipDecompressor.cs @@ -108,4 +108,18 @@ public static byte[] DecompressBlock(ReadOnlySpan block) { return DeflateDecompressor.Decompress(block[2..]); } + + /// + /// Decompresses one MSZIP block of a folder: the Deflate data may refer back up to 32 KiB + /// into , the output of the folder's previous blocks ([MS-MCI] 2.2: + /// the history buffer is not reset between blocks). + /// + /// The block, including its "CK" prefix. + /// The folder's output so far (only the last 32 KiB matter). + /// The block's uncompressed bytes. + public static byte[] DecompressBlock(ReadOnlySpan block, ReadOnlySpan history) { + if (block.Length < 2 || block[0] != MsZipCompressor.SignatureByte0 || block[1] != MsZipCompressor.SignatureByte1) + throw new InvalidDataException("MSZIP: invalid block signature."); + return DeflateDecompressor.Decompress(block[2..], history); + } } diff --git a/Compression.Core/Dictionary/Lzx/LzxDecompressor.cs b/Compression.Core/Dictionary/Lzx/LzxDecompressor.cs index 8e6db23ad..b7e526000 100644 --- a/Compression.Core/Dictionary/Lzx/LzxDecompressor.cs +++ b/Compression.Core/Dictionary/Lzx/LzxDecompressor.cs @@ -60,6 +60,14 @@ public sealed class LzxDecompressor { private readonly LzxStreamFormat _format; private bool _readStreamHeader; + // A block may span several calls (cabinet frames are 32 KiB, blocks are not bounded by them), + // and so may the match that straddles a frame boundary. + private int _blockType; + private int _blockSize; + private int _blockRemaining; + private int _pendingMatchLength; + private int _pendingMatchOffset; + // E8 call-instruction translation state private readonly bool _e8Translation; private long _totalDecompressedBytes; @@ -128,41 +136,40 @@ public byte[] Decompress(int uncompressedSize) { } while (outPos < uncompressedSize) { - var blockType = (int)this.ReadBits(3); - - // Block size: twenty-four bits in a cabinet; in a WIM, one bit meaning - // "the usual 32 768" or sixteen bits otherwise. - int blockSize; - if (this._format == LzxStreamFormat.Cab) - blockSize = (int)this.ReadBits(24); - else if (this.ReadBits(1) == 1) - blockSize = LzxConstants.DefaultBlockSize; - else - blockSize = (int)this.ReadBits(16); - - blockSize = Math.Min(blockSize, uncompressedSize - outPos); - - switch (blockType) { - case LzxConstants.BlockTypeVerbatim: - this.ReadVerbatimBlockHeader(); - this.DecodeBlock(false, output, outPos, blockSize); - break; - - case LzxConstants.BlockTypeAligned: - this.ReadAlignedBlockHeader(); - this.DecodeBlock(true, output, outPos, blockSize); - break; - - case LzxConstants.BlockTypeUncompressed: - this.DecodeUncompressedBlock(output, outPos, blockSize); - break; - - default: - ThrowInvalidBlockType(blockType); - break; + var starting = this._blockRemaining == 0; + if (starting) { + this._blockType = (int)this.ReadBits(3); + + // Block size: twenty-four bits in a cabinet; in a WIM, one bit meaning + // "the usual 32 768" or sixteen bits otherwise. + if (this._format == LzxStreamFormat.Cab) + this._blockSize = (int)this.ReadBits(24); + else if (this.ReadBits(1) == 1) + this._blockSize = LzxConstants.DefaultBlockSize; + else + this._blockSize = (int)this.ReadBits(16); + if (this._blockSize == 0) + throw new InvalidDataException("LZX block of length zero."); + this._blockRemaining = this._blockSize; + + switch (this._blockType) { + case LzxConstants.BlockTypeVerbatim: this.ReadVerbatimBlockHeader(); break; + case LzxConstants.BlockTypeAligned: this.ReadAlignedBlockHeader(); break; + case LzxConstants.BlockTypeUncompressed: break; + default: ThrowInvalidBlockType(this._blockType); break; + } } - outPos += blockSize; + // The block may run past what this call is asked for; the rest of it follows in the + // next call (the next cabinet frame). + var count = Math.Min(this._blockRemaining, uncompressedSize - outPos); + if (this._blockType == LzxConstants.BlockTypeUncompressed) + this.DecodeUncompressedBlock(output, outPos, count, starting, count == this._blockRemaining); + else + this.DecodeBlock(this._blockType == LzxConstants.BlockTypeAligned, output, outPos, count); + + outPos += count; + this._blockRemaining -= count; } if (this._e8Translation && uncompressedSize > 10) @@ -197,14 +204,16 @@ private void ReadAlignedBlockHeader() { this.ReadVerbatimBlockHeader(); } - private void DecodeUncompressedBlock(byte[] output, int outPos, int blockSize) { - // Discard partial bits to align to the next 16-bit word boundary - this.AlignTo16Bits(); + private void DecodeUncompressedBlock(byte[] output, int outPos, int blockSize, bool starting, bool ending) { + if (starting) { + // Discard partial bits to align to the next 16-bit word boundary + this.AlignTo16Bits(); - // R0/R1/R2 are stored as three LE 32-bit integers - this._r0 = this.ReadRawInt32LE(); - this._r1 = this.ReadRawInt32LE(); - this._r2 = this.ReadRawInt32LE(); + // R0/R1/R2 are stored as three LE 32-bit integers + this._r0 = this.ReadRawInt32LE(); + this._r1 = this.ReadRawInt32LE(); + this._r2 = this.ReadRawInt32LE(); + } // Copy raw bytes var bytesRead = this._input.Read(output, outPos, blockSize); @@ -212,7 +221,7 @@ private void DecodeUncompressedBlock(byte[] output, int outPos, int blockSize) { ThrowUnexpectedEndOfStream(); // If block length is odd, consume one padding byte so stream stays word-aligned - if ((blockSize & 1) != 0) + if (ending && (this._blockSize & 1) != 0) this._input.ReadByte(); // Mirror into the circular window @@ -291,6 +300,20 @@ private void ReadPreTreeAndApply(int[] lengths, int start, int count) { private void DecodeBlock(bool isAligned, byte[] output, int outPos, int blockSize) { var end = outPos + blockSize; + + // The tail of a match that the previous call's end cut off. + if (this._pendingMatchLength > 0) { + var src = (this._windowPos - this._pendingMatchOffset + this._windowSize) & (this._windowSize - 1); + while (this._pendingMatchLength > 0 && outPos < end) { + var b = this._window[src]; + output[outPos++] = b; + this._window[this._windowPos] = b; + this._windowPos = (this._windowPos + 1) & (this._windowSize - 1); + src = (src + 1) & (this._windowSize - 1); + --this._pendingMatchLength; + } + } + while (outPos < end) { var mainSym = this.DecodeSymbol(this._mainTable!, LzxDecompressor.MainTableBits, this._mainLengths, this._numMainSymbols); @@ -318,13 +341,18 @@ private void DecodeBlock(bool isAligned, byte[] output, int outPos, int blockSiz // Copy from sliding window (handles run-length / overlapping copies) var srcPos = (this._windowPos - matchOffset + this._windowSize) & (this._windowSize - 1); - for (var i = 0; i < matchLength && outPos < end; ++i) { + var i = 0; + for (; i < matchLength && outPos < end; ++i) { var b = this._window[srcPos]; output[outPos++] = b; this._window[this._windowPos] = b; this._windowPos = (this._windowPos + 1) & (this._windowSize - 1); srcPos = (srcPos + 1) & (this._windowSize - 1); } + if (i < matchLength) { + this._pendingMatchLength = matchLength - i; + this._pendingMatchOffset = matchOffset; + } } } } diff --git a/Compression.Core/REFERENCE.md b/Compression.Core/REFERENCE.md index 86510a075..7d80e532c 100644 --- a/Compression.Core/REFERENCE.md +++ b/Compression.Core/REFERENCE.md @@ -390,6 +390,7 @@ Decompresses data in the DEFLATE format (RFC 1951). | `Decompress` | `int Decompress(Span output)` | Decompresses data into `output`, filling it unless the DEFLATE stream ends first. | | `Decompress` | `int Decompress(byte[] output, int offset, int count)` | Decompresses data from the input stream into the provided buffer. Returns the number of bytes written. Returns 0 when decompression is complete. | | `Decompress` | `static byte[] Decompress(ReadOnlySpan compressedData)` | Decompresses DEFLATE data in one shot. | +| `Decompress` | `static byte[] Decompress(ReadOnlySpan compressedData, ReadOnlySpan dictionary)` | Decompresses DEFLATE data whose matches may reach back into `dictionary`, the bytes that precede it (MSZIP keeps the previous block's output as history). | | `ReadRemainder` | `int ReadRemainder(Span destination)` | Reads bytes that follow the end of the DEFLATE data — a container's trailer, or the next member — taking the decompressor's read-ahead first and then the underlying stream. | #### `DeflateHuffmanTable` @@ -433,6 +434,7 @@ Decompresses data in the MSZIP format: a sequence of "CK"-prefixed Deflate block | --- | --- | --- | | `BlockSize` | `const int BlockSize` | The uncompressed block size limit: 32 768 bytes. | | `DecompressBlock` | `static byte[] DecompressBlock(ReadOnlySpan block)` | Decompresses a single MSZIP block (including its "CK" prefix) and returns the uncompressed bytes. | +| `DecompressBlock` | `static byte[] DecompressBlock(ReadOnlySpan block, ReadOnlySpan history)` | Decompresses one MSZIP block of a folder: the Deflate data may refer back up to 32 KiB into `history`, the output of the folder's previous blocks ([MS-MCI] 2.2: the history buffer is not reset between blocks). | | `Decompress` | `static byte[] Decompress(ReadOnlySpan data, int uncompressedSize = -1)` | Decompresses an MSZIP byte stream. | #### `RawDeflateStream` diff --git a/Compression.Lib/FormatDetector.cs b/Compression.Lib/FormatDetector.cs index 849934333..665524376 100644 --- a/Compression.Lib/FormatDetector.cs +++ b/Compression.Lib/FormatDetector.cs @@ -504,6 +504,9 @@ private static Format DetectInstaller(string path) { var buf = new byte[2]; var isPe = fs.Read(buf, 0, 2) >= 2 && buf[0] == 'M' && buf[1] == 'Z'; + // IExpress / wextract: the cabinet lives in the RT_RCDATA resource "CABINET". + if (isPe && FileFormat.IExpress.IExpressFormatDescriptor.IsIExpress(fs)) return Format.IExpress; + // An NSIS first header sits right after the exe stub, at a multiple of 512 bytes (the // exehead itself only looks there), so a large installer keeps it far from the tail. if (HasNsisFirstHeader(fs)) return Format.Nsis; diff --git a/Compression.Tests/Compression.Tests.csproj b/Compression.Tests/Compression.Tests.csproj index 023dbbe39..c24bdda9e 100644 --- a/Compression.Tests/Compression.Tests.csproj +++ b/Compression.Tests/Compression.Tests.csproj @@ -75,6 +75,10 @@ BcacheFsVectors.%(Filename)%(Extension) + + + IExpressVectors.%(Filename)%(Extension) + NsisVectors.%(Filename)%(Extension) diff --git a/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs new file mode 100644 index 000000000..bbbc898af --- /dev/null +++ b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs @@ -0,0 +1,37 @@ +using Compression.Registry; +using FileFormat.FirmwareHex; + +namespace Compression.Tests.FirmwareHex; + +[TestFixture] +public sealed class FirmwareHexPayloadTests { + [TestCase(false)] + [TestCase(true)] + public void Create_RejectsMultiplePayloadsWithoutWritingOutput(bool emptyFirst) { + var descriptor = new IntelHexFormatDescriptor(); + using var output = new MemoryStream(); + var inputs = new[] { + ArchiveInputInfo.InMemory("first.bin", emptyFirst ? [] : [1, 2, 3]), + ArchiveInputInfo.InMemory("second.bin", [9, 8, 7]), + }; + + Assert.Throws(() => + ((IArchiveCreatable)descriptor).Create(output, inputs, new FormatCreateOptions())); + Assert.That(output.Length, Is.Zero); + } + + [Test] + public void Add_RejectsAnotherPayloadWithoutChangingTheArchive() { + var descriptor = new IntelHexFormatDescriptor(); + using var archive = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(archive, + [ArchiveInputInfo.InMemory("firmware.bin", [1, 2, 3])], new FormatCreateOptions()); + var original = archive.ToArray(); + archive.Position = 0; + + Assert.Throws(() => ((IArchiveModifiable)descriptor).Add(archive, + [ArchiveInputInfo.InMemory("ADD.BIN", [9, 8, 7])])); + Assert.That(archive.ToArray(), Is.EqualTo(original)); + } + +} diff --git a/Compression.Tests/IExpress/IExpressReferenceVectorTests.cs b/Compression.Tests/IExpress/IExpressReferenceVectorTests.cs new file mode 100644 index 000000000..fb7014047 --- /dev/null +++ b/Compression.Tests/IExpress/IExpressReferenceVectorTests.cs @@ -0,0 +1,88 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using Compression.Lib; +using FileFormat.IExpress; + +namespace Compression.Tests.IExpress; + +/// +/// A package built by Windows iexpress.exe (see ReferenceVectors/README.md), checked against what +/// Windows expand.exe extracts from its CABINET resource (oracle.tsv). +/// +[TestFixture] +public class IExpressReferenceVectorTests { + + private const string Fixture = "iexpress-lzx.exe"; + + private static byte[] Load() { + using var resource = typeof(IExpressReferenceVectorTests).Assembly.GetManifestResourceStream("IExpressVectors." + Fixture + ".gz")!; + using var gz = new GZipStream(resource, CompressionMode.Decompress); // BCL as the test-side oracle + using var ms = new MemoryStream(); + gz.CopyTo(ms); + return ms.ToArray(); + } + + private static Dictionary Oracle() { + using var resource = typeof(IExpressReferenceVectorTests).Assembly.GetManifestResourceStream("IExpressVectors.oracle.tsv")!; + using var reader = new StreamReader(resource); + var result = new Dictionary(); + while (reader.ReadLine() is { } line) + if (line.Length > 0 && !line.StartsWith('#') && line.Split('\t') is var f && f[0] == Fixture) + result[f[1]] = (long.Parse(f[2], System.Globalization.CultureInfo.InvariantCulture), f[3]); + return result; + } + + [Test] + public void GivenIExpressPackage_WhenListed_ThenTheCabinetFilesMatchExpand() { + var oracle = Oracle(); + var entries = new IExpressFormatDescriptor().List(new MemoryStream(Load()), null).Where(e => e.Kind != "metadata").ToList(); + Assert.That(entries.Select(e => e.Name), Is.EquivalentTo(oracle.Keys)); + foreach (var e in entries) + Assert.That(e.OriginalSize, Is.EqualTo(oracle[e.Name].Size), e.Name); + } + + [Test] + public void GivenIExpressPackage_WhenExtracted_ThenEveryFileMatchesExpand() { + var oracle = Oracle(); + var dir = Path.Combine(Path.GetTempPath(), "cwb-iex-" + Guid.NewGuid().ToString("N")); + try { + new IExpressFormatDescriptor().Extract(new MemoryStream(Load()), dir, null, null); + foreach (var (name, (_, sha)) in oracle) + Assert.That(Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(Path.Combine(dir, name)))), Is.EqualTo(sha), name); + var ini = File.ReadAllText(Path.Combine(dir, "iexpress.ini"), Encoding.UTF8); + Assert.That(ini, Does.Contain("title=Fixture").And.Contain("parse_status=ok")); + } finally { + if (Directory.Exists(dir)) + Directory.Delete(dir, recursive: true); + } + } + + [Test] + public void GivenIExpressPackageOnDisk_WhenDetected_ThenIExpressIsChosen() { + var path = Path.Combine(Path.GetTempPath(), "cwb-iex-" + Guid.NewGuid().ToString("N") + ".exe"); + try { + File.WriteAllBytes(path, Load()); + Assert.That(FormatDetector.Detect(path), Is.EqualTo(FormatDetector.Format.IExpress)); + } finally { + File.Delete(path); + } + } + + [Test] + public void GivenPeWithoutCabinet_WhenProbed_ThenItIsNotIExpress() { + var data = Load(); + // Rename the resource: "CABINET" (UTF-16) becomes "CABINEX". + var name = Encoding.Unicode.GetBytes("CABINET"); + var at = data.AsSpan().IndexOf(name); + Assert.That(at, Is.GreaterThan(0)); + data[at + 12] = (byte)'X'; + Assert.That(IExpressFormatDescriptor.IsIExpress(new MemoryStream(data)), Is.False); + var entries = new IExpressFormatDescriptor().List(new MemoryStream(data), null); + Assert.That(entries.Select(e => e.Name), Is.EqualTo(new[] { "iexpress.ini" })); + } + + [Test] + public void GivenNonPeBytes_WhenProbed_ThenItIsNotIExpress() + => Assert.That(IExpressFormatDescriptor.IsIExpress(new MemoryStream(new byte[100])), Is.False); +} diff --git a/Compression.Tests/IExpress/ReferenceVectors/README.md b/Compression.Tests/IExpress/ReferenceVectors/README.md new file mode 100644 index 000000000..2620a58c6 --- /dev/null +++ b/Compression.Tests/IExpress/ReferenceVectors/README.md @@ -0,0 +1,10 @@ +# IExpress reference package + +`iexpress-lzx.exe` was built by Windows' own `C:\Windows\System32\iexpress.exe` (Windows 11 Pro +10.0.26200), never by CompressionWorkbench, from the SED file that `mkiexpress.py` writes +(`python mkiexpress.py iexpress-lzx`): an extract-only package of three files (a short +text, 3000 seeded random bytes, 32000 bytes of a repeated pattern). It is stored gzip-compressed and +otherwise unchanged. + +`oracle.tsv` is what Windows `expand.exe -F:*` writes from the package's `CABINET` resource (taken +out with 7-Zip 26.03): name, size and SHA-256 per file. diff --git a/Compression.Tests/IExpress/ReferenceVectors/iexpress-lzx.exe.gz b/Compression.Tests/IExpress/ReferenceVectors/iexpress-lzx.exe.gz new file mode 100644 index 000000000..984da14c1 Binary files /dev/null and b/Compression.Tests/IExpress/ReferenceVectors/iexpress-lzx.exe.gz differ diff --git a/Compression.Tests/IExpress/ReferenceVectors/mkiexpress.py b/Compression.Tests/IExpress/ReferenceVectors/mkiexpress.py new file mode 100644 index 000000000..ab914043a --- /dev/null +++ b/Compression.Tests/IExpress/ReferenceVectors/mkiexpress.py @@ -0,0 +1,30 @@ +import os, random, subprocess, sys +# usage: mkiexpress.py [compression] -- builds an IExpress package with C:\Windows\System32\iexpress.exe +W, name = sys.argv[1], sys.argv[2] +comp = sys.argv[3] if len(sys.argv) > 3 else None +W = os.path.abspath(W) +os.makedirs(os.path.join(W, 'payload'), exist_ok=True) +p = os.path.join(W, 'payload') +open(os.path.join(p, 'readme.txt'), 'wb').write(b'Hello from an IExpress fixture.\r\nLine two.\r\n') +r = random.Random(1234) +open(os.path.join(p, 'random.bin'), 'wb').write(bytes(r.getrandbits(8) for _ in range(3000))) +open(os.path.join(p, 'repeat.txt'), 'w').write('abcdefgh' * 4000) +files = ['readme.txt', 'random.bin', 'repeat.txt'] +target = os.path.join(W, name + '.exe') +sed = ['[Version]', 'Class=IEXPRESS', 'SEDVersion=3', '[Options]', 'PackagePurpose=ExtractOnly', 'ShowInstallProgramWindow=0', + 'HideExtractAnimation=1', 'UseLongFileName=1', 'InsideCompressed=0', 'CAB_FixedSize=0', 'CAB_ResvCodeSigning=0', + 'RebootMode=N', 'InstallPrompt=%InstallPrompt%', 'DisplayLicense=%DisplayLicense%', 'FinishMessage=%FinishMessage%', + 'TargetName=%TargetName%', 'FriendlyName=%FriendlyName%', 'AppLaunched=%AppLaunched%', 'PostInstallCmd=%PostInstallCmd%', + 'AdminQuietInstCmd=%AdminQuietInstCmd%', 'UserQuietInstCmd=%UserQuietInstCmd%', 'SourceFiles=SourceFiles'] +if comp: + sed.append('CompressionType=' + comp) +sed += ['[Strings]', 'InstallPrompt=', 'DisplayLicense=', 'FinishMessage=', f'TargetName={target}', 'FriendlyName=Fixture', + 'AppLaunched=', 'PostInstallCmd=', 'AdminQuietInstCmd=', 'UserQuietInstCmd='] +for i, f in enumerate(files): + sed.append(f'FILE{i}="{f}"') +sed += ['[SourceFiles]', f'SourceFiles0={p}\\', '[SourceFiles0]'] +sed += [f'%FILE{i}%=' for i in range(len(files))] +sedpath = os.path.join(W, name + '.sed') +open(sedpath, 'w').write('\r\n'.join(sed) + '\r\n') +res = subprocess.run([r'C:\Windows\System32\iexpress.exe', '/N', '/Q', sedpath], capture_output=True, text=True, timeout=300) +print(name, res.returncode, os.path.getsize(target) if os.path.exists(target) else -1) diff --git a/Compression.Tests/IExpress/ReferenceVectors/oracle.tsv b/Compression.Tests/IExpress/ReferenceVectors/oracle.tsv new file mode 100644 index 000000000..d0addc4d1 --- /dev/null +++ b/Compression.Tests/IExpress/ReferenceVectors/oracle.tsv @@ -0,0 +1,4 @@ +# fixture path size sha256 -- CABINET resource taken out by 7-Zip 26.03, expanded by Windows expand.exe +iexpress-lzx.exe random.bin 3000 ea9a04acd72963e6c050896f128a26ced2a8a41696550c6cf4a81fb57f530a91 +iexpress-lzx.exe readme.txt 44 6aa372797726bab30b5d04aafd1c2c67e32d3aadf8035941d6f2371bcd5abbba +iexpress-lzx.exe repeat.txt 32000 755a009b306c9e7bedbb976fbf50710eb9dddd63e9e17ac59ca8d32769fd7627 diff --git a/Compression.Tests/Msu/MsuTests.cs b/Compression.Tests/Msu/MsuTests.cs new file mode 100644 index 000000000..6164941b2 --- /dev/null +++ b/Compression.Tests/Msu/MsuTests.cs @@ -0,0 +1,50 @@ +using System.Security.Cryptography; +using Compression.Lib; +using FileFormat.Msu; + +namespace Compression.Tests.Msu; + +[TestFixture] +public class MsuTests { + + [Test] + public void GivenMsuPath_WhenDetected_ThenMsuIsChosen() + => Assert.That(FormatDetector.DetectByExtension("update.msu"), Is.EqualTo(FormatDetector.Format.Msu)); + + [Test] + public void GivenBytesThatAreNeitherCabNorWim_WhenListed_ThenAPartialMetadataEntryIsReturned() { + var entries = new MsuFormatDescriptor().List(new MemoryStream(new byte[64]), null); + Assert.That(entries.Select(e => e.Name), Is.EqualTo(new[] { "metadata.ini" })); + } + + [Test] + public void GivenEmptyStream_WhenListed_ThenAPartialMetadataEntryIsReturned() { + var entries = new MsuFormatDescriptor().List(new MemoryStream(), null); + Assert.That(entries.Select(e => e.Name), Is.EqualTo(new[] { "metadata.ini" })); + } + + /// + /// The real package from the Microsoft Update Catalog (see ReferenceVectors/README.md; set + /// CWB_MSU_FIXTURES): every part must match what 7-Zip extracts (oracle.tsv). + /// + [Test] + public void GivenRealMsu_WhenExtracted_ThenEveryPartMatches7Zip() { + var dir = Environment.GetEnvironmentVariable("CWB_MSU_FIXTURES"); + var path = dir == null ? null : Path.Combine(dir, "windows6.0-kb2999226-x86.msu"); + if (path == null || !File.Exists(path)) + Assert.Ignore("CWB_MSU_FIXTURES not set."); + + var oracle = File.ReadAllLines(Path.Combine(TestContext.CurrentContext.TestDirectory, "..", "..", "..", "Msu", "ReferenceVectors", "oracle.tsv")) + .Where(l => l.Length > 0 && !l.StartsWith('#')).Select(l => l.Split('\t')).ToDictionary(f => f[1], f => f[3]); + var outDir = Path.Combine(Path.GetTempPath(), "cwb-msu-" + Guid.NewGuid().ToString("N")); + try { + using (var fs = File.OpenRead(path!)) + new MsuFormatDescriptor().Extract(fs, outDir, null, null); + var got = Directory.GetFiles(outDir).ToDictionary(p => Path.GetFileName(p), p => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(p)))); + Assert.That(got, Is.EquivalentTo(oracle)); + } finally { + if (Directory.Exists(outDir)) + Directory.Delete(outDir, recursive: true); + } + } +} diff --git a/Compression.Tests/Msu/ReferenceVectors/README.md b/Compression.Tests/Msu/ReferenceVectors/README.md new file mode 100644 index 000000000..dc5a76057 --- /dev/null +++ b/Compression.Tests/Msu/ReferenceVectors/README.md @@ -0,0 +1,10 @@ +# MSU reference package (not checked in) + +The package itself is a Microsoft binary and is not redistributed here; download it from the +Microsoft Update Catalog (update 7c75337c-6783-4889-8e95-a13f285d3a54, "Update for Windows Vista +(KB2999226)"): + +https://catalog.s.download.windowsupdate.com/c/msdownload/update/software/updt/2015/09/windows6.0-kb2999226-x86_38a521378445b34bd222fa354d2d5059707285a6.msu + +Save it as `windows6.0-kb2999226-x86.msu` in a directory and point `CWB_MSU_FIXTURES` at it. +`oracle.tsv` is what 7-Zip 26.03 extracts from it (`7z x`): name, size and SHA-256 per file. diff --git a/Compression.Tests/Msu/ReferenceVectors/oracle.tsv b/Compression.Tests/Msu/ReferenceVectors/oracle.tsv new file mode 100644 index 000000000..e6fd3d253 --- /dev/null +++ b/Compression.Tests/Msu/ReferenceVectors/oracle.tsv @@ -0,0 +1,5 @@ +# fixture path size sha256 -- produced by 7-Zip 26.03 (x64) "7z x" +windows6.0-kb2999226-x86.msu WSUSSCAN.cab 172512 218daa0060821d5c71863ac07722ed8259cd8fba0f0d861e7d3101d7858bf113 +windows6.0-kb2999226-x86.msu Windows6.0-KB2999226-x86-pkgProperties.txt 486 7ab569cb851052f1e8049ab0be5f34851a8524a6113b69c28ef8ed35dc639e17 +windows6.0-kb2999226-x86.msu Windows6.0-KB2999226-x86.cab 508091 c381d920c81cfc6ca52cdcc1758556eca3fe98804b571b1379ce4878ff746f92 +windows6.0-kb2999226-x86.msu Windows6.0-KB2999226-x86.xml 444 eb7cc14531a3ac6b4228155a71189a01b50f8a47d49a0d49c7da2b7d945d8a51 diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Cab/CabReader.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Cab/CabReader.cs index 8b3ae61c6..f437ebe69 100644 --- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Cab/CabReader.cs +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Cab/CabReader.cs @@ -187,7 +187,8 @@ private byte[] DecompressFolder(FolderInfo folder) { break; case CabCompressionType.MsZip: { - var decompressed = MsZipDecompressor.DecompressBlock(abData); + // Blocks of a folder share the 32 KiB Deflate history ([MS-MCI] 2.2). + var decompressed = MsZipDecompressor.DecompressBlock(abData, System.Runtime.InteropServices.CollectionsMarshal.AsSpan(output)); output.AddRange(decompressed); break; } diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.IExpress/IExpressFormatDescriptor.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.IExpress/IExpressFormatDescriptor.cs new file mode 100644 index 000000000..642cfd8b9 --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.IExpress/IExpressFormatDescriptor.cs @@ -0,0 +1,116 @@ +#pragma warning disable CS1591 +using System.Text; +using Compression.Registry; +using FileFormat.Cab; +using FileFormat.PeResources; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.IExpress; + +/// +/// IExpress self-extractor (Windows' iexpress.exe / wextract.exe stub): a PE whose RT_RCDATA +/// resource "CABINET" holds a Microsoft Cabinet with the payload, next to text resources that +/// configure the run (TITLE, RUNPROGRAM, POSTRUNPROGRAM, LICENSE, FINISHMSG, ...). +/// +/// References: +/// +/// https://learn.microsoft.com/windows/win32/msi/cabinet-files — the cabinet format +/// https://learn.microsoft.com/previous-versions/windows/it-pro/windows-server-2003/cc782932(v=ws.10) — IExpress +/// +/// +public sealed class IExpressFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { + + private const ushort RtRcData = 10; + private static readonly string[] TextResources = ["TITLE", "RUNPROGRAM", "POSTRUNPROGRAM", "ADMQCMD", "USRQCMD", "LICENSE", "FINISHMSG", "UPROMPT"]; + + public string Id => "IExpress"; + public string DisplayName => "IExpress self-extractor"; + public FormatCategory Category => FormatCategory.Archive; + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; + public string DefaultExtension => ".exe"; + // Detected by its CABINET resource (FormatDetector.DetectInstaller), never by ".exe". + public IReadOnlyList Extensions => []; + public IReadOnlyList CompoundExtensions => []; + public IReadOnlyList MagicSignatures => []; + public IReadOnlyList Methods => [new("mszip", "MSZIP"), new("lzx", "LZX"), new("quantum", "Quantum"), new("stored", "Stored")]; + public string? TarCompressionFormatId => null; + public AlgorithmFamily Family => AlgorithmFamily.Archive; + public string Description => "IExpress / wextract self-extracting executable: the files of its embedded cabinet"; + + /// True when the PE carries an RT_RCDATA "CABINET" resource that starts with "MSCF". + public static bool IsIExpress(Stream stream) { + if (PeResourceLocator.Find(stream, RtRcData, "CABINET") is not { Size: >= 36 } at) + return false; + Span magic = stackalloc byte[4]; + stream.Position = at.Offset; + return stream.ReadAtLeast(magic, 4, throwOnEndOfStream: false) == 4 && magic.SequenceEqual("MSCF"u8); + } + + /// Returns the embedded cabinet's bytes, or null when there is none. + public static byte[]? ReadCabinet(Stream stream) => ReadResource(stream, "CABINET"); + + private static byte[]? ReadResource(Stream stream, string name) { + if (PeResourceLocator.Find(stream, RtRcData, name) is not { } at) + return null; + var data = new byte[at.Size]; + stream.Position = at.Offset; + stream.ReadExactly(data); + return data; + } + + public List List(Stream stream, string? password) { + byte[]? cab; + try { + cab = ReadCabinet(stream); + } catch (Exception ex) when (ex is IOException or InvalidDataException) { + cab = null; + } + var metadata = Metadata(stream, cab == null ? "no CABINET resource" : null); + var result = new List { new(0, "iexpress.ini", metadata.Length, metadata.Length, "stored", false, false, null, Kind: "metadata") }; + if (cab == null) + return result; + try { + using var reader = new CabReader(new MemoryStream(cab, writable: false)); + result.AddRange(reader.Entries.Select((e, i) => new ArchiveEntryInfo(i + 1, e.FileName, e.UncompressedSize, -1, "cab", false, false, e.LastModified))); + } catch (InvalidDataException) { + // The metadata entry still describes the package. + } + return result; + } + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + var cab = ReadCabinet(stream); + if (files == null || MatchesFilter("iexpress.ini", files)) + WriteFile(outputDir, "iexpress.ini", Metadata(stream, cab == null ? "no CABINET resource" : null)); + if (cab == null) + return; + using var reader = new CabReader(new MemoryStream(cab, writable: false)); + foreach (var e in reader.Entries) { + if (files != null && !MatchesFilter(e.FileName, files)) + continue; + WriteFile(outputDir, e.FileName, reader.ExtractEntry(e)); + } + } + + private static byte[] Metadata(Stream stream, string? error) { + var sb = new StringBuilder("[iexpress]\n"); + sb.Append("parse_status=").Append(error == null ? "ok" : "partial").Append('\n'); + if (error != null) + sb.Append("error=").Append(error).Append('\n'); + foreach (var name in TextResources) { + byte[]? raw; + try { + raw = ReadResource(stream, name); + } catch (IOException) { + raw = null; + } + if (raw == null) + continue; + // The stub stores these as ANSI strings with a terminating NUL. + var text = Encoding.Latin1.GetString(raw).TrimEnd('\0').Replace("\r", "\\r").Replace("\n", "\\n"); + sb.Append(name.ToLowerInvariant()).Append('=').Append(text).Append('\n'); + } + return Encoding.UTF8.GetBytes(sb.ToString()); + } +} diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Msu/MsuFormatDescriptor.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Msu/MsuFormatDescriptor.cs new file mode 100644 index 000000000..6410f1902 --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Msu/MsuFormatDescriptor.cs @@ -0,0 +1,68 @@ +#pragma warning disable CS1591 +using System.Text; +using Compression.Registry; +using FileFormat.Cab; +using FileFormat.Wim; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.Msu; + +/// +/// Windows Update standalone package (.msu): a Microsoft Cabinet holding the update's own +/// cabinet(s), its applicability XML, a properties text and the WSUS scan cabinet; recent +/// packages may be WIM containers instead. The container is recognised by its magic and the +/// listing is the container's own, read by our Cab or WIM reader. +/// +/// References: +/// +/// https://learn.microsoft.com/troubleshoot/windows-client/installing-updates-features-roles/description-of-windows-update-standalone-installer — the package and its parts +/// +/// +public sealed class MsuFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { + public string Id => "Msu"; + public string DisplayName => "Windows Update package (MSU)"; + public FormatCategory Category => FormatCategory.Archive; + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; + public string DefaultExtension => ".msu"; + public IReadOnlyList Extensions => [".msu"]; + public IReadOnlyList CompoundExtensions => []; + // Extension-only: the CAB and WIM descriptors own the container magics. + public IReadOnlyList MagicSignatures => []; + public IReadOnlyList Methods => [new("cab", "Cabinet"), new("wim", "WIM")]; + public string? TarCompressionFormatId => null; + public AlgorithmFamily Family => AlgorithmFamily.Archive; + public string Description => "Windows Update standalone package: the cabinet (or WIM) with the update parts"; + + private static IArchiveFormatOperations Container(Stream stream) { + Span magic = stackalloc byte[5]; + stream.Position = 0; + var n = stream.ReadAtLeast(magic, 5, throwOnEndOfStream: false); + stream.Position = 0; + if (n >= 4 && magic[..4].SequenceEqual("MSCF"u8)) + return new CabFormatDescriptor(); + if (n >= 5 && magic.SequenceEqual("MSWIM"u8)) + return new WimFormatDescriptor(); + throw new InvalidDataException("Not an MSU package: neither a cabinet (MSCF) nor a WIM (MSWIM)."); + } + + public List List(Stream stream, string? password) { + try { + return Container(stream).List(stream, password); + } catch (Exception ex) when (ex is InvalidDataException or EndOfStreamException or NotSupportedException) { + var note = Encoding.UTF8.GetBytes($"format=msu\nparse_status=partial\nerror={ex.Message.Replace('\n', ' ')}\n"); + return [new ArchiveEntryInfo(0, "metadata.ini", note.Length, note.Length, "stored", false, false, null, Kind: "metadata")]; + } + } + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + IArchiveFormatOperations container; + try { + container = Container(stream); + } catch (InvalidDataException ex) { + WriteFile(outputDir, "metadata.ini", Encoding.UTF8.GetBytes($"format=msu\nparse_status=partial\nerror={ex.Message}\n")); + return; + } + container.Extract(stream, outputDir, password, files); + } +} diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.PeResources/PeResourceLocator.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.PeResources/PeResourceLocator.cs new file mode 100644 index 000000000..a9976411b --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.PeResources/PeResourceLocator.cs @@ -0,0 +1,118 @@ +using System.Buffers.Binary; +using System.Text; + +namespace FileFormat.PeResources; + +/// +/// Finds one resource in a PE image without reading any resource data: walks the +/// IMAGE_RESOURCE_DIRECTORY tree (type → name → first language) and returns where the bytes are. +/// +public static class PeResourceLocator { + + /// File offset and size of a located resource. + public readonly record struct Location(long Offset, int Size); + + /// Finds a resource by numeric type and string name (e.g. RT_RCDATA "CABINET"). + public static Location? Find(Stream stream, ushort type, string name) => Find(stream, type, name, 0); + + /// Finds a resource by numeric type and numeric id. + public static Location? Find(Stream stream, ushort type, ushort id) => Find(stream, type, null, id); + + private static Location? Find(Stream stream, ushort type, string? name, ushort id) { + ArgumentNullException.ThrowIfNull(stream); + try { + if (!stream.CanSeek || stream.Length < 0x40) + return null; + var head = new byte[4096]; + stream.Position = 0; + var n = stream.ReadAtLeast(head, head.Length, throwOnEndOfStream: false); + if (head[0] != 'M' || head[1] != 'Z') + return null; + var pe = BinaryPrimitives.ReadInt32LittleEndian(head.AsSpan(0x3C)); + if (pe < 0 || pe + 24 > n || BinaryPrimitives.ReadUInt32LittleEndian(head.AsSpan(pe)) != 0x00004550) + return null; + var sections = BinaryPrimitives.ReadUInt16LittleEndian(head.AsSpan(pe + 6)); + var optSize = BinaryPrimitives.ReadUInt16LittleEndian(head.AsSpan(pe + 20)); + var opt = pe + 24; + var dirs = opt + (BinaryPrimitives.ReadUInt16LittleEndian(head.AsSpan(opt)) == 0x20B ? 112 : 96); + if (dirs + 24 > n) + return null; + var resRva = BinaryPrimitives.ReadUInt32LittleEndian(head.AsSpan(dirs + 16)); + if (resRva == 0) + return null; + var table = opt + optSize; + + long ToOffset(uint rva) { + for (var i = 0; i < sections && table + 40 * i + 40 <= n; ++i) { + var s = head.AsSpan(table + 40 * i); + var va = BinaryPrimitives.ReadUInt32LittleEndian(s[12..]); + var size = Math.Max(BinaryPrimitives.ReadUInt32LittleEndian(s[8..]), BinaryPrimitives.ReadUInt32LittleEndian(s[16..])); + if (rva >= va && rva < va + size) + return BinaryPrimitives.ReadUInt32LittleEndian(s[20..]) + (long)(rva - va); + } + return -1; + } + + var root = ToOffset(resRva); + if (root < 0) + return null; + if (Entry(stream, root, root, null, type) is not { IsDirectory: true } t) + return null; + if (Entry(stream, root, t.Offset, name, id) is not { IsDirectory: true } nm) + return null; + if (Entry(stream, root, nm.Offset, null, null) is not { IsDirectory: false } leaf) + return null; + Span data = stackalloc byte[8]; + stream.Position = leaf.Offset; + stream.ReadExactly(data); + var off = ToOffset(BinaryPrimitives.ReadUInt32LittleEndian(data)); + var len = BinaryPrimitives.ReadInt32LittleEndian(data[4..]); + return off < 0 || len < 0 || off + len > stream.Length ? null : new Location(off, len); + } catch (Exception ex) when (ex is IOException or ArgumentException) { + return null; + } + } + + /// + /// Finds the entry matching (string) or + /// (numeric); with neither, returns the first entry. + /// + private static (long Offset, bool IsDirectory)? Entry(Stream stream, long root, long dir, string? name, ushort? id) { + Span h = stackalloc byte[16]; + stream.Position = dir; + if (stream.ReadAtLeast(h, 16, throwOnEndOfStream: false) < 16) + return null; + var count = BinaryPrimitives.ReadUInt16LittleEndian(h[12..]) + BinaryPrimitives.ReadUInt16LittleEndian(h[14..]); + Span e = stackalloc byte[8]; + for (var i = 0; i < Math.Min(count, 4096); ++i) { + stream.Position = dir + 16 + 8 * i; + if (stream.ReadAtLeast(e, 8, throwOnEndOfStream: false) < 8) + return null; + var nameField = BinaryPrimitives.ReadUInt32LittleEndian(e); + var target = BinaryPrimitives.ReadUInt32LittleEndian(e[4..]); + var match = (name, id) switch { + (null, null) => true, + (not null, _) => (nameField & 0x80000000) != 0 && NameEquals(stream, root + (nameField & 0x7FFFFFFF), name), + _ => (nameField & 0x80000000) == 0 && nameField == id, + }; + if (match) + return (root + (target & 0x7FFFFFFF), (target & 0x80000000) != 0); + } + return null; + } + + // IMAGE_RESOURCE_DIR_STRING_U: u16 length then UTF-16LE characters; names compare case-insensitively. + private static bool NameEquals(Stream stream, long at, string name) { + Span len = stackalloc byte[2]; + stream.Position = at; + if (stream.ReadAtLeast(len, 2, throwOnEndOfStream: false) < 2) + return false; + var chars = BinaryPrimitives.ReadUInt16LittleEndian(len); + if (chars != name.Length) + return false; + var buf = new byte[chars * 2]; + if (stream.ReadAtLeast(buf, buf.Length, throwOnEndOfStream: false) < buf.Length) + return false; + return string.Equals(Encoding.Unicode.GetString(buf), name, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md index 9357f828e..19eec6d4b 100644 --- a/Hawkynt.FileFormats.Archives/README.md +++ b/Hawkynt.FileFormats.Archives/README.md @@ -185,11 +185,13 @@ Every State, Test, Maintenance, Compress/Decompress and Demux/Mux/Remux cell is | [Debian .deb](https://en.wikipedia.org/wiki/Deb_(file_format)) | `Deb` | `.deb` | R/W | ✅ | defrag · wipe | | [debian.org](https://www.debian.org/doc/debian-policy/) | | [EAR](https://en.wikipedia.org/wiki/EAR_(file_format)) | `Ear` | `.ear` | R/W | ✅ | defrag · wipe | | [jakarta.ee](https://jakarta.ee/specifications/platform/) | | [Ruby gem](https://en.wikipedia.org/wiki/RubyGems) | `Gem` | `.gem` | WORM | ✅ | — | TAR with gzip-compressed metadata and data members | [docs.ruby-lang.org](https://docs.ruby-lang.org/en/3.0/Gem/Format.html) | +| [IExpress](https://en.wikipedia.org/wiki/IExpress) | `IExpress` | | R | ✅ | — | Files of the cabinet in the `CABINET` resource, plus the package text resources; checked against expand.exe | [Microsoft Learn](https://learn.microsoft.com/windows/win32/msi/cabinet-files) | | [Inno Setup](https://en.wikipedia.org/wiki/Inno_Setup) | `InnoSetup` | | WORM | ✅ | — | Extraction plus signature/container output, not an installer compiler; some versions expose no per-file extraction | [sourceforge.net](https://sourceforge.net/projects/innounp/) | | [IPA](https://en.wikipedia.org/wiki/.ipa) | `Ipa` | `.ipa` | R/W | ✅ | defrag · wipe | | [pkware.cachefly.net](https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT) | | [JAR](https://en.wikipedia.org/wiki/JAR_(file_format)) | `Jar` | `.jar` | R/W | ✅ | defrag · wipe | | [docs.oracle.com](https://docs.oracle.com/en-us/javase/8/docs/technotes/guides/jar/jar.html) | | [MSI](https://en.wikipedia.org/wiki/Windows_Installer) | `Msi` | `.msi` `.msp` `.mst` | R/W | ✅ | wipe | CFB envelope; a functional Installer database is not synthesised | [Microsoft Learn](https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-cfb/) | | [MSIX](https://en.wikipedia.org/wiki/MSIX) | `Msix` | `.msix` `.msixbundle` | R/W | ✅ | defrag · wipe | Unsigned fresh package output | [Microsoft Learn](https://learn.microsoft.com/en-us/windows/msix/) | +| Windows Update package | `Msu` | `.msu` | R | ✅ | — | The package cabinet (or WIM) and its parts; checked against 7-Zip | [Microsoft Learn](https://learn.microsoft.com/troubleshoot/windows-client/installing-updates-features-roles/description-of-windows-update-standalone-installer) | | [NSIS](https://en.wikipedia.org/wiki/Nullsoft_Scriptable_Install_System) | `Nsis` | | R | ✅ | — | Files of the install script, zlib/bzip2/LZMA, solid or not, NSIS 2 and 3; checked against 7-Zip. LZMA behind the x86 branch filter is refused | [nsis.sourceforge.io](https://nsis.sourceforge.io/Docs/) | | [NuGet .nupkg](https://en.wikipedia.org/wiki/NuGet) | `NuPkg` | `.nupkg` | R/W | ✅ | defrag · wipe | | [Microsoft Learn](https://learn.microsoft.com/nuget/reference/nuspec) | | [OVA](https://en.wikipedia.org/wiki/Open_Virtualization_Format) | `Ova` | `.ova` | WORM | ✅ | — | Stays WORM: the manifest must cover every member | [dmtf.org](https://www.dmtf.org/standards/ovf) | @@ -544,7 +546,7 @@ Use it when a .NET process needs to enumerate, extract, test, create, edit or in -Every public and protected member of all 2571 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). +Every public and protected member of all 2575 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). diff --git a/Hawkynt.FileFormats.Archives/REFERENCE.md b/Hawkynt.FileFormats.Archives/REFERENCE.md index 464eb0ba6..4011c2c22 100644 --- a/Hawkynt.FileFormats.Archives/REFERENCE.md +++ b/Hawkynt.FileFormats.Archives/REFERENCE.md @@ -18427,6 +18427,36 @@ Writes a big-endian GameCube `.hps` (HAL HALPST) carrying DSP-ADPCM, laid out pe | `HpsWriter` | `HpsWriter()` | | | `Write` | `byte[] Write(IReadOnlyList channels, int sampleRate, int loopStart = 0)` | Encodes per-channel mono PCM16 to DSP-ADPCM and serialises a complete HPS. All channels must share the same sample count. | +### Namespace `FileFormat.IExpress` + +[`IExpressFormatDescriptor`](#iexpressformatdescriptor) + +#### `IExpressFormatDescriptor` + +IExpress self-extractor (Windows' iexpress.exe / wextract.exe stub): a PE whose RT_RCDATA resource "CABINET" holds a Microsoft Cabinet with the payload, next to text resources that configure the run (TITLE, RUNPROGRAM, POSTRUNPROGRAM, LICENSE, FINISHMSG, ...). References: `https://learn.microsoft.com/windows/win32/msi/cabinet-files` — the cabinet format`https://learn.microsoft.com/previous-versions/windows/it-pro/windows-server-2003/cc782932(v=ws.10)` — IExpress + +Implements `IArchiveFormatOperations`, `IFormatDescriptor`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `IExpressFormatDescriptor` | `IExpressFormatDescriptor()` | | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `IsIExpress` | `static bool IsIExpress(Stream stream)` | True when the PE carries an RT_RCDATA "CABINET" resource that starts with "MSCF". | +| `List` | `List List(Stream stream, string password)` | | +| `ReadCabinet` | `static byte[] ReadCabinet(Stream stream)` | Returns the embedded cabinet's bytes, or null when there is none. | + ### Namespace `FileFormat.IcePacker` [`IcePackerFormatDescriptor`](#icepackerformatdescriptor) · [`IcePackerStream`](#icepackerstream) @@ -22167,6 +22197,34 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati | `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes named entries via `ZipModifier`. The synthetic `metadata.ini` listing entry is a derived view and is skipped. | | `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Zeros every dead byte in the package: gaps between entries not covered by a live extent in the ZIP layout map. Local headers, entry data, the central directory and EOCD are live and preserved. Cluster-tip wiping is N/A (ZIP packs entries back to back with no per-file slack). | +### Namespace `FileFormat.Msu` + +[`MsuFormatDescriptor`](#msuformatdescriptor) + +#### `MsuFormatDescriptor` + +Windows Update standalone package (.msu): a Microsoft Cabinet holding the update's own cabinet(s), its applicability XML, a properties text and the WSUS scan cabinet; recent packages may be WIM containers instead. The container is recognised by its magic and the listing is the container's own, read by our Cab or WIM reader. References: `https://learn.microsoft.com/troubleshoot/windows-client/installing-updates-features-roles/description-of-windows-update-standalone-installer` — the package and its parts + +Implements `IArchiveFormatOperations`, `IFormatDescriptor`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `MsuFormatDescriptor` | `MsuFormatDescriptor()` | | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | + ### Namespace `FileFormat.Mtm` [`MtmFormatDescriptor`](#mtmformatdescriptor) @@ -25010,7 +25068,28 @@ Writes a minimal PDF 1.7 file containing the input files as embedded file attach ### Namespace `FileFormat.PeResources` -[`PeResourcesFormatDescriptor`](#peresourcesformatdescriptor) +[`PeResourceLocator`](#peresourcelocator) · [`PeResourceLocator.Location`](#peresourcelocatorlocation) · [`PeResourcesFormatDescriptor`](#peresourcesformatdescriptor) + +#### `PeResourceLocator` + +Finds one resource in a PE image without reading any resource data: walks the IMAGE_RESOURCE_DIRECTORY tree (type → name → first language) and returns where the bytes are. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Find` | `static Location? Find(Stream stream, ushort type, string name)` | Finds a resource by numeric type and string name (e.g. RT_RCDATA "CABINET"). | +| `Find` | `static Location? Find(Stream stream, ushort type, ushort id)` | Finds a resource by numeric type and numeric id. | + +#### `PeResourceLocator.Location` + +File offset and size of a located resource. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Location` | `Location(long Offset, int Size)` | File offset and size of a located resource. | +| `Offset` | `long Offset { get; init; }` | | +| `Size` | `int Size { get; init; }` | | #### `PeResourcesFormatDescriptor` diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs index 6c7d4b452..1c9e17266 100644 --- a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs @@ -33,6 +33,7 @@ public static class FirmwareHexWriter { /// one the reader rendered. An input list with no payload describes an image /// with no data, which both formats can write. /// + /// More than one non-metadata payload is supplied. public static FirmwareImage ImageFrom(IReadOnlyList inputs, string sourceFormat) { ArgumentNullException.ThrowIfNull(inputs); @@ -53,7 +54,9 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st } // Any other single input is the payload, whatever it is called: a caller // converting a .bin into a .hex should not have to rename it first. - payload ??= input.ReadContent(); + if (payload is not null) + throw new InvalidDataException($"{sourceFormat} represents one logical firmware payload; multiple payload inputs are ambiguous."); + payload = input.ReadContent(); } List<(uint Address, byte[] Data)> segments; diff --git a/docs/catalogue/our-registry.csv b/docs/catalogue/our-registry.csv index bd9da9071..4a363f275 100644 --- a/docs/catalogue/our-registry.csv +++ b/docs/catalogue/our-registry.csv @@ -239,6 +239,7 @@ format,Iceberg,Apache Iceberg metadata,Archive,,,Y,Y,, format,IcePacker,ICE Packer,Stream,,.ice,,Y,Y, format,Icns,ICNS (Apple icon),Archive,,.icns,Y,Y,, format,Ico,Windows ICO/CUR,Archive,,.ico,Y,Y,Y,Y +format,IExpress,IExpress self-extractor,Archive,,,Y,Y,, format,IffCdaf,IFF CDAF,Archive,,.cdaf,Y,Y,Y,Y format,Imd,ImageDisk (IMD),Archive,,.imd,Y,Y,, format,InnoSetup,Inno Setup,Archive,,,Y,Y,Y, @@ -334,6 +335,7 @@ format,Msa,MSA (Magic Shadow Archiver),Archive,Y,.msa,Y,Y,Y,Y format,Msg,MSG,Archive,,.msg,Y,Y,Y,Y format,Msi,MSI (OLE Compound File),Archive,,.msi;.msp;.mst,Y,Y,Y,Y format,Msix,MSIX,Archive,,.msix;.msixbundle,Y,Y,Y,Y +format,Msu,Windows Update package (MSU),Archive,,.msu,Y,Y,, format,Mtm,MTM (MultiTracker),Audio,,.mtm,Y,Y,, format,Mtree,mtree,Archive,,.mtree,Y,,Y, format,Mus,DMX/Doom MUS,Audio,,.mus,Y,Y,, diff --git a/docs/catalogue/reference-formats.csv b/docs/catalogue/reference-formats.csv index d168a2e19..6e595ea56 100644 --- a/docs/catalogue/reference-formats.csv +++ b/docs/catalogue/reference-formats.csv @@ -27,10 +27,10 @@ Internet Archive ARC (crawl),arc,archive,,,,,,fmt/410;x-fmt/219,,1,https://archi XAR (eXtensible ARchive),xar;pkg;xip;mpkg,archive,R,RW,R,,,fmt/600,Y,5,https://github.com/mackyle/xar/wiki/xarformat,Xar,Y,Y,Y,OK,,"every xar encoding (none, gzip/zlib, bzip2, lzma, xz) decoded and checked against the TOC checksum; verified on bsdtar-written archives" Xar Image Format (PRONOM fmt/922),xar,archive,,,,,,fmt/922,,1,,,,,,MISSING,,PRONOM lists this separately from eXtensible ARchive Microsoft Cabinet (CAB),cab;exe;mscf,archive,R,R,R,R,,x-fmt/414,RW,6,https://learn.microsoft.com/previous-versions/bb417343(v=msdn.10),Cab,Y,Y,Y,OK,, -CAB self-extractor (IExpress / MSCF SFX),exe,installer,,,,,R,,,1,,,,,,MISSING,, +CAB self-extractor (IExpress / MSCF SFX),exe,installer,,,,,R,,,1,https://learn.microsoft.com/windows/win32/msi/cabinet-files,IExpress,Y,,,READ-ONLY-BUT-SOURCES-WRITE,iexpress.exe,"cabinets appended as a PE overlay are read through the generic self-extractor path" Compound File Binary (OLE2),cfb;ole;doc;xls;ppt,document,R,,R,,,fmt/111,,3,https://learn.microsoft.com/openspecs/windows_protocols/ms-cfb/,CompoundFileBinary;Doc;Xls;Ppt;LegacyOfficeCompoundFile,Y,Y,Y,OK,, Windows Installer (MSI/MSP/MSM),msi;msp;msm;mst,installer,R,,R,,R,,Y,4,https://learn.microsoft.com/windows/win32/msi/windows-installer-portal,Msi,Y,Y,Y,OK,, -Windows Update package (MSU),msu,installer,,,,,R,,,1,,,,,,MISSING,, +Windows Update package (MSU),msu,installer,,,,,R,,,1,https://learn.microsoft.com/troubleshoot/windows-client/installing-updates-features-roles/description-of-windows-update-standalone-installer,Msu,Y,,,OK,, Compiled HTML Help (CHM),chm;chi;chq;chw;hxs,document,R,,,,R,fmt/634,,3,https://www.nongnu.org/chmspec/latest/,Chm,Y,Y,Y,PARTIAL,,content sections above 1 unsupported (ChmReader.cs) Windows Imaging (WIM/SWM/ESD),wim;swm;esd;ppkg,disk-image,RW,,,,R,,Y,3,https://learn.microsoft.com/windows-hardware/manufacture/desktop/wim-vs-ffu-image-file-formats,Wim;Swm;Esd,Y,Y,,OK,, ARJ,arj,archive,R,,R,D,R,fmt/610,RW,6,https://en.wikipedia.org/wiki/ARJ,Arj,Y,Y,Y,OK,,methods 0-4 decoded; method 4 verified on archives the open-source ARJ port wrote