diff --git a/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs new file mode 100644 index 000000000..bbbc898af --- /dev/null +++ b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs @@ -0,0 +1,37 @@ +using Compression.Registry; +using FileFormat.FirmwareHex; + +namespace Compression.Tests.FirmwareHex; + +[TestFixture] +public sealed class FirmwareHexPayloadTests { + [TestCase(false)] + [TestCase(true)] + public void Create_RejectsMultiplePayloadsWithoutWritingOutput(bool emptyFirst) { + var descriptor = new IntelHexFormatDescriptor(); + using var output = new MemoryStream(); + var inputs = new[] { + ArchiveInputInfo.InMemory("first.bin", emptyFirst ? [] : [1, 2, 3]), + ArchiveInputInfo.InMemory("second.bin", [9, 8, 7]), + }; + + Assert.Throws(() => + ((IArchiveCreatable)descriptor).Create(output, inputs, new FormatCreateOptions())); + Assert.That(output.Length, Is.Zero); + } + + [Test] + public void Add_RejectsAnotherPayloadWithoutChangingTheArchive() { + var descriptor = new IntelHexFormatDescriptor(); + using var archive = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(archive, + [ArchiveInputInfo.InMemory("firmware.bin", [1, 2, 3])], new FormatCreateOptions()); + var original = archive.ToArray(); + archive.Position = 0; + + Assert.Throws(() => ((IArchiveModifiable)descriptor).Add(archive, + [ArchiveInputInfo.InMemory("ADD.BIN", [9, 8, 7])])); + Assert.That(archive.ToArray(), Is.EqualTo(original)); + } + +} diff --git a/Compression.Tests/NowCompress/NowCompressTests.cs b/Compression.Tests/NowCompress/NowCompressTests.cs new file mode 100644 index 000000000..005a0761b --- /dev/null +++ b/Compression.Tests/NowCompress/NowCompressTests.cs @@ -0,0 +1,87 @@ +using System.Buffers.Binary; +using FileFormat.NowCompress; + +namespace Compression.Tests.NowCompress; + +/// +/// Now Compress: structural checks plus real archives. The only real archives found are commercial +/// Mac installers (the dexvert sample set), which cannot be redistributed, so those checks run on a +/// directory named by CWB_NOWCOMPRESS_SAMPLES (raw or MacBinary-wrapped). +/// +/// +/// The format stores no checksum of the data, so each decoded resource fork is checked against its +/// own header (data offset + data length = map offset, map offset + map length = fork size): a +/// mis-decoded fork does not keep that invariant. Where The Unarchiver's unar can extract a sample +/// too, every data fork is compared with its output. +/// +[TestFixture] +public class NowCompressTests { + + [Test, Category("Exception")] + public void GivenRandomBytes_WhenRead_ThenRefused() { + var bytes = new byte[512]; + new Random(3).NextBytes(bytes); + bytes[0] = 0; bytes[1] = 2; + Assert.That(NowCompressArchive.IsNowCompress(bytes), Is.False); + Assert.That(() => NowCompressArchive.Read(new MemoryStream(bytes)), Throws.InstanceOf()); + } + + [Test, Category("Exception")] + public void GivenTooShortInput_WhenRead_ThenRefused() + => Assert.That(() => NowCompressArchive.Read(new MemoryStream([0, 2, 0, 0x60])), Throws.InstanceOf()); + + private static IEnumerable<(string File, byte[] Bytes, int Offset)> Samples() { + var samples = Environment.GetEnvironmentVariable("CWB_NOWCOMPRESS_SAMPLES"); + if (string.IsNullOrEmpty(samples) || !Directory.Exists(samples)) Assert.Ignore("CWB_NOWCOMPRESS_SAMPLES is not set."); + foreach (var file in Directory.EnumerateFiles(samples!)) { + var bytes = File.ReadAllBytes(file); + var offset = bytes.Length > 140 && NowCompressArchive.IsNowCompress(bytes.AsSpan(128)) ? 128 : 0; + if (NowCompressArchive.IsNowCompress(bytes.AsSpan(offset))) yield return (file, bytes, offset); + } + } + + [Test, Category("ThemVsUs")] + public void GivenLocalSampleArchives_WhenDecoded_ThenEveryResourceForkIsSelfConsistent() { + var forks = 0; + foreach (var (file, bytes, offset) in Samples()) { + foreach (var e in NowCompressArchive.Read(new MemoryStream(bytes, offset, bytes.Length - offset)).Where(x => x.ResourceSize >= 16)) { + var r = e.Resource; + var dataOffset = BinaryPrimitives.ReadUInt32BigEndian(r); + var mapOffset = BinaryPrimitives.ReadUInt32BigEndian(r.AsSpan(4)); + var dataLength = BinaryPrimitives.ReadUInt32BigEndian(r.AsSpan(8)); + var mapLength = BinaryPrimitives.ReadUInt32BigEndian(r.AsSpan(12)); + Assert.That((long)dataOffset + dataLength, Is.EqualTo(mapOffset), $"{file}: {e.Path} resource data does not meet its map"); + Assert.That((long)mapOffset + mapLength, Is.EqualTo(r.Length), $"{file}: {e.Path} resource map does not end the fork"); + ++forks; + } + } + Assert.That(forks, Is.GreaterThan(0), "no resource fork found in the samples"); + } + + [Test, Category("ExternalInterop")] + public void GivenLocalSampleArchives_WhenUnarCanExtractThem_ThenEveryDataForkMatches() { + if (OperatingSystem.IsWindows() && !FsInteropToolbox.WslAvailable) Assert.Ignore("WSL is not available."); + if (!FsInteropToolbox.WslHasTool("unar")) Assert.Ignore("unar (The Unarchiver) is not installed."); + var compared = 0; + var tmp = Path.Combine(Path.GetTempPath(), $"cwb_now_{Guid.NewGuid():N}"); + try { + foreach (var (file, bytes, offset) in Samples()) { + Directory.CreateDirectory(tmp); + var raw = Path.Combine(tmp, Path.GetFileName(file) + ".now"); + File.WriteAllBytes(raw, bytes[offset..]); + var outDir = Path.Combine(tmp, Path.GetFileName(file) + ".out"); + var r = FsInteropToolbox.RunWsl($"unar -q -f -k hidden -D -o '{FsInteropToolbox.WinToWsl(outDir)}' '{FsInteropToolbox.WinToWsl(raw)}'"); + if (r.ExitCode != 0) continue; // unar 1.10 stops on these samples with "read more data than was available" + foreach (var e in NowCompressArchive.Read(new MemoryStream(bytes, offset, bytes.Length - offset)).Where(x => !x.IsDirectory && x.DataSize > 0)) { + var theirs = Path.Combine(outDir, e.Path.Replace('/', Path.DirectorySeparatorChar)); + if (!File.Exists(theirs)) continue; + Assert.That(e.Data, Is.EqualTo(File.ReadAllBytes(theirs)), $"{file}: {e.Path}"); + ++compared; + } + } + } finally { + try { Directory.Delete(tmp, true); } catch { /* best effort */ } + } + if (compared == 0) Assert.Inconclusive("unar extracted none of the samples, so nothing could be compared."); + } +} diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.NowCompress/NowCompressArchive.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.NowCompress/NowCompressArchive.cs new file mode 100644 index 000000000..841b3953b --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.NowCompress/NowCompressArchive.cs @@ -0,0 +1,390 @@ +using System.Buffers.Binary; +using System.Text; + +namespace FileFormat.NowCompress; + +/// One file or folder of a Now Compress archive. +public sealed class NowCompressEntry { + /// Gets the path inside the archive, '/'-separated. + public string Path { get; init; } = ""; + /// Gets whether this is a folder. + public bool IsDirectory { get; init; } + /// Gets the data fork size. + public long DataSize { get; init; } + /// Gets the resource fork size. + public long ResourceSize { get; init; } + /// Gets the modification time. + public DateTime? LastModified { get; init; } + /// Gets the decoded data fork. + public byte[] Data { get; internal set; } = []; + /// Gets the decoded resource fork. + public byte[] Resource { get; internal set; } = []; + + internal uint StreamOffset { get; init; } +} + +/// +/// Now Compress (Now Software, Macintosh, early 1990s): a directory of 110-byte records followed by +/// one stream per file, each a checksummed table of blocks; blocks are stored or compressed with a +/// 32 KiB LZSS, a static Huffman code, both, or a later Huffman-coded LZSS. Each file's stream +/// carries the resource fork, then the data fork. The format stores no checksum of the data itself, +/// so decoding errors surface only as structural failures. +/// +/// Converted from XADMaster's XADNowCompressParser / XADNowCompressHandle (LGPL-2.1-or-later). +public static class NowCompressArchive { + private const int RecordSize = 110; + private const string MacRomanHigh = + "ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñóòôöõúùûü†°¢£§•¶ß®©™´¨≠ÆØ∞±≤≥¥µ∂∑∏π∫ªºΩæø¿¡¬√ƒ≈∆«»… ÀÃÕŒœ–—“”‘’÷◊ÿŸ⁄€‹›fifl‡·‚„‰ÂÊÁËÈÍÎÏÌÓÔÒÚÛÙıˆ˜¯˘˙˚¸˝˛ˇ"; + + /// Whether the bytes start like a Now Compress archive (magic, file count, first name, header checksum). + public static bool IsNowCompress(ReadOnlySpan b) { + if (b.Length < 134 || b[0] != 0x00 || b[1] != 0x02) return false; + var count = BinaryPrimitives.ReadUInt32BigEndian(b[8..]); + if (count == 0 || count > 0xFFFF) return false; + int nameLength = b[24]; + if (nameLength is 0 or > 31) return false; + for (var i = 0; i < nameLength; ++i) if (b[25 + i] < 32) return false; + uint sum = 0; + for (var i = 24; i < 130; ++i) sum += b[i]; + return sum == BinaryPrimitives.ReadUInt32BigEndian(b[130..]); + } + + /// Parses the directory and decodes every fork. + /// Not a Now Compress archive, or a stream is damaged. + public static List Read(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + if (stream.CanSeek) stream.Position = 0; + using var ms = new MemoryStream(); + stream.CopyTo(ms); + var b = ms.ToArray(); + if (!IsNowCompress(b)) throw new InvalidDataException("Not a Now Compress archive."); + + var total = BinaryPrimitives.ReadUInt32BigEndian(b.AsSpan(8)); + var entries = new List(); + var p = 24; + var seen = 0u; + ParseDirectory(b, ref p, "", int.MaxValue, total, ref seen, entries); + + // The streams form one solid sequence: a block that starts a stream restores the history the + // previous stream's first block left behind (XADNowCompressHandle's dictionary cache). + var decoder = new BlockDecoder(b); + foreach (var e in entries.Where(x => !x.IsDirectory)) { + var bytes = decoder.DecodeFile(e.StreamOffset, e.ResourceSize + e.DataSize); + e.Resource = bytes[..(int)e.ResourceSize]; + e.Data = bytes[(int)e.ResourceSize..]; + } + return entries; + } + + private static void ParseDirectory(byte[] b, ref int p, string parent, int count, uint total, ref uint seen, List entries) { + for (var i = 0; i < count && seen < total; ++i, ++seen) { + if (p + RecordSize > b.Length) throw new InvalidDataException("Now Compress directory is truncated."); + var r = b.AsSpan(p, RecordSize); + p += RecordSize; + int nameLength = Math.Min((int)r[0], 31); + var name = MacRoman(r.Slice(1, nameLength)).Replace('/', ':'); + var flags = BinaryPrimitives.ReadUInt16BigEndian(r[36..]); + var modified = BinaryPrimitives.ReadUInt32BigEndian(r[58..]); + var children = BinaryPrimitives.ReadUInt16BigEndian(r[82..]); + var dataSize = BinaryPrimitives.ReadUInt32BigEndian(r[86..]); + var rsrcSize = BinaryPrimitives.ReadUInt32BigEndian(r[90..]); + var dataStart = BinaryPrimitives.ReadUInt32BigEndian(r[98..]); + var path = parent.Length == 0 ? name : parent + "/" + name; + var time = modified == 0 ? (DateTime?)null : new DateTime(1904, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(modified); + if ((flags & 0x10) != 0) { + entries.Add(new NowCompressEntry { Path = path, IsDirectory = true, LastModified = time }); + // As in XADMaster, the folder record itself is counted only after its children. + ParseDirectory(b, ref p, path, children, total, ref seen, entries); + } else { + entries.Add(new NowCompressEntry { Path = path, DataSize = dataSize, ResourceSize = rsrcSize, LastModified = time, StreamOffset = dataStart }); + } + } + } + + private static string MacRoman(ReadOnlySpan raw) { + var sb = new StringBuilder(raw.Length); + foreach (var c in raw) sb.Append(c < 0x80 ? (char)c : MacRomanHigh[c - 0x80]); + return sb.ToString(); + } + + // ── stream decoding ───────────────────────────────────────────────── + + private sealed class BlockDecoder(byte[] archive) { + private readonly byte[] _dictionaryCache = new byte[0x8000]; + private readonly byte[] _out = new byte[0x10000]; // 32 KiB history + 32 KiB block + + public byte[] DecodeFile(uint headerOffset, long size) { + if (size > int.MaxValue) throw new NotSupportedException("Now Compress file exceeds the 2 GiB in-memory limit."); + var result = new byte[size]; + var o = 0; + var blocks = this.ReadBlockTable(headerOffset); + foreach (var (offset, length, flags) in blocks) { + if (o >= size) break; + var produced = this.DecodeBlock(offset, length, flags); + var take = Math.Min(produced.Length, (int)(size - o)); + produced.AsSpan(0, take).CopyTo(result.AsSpan(o)); + o += take; + } + if (o != size) throw new InvalidDataException("Now Compress stream ended before the recorded size."); + return result; + } + + private static int Sum32(uint v) => (int)((v >> 24) + ((v >> 16) & 0xFF) + ((v >> 8) & 0xFF) + (v & 0xFF)); + private static int Sum16(int v) => (v >> 8) + (v & 0xFF); + + private uint U32(long at) { + if (at < 0 || at + 4 > archive.Length) throw new InvalidDataException("Now Compress stream header runs past the archive."); + return BinaryPrimitives.ReadUInt32BigEndian(archive.AsSpan((int)at)); + } + + private int U16(long at) { + if (at < 0 || at + 2 > archive.Length) throw new InvalidDataException("Now Compress stream header runs past the archive."); + return BinaryPrimitives.ReadUInt16BigEndian(archive.AsSpan((int)at)); + } + + private List<(long Offset, int Length, int Flags)> ReadBlockTable(uint headerOffset) { + var first = U32(headerOffset); + // The recorded offsets are sometimes shifted; XADMaster finds the delta from the header checksum. + if (first >= headerOffset + 20 && this.TryParse(headerOffset, first, 0, out var table)) return table; + var delta = this.FindDelta(headerOffset, first); + if (this.TryParse(headerOffset, first, delta, out table)) return table; + throw new InvalidDataException("Now Compress stream header fails its checksum."); + } + + private bool TryParse(uint headerOffset, uint first, int delta, out List<(long Offset, int Length, int Flags)> table) { + table = []; + try { + var count = (int)((first + delta - headerOffset - 4) / 8) - 1; + if (count < 0 || count > 0x100000) return false; + var p = (long)headerOffset + 4; + var last = first; + long checksum = Sum32(first); + var nextIsStart = true; + for (var i = 0; i < count; ++i) { + var flags = U16(p); + var padding = U16(p + 2); + var next = U32(p + 4); + p += 8; + checksum += Sum16(flags) + Sum16(padding) + Sum32(next); + if (next == last) { nextIsStart = true; continue; } + table.Add((last + delta, (int)(next - last - padding - 4), flags | (nextIsStart ? 0x10000 : 0))); + last = next; + nextIsStart = false; + } + for (var i = 0; i < 4; ++i) checksum += archive[p + i]; + return (uint)checksum == U32(p + 4); + } catch (Exception e) when (e is InvalidDataException or IndexOutOfRangeException or OverflowException) { + return false; + } + } + + private int FindDelta(uint headerOffset, uint first) { + var p = (long)headerOffset; + uint checksum = 0; + for (var i = 0; i < 16; ++i) checksum += archive[p++]; + for (var n = 2; n < 0x2000; ++n) { + if (U32(p) == checksum) return (int)(headerOffset + n * 8 + 4 - first); + for (var i = 0; i < 8; ++i) checksum += archive[p++]; + } + return 0; + } + + private byte[] DecodeBlock(long offset, int length, int flags) { + if (length < 0 || length > 0x8000 || offset < 0 || offset + length > archive.Length) + throw new InvalidDataException("Now Compress block is out of range."); + if ((flags & 0x10000) != 0) this._dictionaryCache.CopyTo(this._out, 0); + var input = archive.AsSpan((int)offset, length); + var outStart = 0x8000; + int produced; + if ((flags & 0x20) != 0) { + if ((flags & 0x1F) != 0) { + var huff = UnpackHuffman(input, 0x8000, 0x100); + produced = UnpackLzss(huff, this._out, outStart); + } else { + var huff = UnpackHuffman(input, 0x8000, 0x100); + huff.CopyTo(this._out, outStart); + produced = huff.Length; + } + } else if ((flags & 0x40) != 0) { + produced = UnpackNewLzss(input, this._out, outStart); + } else if ((flags & 0x1F) != 0) { + produced = UnpackLzss(input, this._out, outStart); + } else { + input.CopyTo(this._out.AsSpan(outStart)); + produced = length; + } + var block = this._out.AsSpan(outStart, produced).ToArray(); + // Slide: the last 32 KiB of history + output become the next block's history. + Buffer.BlockCopy(this._out, produced, this._out, 0, 0x8000); + if ((flags & 0x10000) != 0) Buffer.BlockCopy(this._out, 0, this._dictionaryCache, 0, 0x8000); + return block; + } + } + + private static int UnpackLzss(ReadOnlySpan src, byte[] dst, int dstStart) { + var s = 2; + var d = dstStart; + int bits = 0, numBits = 0; + while (s < src.Length) { + if (numBits == 0) { + bits = src[s++]; + numBits = 8; + if (s >= src.Length) throw new InvalidDataException("Now Compress LZSS data ends early."); + } + if ((bits & 0x80) != 0) { + if (d >= dst.Length) throw new InvalidDataException("Now Compress LZSS block overflows."); + dst[d++] = src[s++]; + } else { + int b1 = src[s++]; + if (s >= src.Length) throw new InvalidDataException("Now Compress LZSS data ends early."); + int b2 = src[s++]; + var offset = ((b1 & 0xF8) << 5) | b2; + var length = b1 & 0x07; + if (length == 0) { + if (s >= src.Length) throw new InvalidDataException("Now Compress LZSS data ends early."); + length = src[s++]; + } + length += 2; + if (d - offset < dstStart) throw new InvalidDataException("Now Compress LZSS match reaches before the block."); + for (var i = 0; i < length; ++i) { + if (d >= dst.Length) throw new InvalidDataException("Now Compress LZSS block overflows."); + dst[d] = dst[d - offset]; + ++d; + } + } + bits <<= 1; + --numBits; + } + return d - dstStart; + } + + private static readonly int[] LengthExtra = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,2,2,2,2,2,3,3,3,4,4,4,4,5,5,5,5]; + private static readonly int[] LengthBase = [0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x12,0x14,0x18,0x1c,0x20,0x24,0x28,0x30,0x38,0x40,0x50,0x60,0x70,0x80,0xa0,0xc0,0xe0]; + private static readonly int[] OffsetExtra = [0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,8,8,8,8,8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11,12,12,12,12,13,13,13,13]; + private static readonly int[] OffsetBase = [0x0000,0x0001,0x0002,0x0003,0x0004,0x0005,0x0006,0x0007,0x0008,0x000a,0x000c,0x000e,0x0010,0x0014,0x0018,0x001c,0x0020,0x0028,0x0030,0x0038,0x0040,0x0050,0x0060,0x0070,0x0080,0x00a0,0x00c0,0x00e0,0x0100,0x0200,0x0300,0x0400,0x0500,0x0600,0x0700,0x0800,0x0900,0x0b00,0x0d00,0x0f00,0x1100,0x1500,0x1900,0x1d00,0x2100,0x2900,0x3100,0x3900,0x4100,0x5100,0x6100,0x7100,0x8100,0xa100,0xc100,0xe100]; + + private static int UnpackNewLzss(ReadOnlySpan src, byte[] dst, int dstStart) { + if (src.Length < 4) throw new InvalidDataException("Now Compress block is too short."); + var headerSize = BinaryPrimitives.ReadUInt16BigEndian(src) - 0x2F59; + int endBits = src[3]; + var s = 4; + if (headerSize < 0 || s + headerSize > src.Length) throw new InvalidDataException("Now Compress block header is out of range."); + var header = UnpackHuffman(src.Slice(s, headerSize), 0x15A, 20); + if (header.Length != 0x15A) throw new InvalidDataException("Now Compress block code table is incomplete."); + s += headerSize; + if ((s & 1) != 0) ++s; + var mainLengths = new int[0x122]; + for (var i = 0; i < 0x122; ++i) mainLengths[i] = header[i]; + var offsetLengths = new int[0x38]; + for (var i = 0; i < 0x38; ++i) offsetLengths[i] = header[i + 0x122] != 0 ? header[i + 0x122] - 2 : 0; + var main = new Canonical(mainLengths, 20); + var offs = new Canonical(offsetLengths, 20); + var reader = new MsbReader(src[s..]); + long numBits = (long)(src.Length - s) * 8; + if (endBits != 0) numBits -= 16 - endBits; + var d = dstStart; + while (reader.Position < numBits) { + var symbol = main.Decode(ref reader); + if (symbol < 0x100) { + if (d >= dst.Length) throw new InvalidDataException("Now Compress block overflows."); + dst[d++] = (byte)symbol; + continue; + } + var ls = symbol - 0x100; + var length = LengthBase[ls] + (LengthExtra[ls] != 0 ? reader.Bits(LengthExtra[ls]) : 0) + 2; + var os = offs.Decode(ref reader); + var offset = OffsetBase[os] + (OffsetExtra[os] != 0 ? reader.Bits(OffsetExtra[os]) : 0); + if (d + length > dst.Length || d - offset < 0) throw new InvalidDataException("Now Compress match is out of range."); + for (var i = 0; i < length; ++i) { dst[d] = dst[d - offset]; ++d; } + } + return d - dstStart; + } + + private static byte[] UnpackHuffman(ReadOnlySpan src, int maxOutput, int numValues) { + var s = 0; + if (s >= src.Length) throw new InvalidDataException("Now Compress Huffman block is empty."); + int endBits = src[s++]; + var lengths = new int[numValues]; + for (var i = 0; i < numValues / 2; ++i) { + if (s >= src.Length) throw new InvalidDataException("Now Compress Huffman table is truncated."); + var v = src[s++]; + lengths[2 * i] = v >> 4; + lengths[2 * i + 1] = v & 0x0F; + } + if (s >= src.Length) throw new InvalidDataException("Now Compress Huffman table is truncated."); + int extra = src[s++]; + for (var i = 0; i < extra; ++i) { + if (s >= src.Length) throw new InvalidDataException("Now Compress Huffman table is truncated."); + var index = src[s++]; + if (index >= numValues) throw new InvalidDataException("Now Compress Huffman table index is out of range."); + lengths[index] += 16; + } + var code = new Canonical(lengths, 31); + if ((s & 1) != 0) ++s; + var reader = new MsbReader(src[Math.Min(s, src.Length)..]); + long numBits = (long)(src.Length - Math.Min(s, src.Length)) * 8; + if (endBits != 0) numBits -= 16 - endBits; + var output = new List(Math.Min(maxOutput, 0x8000)); + while (reader.Position < numBits) { + if (output.Count >= maxOutput) throw new InvalidDataException("Now Compress Huffman block overflows."); + output.Add((byte)code.Decode(ref reader)); + } + if (reader.Position != numBits) throw new InvalidDataException("Now Compress Huffman block has trailing bits."); + return [.. output]; + } + + private ref struct MsbReader(ReadOnlySpan data) { + private readonly ReadOnlySpan _data = data; + public long Position { get; private set; } + + public int Bit() { + var i = (int)(this.Position >> 3); + if (i >= this._data.Length) throw new InvalidDataException("Now Compress bit stream ended early."); + var bit = (this._data[i] >> (7 - (int)(this.Position & 7))) & 1; + ++this.Position; + return bit; + } + + public int Bits(int n) { + var v = 0; + for (var i = 0; i < n; ++i) v = (v << 1) | this.Bit(); + return v; + } + } + + /// Canonical prefix code, shortest code all zeros, decoded MSB first. + private sealed class Canonical { + private readonly int[] _count; + private readonly int[] _symbols; + private readonly int _maxLen; + + public Canonical(int[] lengths, int maxLen) { + this._maxLen = maxLen; + this._count = new int[maxLen + 1]; + var n = 0; + foreach (var l in lengths) { + if (l > maxLen) throw new InvalidDataException("Now Compress code length is too long."); + if (l > 0) { ++this._count[l]; ++n; } + } + this._symbols = new int[n]; + var offs = new int[maxLen + 2]; + for (var len = 1; len <= maxLen; ++len) offs[len + 1] = offs[len] + this._count[len]; + for (var sym = 0; sym < lengths.Length; ++sym) + if (lengths[sym] > 0) this._symbols[offs[lengths[sym]]++] = sym; + } + + public int Decode(ref MsbReader r) { + int code = 0, first = 0, index = 0; + for (var len = 1; len <= this._maxLen; ++len) { + code |= r.Bit(); + var count = this._count[len]; + if (code - first < count) return this._symbols[index + code - first]; + index += count; + first += count; + first <<= 1; + code <<= 1; + } + throw new InvalidDataException("Now Compress stream uses an unassigned code."); + } + } +} diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.NowCompress/NowCompressFormatDescriptor.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.NowCompress/NowCompressFormatDescriptor.cs new file mode 100644 index 000000000..7e36e9b52 --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.NowCompress/NowCompressFormatDescriptor.cs @@ -0,0 +1,70 @@ +using Compression.Registry; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.NowCompress; + +/// Now Compress archive (Now Software, Macintosh). Read-only. +public sealed class NowCompressFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { + /// Suffix under which a file's resource fork is listed and extracted. + public const string ResourceForkSuffix = ".rsrc"; + + /// + public string Id => "NowCompress"; + /// + public string DisplayName => "Now Compress"; + /// + public FormatCategory Category => FormatCategory.Archive; + /// + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | + FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; + /// + public string DefaultExtension => ".now"; + // .sea is shared with every Mac self-extractor; the archive is recognised by content. + /// + public IReadOnlyList Extensions => [".now"]; + /// + public IReadOnlyList CompoundExtensions => []; + /// + public IReadOnlyList MagicSignatures => [ + new([0x00, 0x02, 0x00, 0x60, 0x00, 0x01], Confidence: 0.6, Mask: [0xFF, 0xFF, 0x00, 0xFF, 0xFF, 0xFF]), + ]; + /// + public IReadOnlyList Methods => [new("now", "Now Compress")]; + /// + public string? TarCompressionFormatId => null; + /// + public AlgorithmFamily Family => AlgorithmFamily.Archive; + /// + public string Description => "Now Compress Macintosh archive (LZSS / Huffman blocks); read-only"; + + private static IEnumerable<(NowCompressEntry Entry, bool Resource, string Name)> Items(List entries) { + foreach (var e in entries) { + if (e.IsDirectory) { yield return (e, false, e.Path); continue; } + if (e.DataSize > 0 || e.ResourceSize == 0) yield return (e, false, e.Path); + if (e.ResourceSize > 0) yield return (e, true, e.Path + ResourceForkSuffix); + } + } + + /// + public List List(Stream stream, string? password) + => Items(NowCompressArchive.Read(stream)).Select((x, i) => new ArchiveEntryInfo(i, x.Name, + x.Entry.IsDirectory ? 0 : x.Resource ? x.Entry.ResourceSize : x.Entry.DataSize, -1, + x.Entry.IsDirectory ? "" : "Now Compress", x.Entry.IsDirectory, false, x.Entry.LastModified)).ToList(); + + /// + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + foreach (var (e, resource, name) in Items(NowCompressArchive.Read(stream))) { + if (e.IsDirectory || files is { Length: > 0 } && !MatchesFilter(name, files)) continue; + WriteFile(outputDir, name, resource ? e.Resource : e.Data); + } + } + + /// + public byte[] ExtractEntryToMemory(Stream archive, string entryName, string? password) { + foreach (var (e, resource, name) in Items(NowCompressArchive.Read(archive))) + if (!e.IsDirectory && string.Equals(name, entryName, StringComparison.OrdinalIgnoreCase)) + return resource ? e.Resource : e.Data; + return []; + } +} diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.NowCompress/THIRD-PARTY-NOTICE.XADMaster.txt b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.NowCompress/THIRD-PARTY-NOTICE.XADMaster.txt new file mode 100644 index 000000000..20b0d9c06 --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.NowCompress/THIRD-PARTY-NOTICE.XADMaster.txt @@ -0,0 +1,14 @@ +Now Compress support in this folder (NowCompressArchive) is converted from XADMaster, the archive +library of The Unarchiver. + +Upstream: https://github.com/MacPaw/XADMaster +Revision: 7cb9ee0abbb163f261e4cb74501e15067032319c +Files: XADNowCompressParser.m, XADNowCompressHandle.m +Licence: GNU Lesser General Public License, version 2.1 or (at your option) any later version. + This repository is distributed under LGPL-3.0-or-later, which the "any later version" + grant permits. + +Copyright (c) 2017-present, MacPaw Way Ltd. and the XADMaster contributors. + +The length/offset base and extra-bit tables of the "new LZSS" blocks are copied verbatim; all other +code was rewritten in C# for this codebase. diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md index 9357f828e..093a1a0d2 100644 --- a/Hawkynt.FileFormats.Archives/README.md +++ b/Hawkynt.FileFormats.Archives/README.md @@ -131,6 +131,7 @@ Every State, Test, Maintenance, Compress/Decompress and Demux/Mux/Remux cell is | [LHA / LZH](https://en.wikipedia.org/wiki/LHA_(file_format)) | `Lzh` | `.lzh` `.lha` | R/W | ✅ | defrag · wipe | -lh2-, -lh3-, -lhx-, -pm1- and -pm2- are read-only | [GitHub](https://github.com/jca02266/lha) | | [LZX (Amiga)](https://en.wikipedia.org/wiki/LZX) | `LzxAmiga` | `.lzx` | R/W | ✅ | defrag · wipe | | [Aminet](https://aminet.net) | | [mtree](https://man.freebsd.org/cgi/man.cgi?query=mtree&sektion=5) | `Mtree` | `.mtree` | WORM | ✅ | — | Filesystem metadata manifest; does not embed file bodies, so CWB does not dereference `contents=` host paths | [FreeBSD mtree(5)](https://man.freebsd.org/cgi/man.cgi?query=mtree&sektion=5) | +| Now Compress | `NowCompress` | `.now` | R | ✅ | — | Read-only; folders, resource forks as `.rsrc`; all four block codings | [GitHub](https://github.com/MacPaw/XADMaster) | | [NuFX / ShrinkIt](https://en.wikipedia.org/wiki/ShrinkIt) | `NuFx` | `.shk` `.sdk` `.bxy` | R/W | ✅ | defrag · shrink · wipe | | [nulib.com](https://nulib.com/library/FTN.e08002.htm) | | PackIt | `PackIt` | `.pit` | R/W | ✅ | defrag · wipe | Reads stored and Huffman entries, writes stored; encrypted entries are refused | [GitHub](https://github.com/MacPaw/XADMaster) | | [RAR](https://en.wikipedia.org/wiki/RAR_(file_format)) | `Rar` | `.rar` | R/W | ✅ | wipe | v1–v5 readers; creation and edits emit RAR4/RAR5 without claiming WinRAR encoder parity | [rarlab.com](https://www.rarlab.com/technote.htm) | @@ -544,7 +545,7 @@ Use it when a .NET process needs to enumerate, extract, test, create, edit or in -Every public and protected member of all 2571 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). +Every public and protected member of all 2574 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). diff --git a/Hawkynt.FileFormats.Archives/REFERENCE.md b/Hawkynt.FileFormats.Archives/REFERENCE.md index 464eb0ba6..151685607 100644 --- a/Hawkynt.FileFormats.Archives/REFERENCE.md +++ b/Hawkynt.FileFormats.Archives/REFERENCE.md @@ -22820,6 +22820,60 @@ The NIfTI header revision the file declares. | `Nifti1` | `1` | | | `Nifti2` | `2` | | +### Namespace `FileFormat.NowCompress` + +[`NowCompressArchive`](#nowcompressarchive) · [`NowCompressEntry`](#nowcompressentry) · [`NowCompressFormatDescriptor`](#nowcompressformatdescriptor) + +#### `NowCompressArchive` + +Now Compress (Now Software, Macintosh, early 1990s): a directory of 110-byte records followed by one stream per file, each a checksummed table of blocks; blocks are stored or compressed with a 32 KiB LZSS, a static Huffman code, both, or a later Huffman-coded LZSS. Each file's stream carries the resource fork, then the data fork. The format stores no checksum of the data itself, so decoding errors surface only as structural failures. + +| Member | Signature | Summary | +| --- | --- | --- | +| `IsNowCompress` | `static bool IsNowCompress(ReadOnlySpan b)` | Whether the bytes start like a Now Compress archive (magic, file count, first name, header checksum). | +| `Read` | `static List Read(Stream stream)` | Parses the directory and decodes every fork. | + +#### `NowCompressEntry` + +One file or folder of a Now Compress archive. + +| Member | Signature | Summary | +| --- | --- | --- | +| `NowCompressEntry` | `NowCompressEntry()` | | +| `DataSize` | `long DataSize { get; init; }` | Gets the data fork size. | +| `Data` | `byte[] Data { get; }` | Gets the decoded data fork. | +| `IsDirectory` | `bool IsDirectory { get; init; }` | Gets whether this is a folder. | +| `LastModified` | `DateTime? LastModified { get; init; }` | Gets the modification time. | +| `Path` | `string Path { get; init; }` | Gets the path inside the archive, '/'-separated. | +| `ResourceSize` | `long ResourceSize { get; init; }` | Gets the resource fork size. | +| `Resource` | `byte[] Resource { get; }` | Gets the decoded resource fork. | + +#### `NowCompressFormatDescriptor` + +Now Compress archive (Now Software, Macintosh). Read-only. + +Implements `IArchiveFormatOperations`, `IFormatDescriptor`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `NowCompressFormatDescriptor` | `NowCompressFormatDescriptor()` | | +| `ResourceForkSuffix` | `const string ResourceForkSuffix` | Suffix under which a file's resource fork is listed and extracted. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | + ### Namespace `FileFormat.Nrbf` [`NrbfFormatDescriptor`](#nrbfformatdescriptor) diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs index 6c7d4b452..1c9e17266 100644 --- a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs @@ -33,6 +33,7 @@ public static class FirmwareHexWriter { /// one the reader rendered. An input list with no payload describes an image /// with no data, which both formats can write. /// + /// More than one non-metadata payload is supplied. public static FirmwareImage ImageFrom(IReadOnlyList inputs, string sourceFormat) { ArgumentNullException.ThrowIfNull(inputs); @@ -53,7 +54,9 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st } // Any other single input is the payload, whatever it is called: a caller // converting a .bin into a .hex should not have to rename it first. - payload ??= input.ReadContent(); + if (payload is not null) + throw new InvalidDataException($"{sourceFormat} represents one logical firmware payload; multiple payload inputs are ambiguous."); + payload = input.ReadContent(); } List<(uint Address, byte[] Data)> segments; diff --git a/docs/LICENCE-PROVENANCE.md b/docs/LICENCE-PROVENANCE.md index ca50e4c15..3a8da34c3 100644 --- a/docs/LICENCE-PROVENANCE.md +++ b/docs/LICENCE-PROVENANCE.md @@ -31,7 +31,7 @@ package carries; everything else keeps a `THIRD-PARTY-NOTICE..txt` beside | Class | Upstream rows | Notes | |---|---|---| -| A | 18 | | +| A | 19 | | B | 17 | FFmpeg alone covers 38 codec folders | | C | 3 | specifications: `docs/CVF-CODECS-ON-DISK.md` (rewrite already under way), `docs/IT-SAMPLE-COMPRESSION-SPEC.md`, `docs/FDS-EXPANSION-SOUND-SPEC.md` | | R | 8 | | @@ -70,6 +70,7 @@ package carries; everything else keeps a `THIRD-PARTY-NOTICE..txt` beside | Claim site | Upstream | Licence | Where the notice is | |---|---|---|---| +| `Hawkynt.FileFormats.Archives/FileFormats/FileFormat.NowCompress/NowCompressArchive.cs` | XADMaster `XADNowCompressParser.m`, `XADNowCompressHandle.m`, revision `7cb9ee0abbb163f261e4cb74501e15067032319c` | LGPL-2.1-or-later | `Hawkynt.FileFormats.Archives/FileFormats/FileFormat.NowCompress/THIRD-PARTY-NOTICE.XADMaster.txt` | | `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImage.cs`, `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImageMetadata.cs` | AOSP mkbootimg `bootimg.h`, `mkbootimg.py` and `unpack_bootimg.py`, revision `d2bb0af5ba6d3198a3e99529c97eda1be0b5a093` | BSD-3-Clause and Apache-2.0 | `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/THIRD-PARTY-NOTICE.mkbootimg.txt` | | `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbVbmeta.cs`, `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbDescriptorText.cs` | AOSP Android Verified Boot headers and `avbtool.py`, revision `c5066a96caa7bf4150c0a8cc8cc14ab81733fdc7` | MIT | `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/THIRD-PARTY-NOTICE.avb.txt` | | `Compression.Core/Dictionary/Lzbitmap/` | libzbitmap (Ernesto A. Fernández, Corellium) | MIT | `THIRD-PARTY-NOTICE.libzbitmap.txt` | diff --git a/docs/catalogue/our-registry.csv b/docs/catalogue/our-registry.csv index bd9da9071..460a8aced 100644 --- a/docs/catalogue/our-registry.csv +++ b/docs/catalogue/our-registry.csv @@ -350,6 +350,7 @@ format,Nifti,NIfTI,Archive,,.nii;.nii.gz,Y,Y,, format,Nilfs1,NILFS v1,Archive,Y,.nilfs1;.nilfs,Y,Y,Y,Y format,Nilfs2,NILFS2,Archive,Y,.nilfs2;.nilfs,Y,Y,Y,Y format,Npy,NumPy NPY,Archive,,.npy,Y,Y,Y, +format,NowCompress,Now Compress,Archive,,.now,Y,Y,, format,Npz,NumPy NPZ,Archive,,.npz,Y,Y,Y, format,Nrbf,.NET BinaryFormatter / NRBF object graph,Archive,,.nrbf,Y,Y,Y, format,Nrg,NRG,Archive,,.nrg,Y,Y,Y,Y diff --git a/docs/catalogue/reference-formats.csv b/docs/catalogue/reference-formats.csv index d168a2e19..55dcba043 100644 --- a/docs/catalogue/reference-formats.csv +++ b/docs/catalogue/reference-formats.csv @@ -48,7 +48,7 @@ StuffIt X,sitx,archive,,,R,,R,fmt/399,RW,4,https://en.wikipedia.org/wiki/StuffIt DiskDoubler,dd,archive,,,R,,,,Y,2,https://en.wikipedia.org/wiki/DiskDoubler,DiskDoubler,Y,Y,,PARTIAL,,"single files only (no DDAR/DDA2 archives); methods 0, 1, 6, 9 decoded and checked against unar on real files; 2-5, 7, 8, 10 and delta refused (DiskDoublerReader.cs)" Compact Pro,cpt,archive,,,R,,,,Y,2,https://en.wikipedia.org/wiki/Compact_Pro,CompactPro,Y,Y,Y,PARTIAL,,stored and RLE only; LZ+RLE and LZ+Huffman unsupported (CompactProReader.cs) PackIt,pit,archive,,,R,,,,Y,2,https://en.wikipedia.org/wiki/PackIt,PackIt,Y,Y,Y,PARTIAL,,"real layout (PMag/PMa4, header and fork CRC-16) verified on real archives against unar; encrypted PMa5/PMa6 entries refused (PackItReader.cs)" -Now Compress,now,archive,,,R,,,,,1,,,,,,MISSING,, +Now Compress,now,archive,,,R,,,,,1,https://github.com/MacPaw/XADMaster,NowCompress,Y,,,OK,,read-only: no writer exists to verify against LZX (Amiga),lzx,archive,,,R,,R,,Y,3,https://en.wikipedia.org/wiki/LZX,LzxAmiga,Y,Y,Y,OK,, NuFX / ShrinkIt,shk;sdk;bxy,archive,,,,,,,Y,1,https://wiki.preterhuman.net/NuFX,NuFx,Y,Y,Y,OK,, Spark (Acorn),spk;arc,archive,,,,,,,Y,1,https://en.wikipedia.org/wiki/Spark_(archiver),Spark,Y,Y,Y,OK,, @@ -189,7 +189,7 @@ DiskDoubler methods (ADn/DDn/Method2),,codec,,,R,,,,,1,,DiskDoubler,Y,Y,,PARTIAL Compact Pro LZH/RLE,,codec,,,R,,,,,1,,,,,,MISSING,, Squeeze (Huffman),,codec,,,R,,,,,1,,Squeeze,Y,Y,,OK,, Crunch CP/M (LZW),,codec,,,R,,,,,1,,Crunch,Y,Y,,OK,, -Now Compress method,,codec,,,R,,,,,1,,,,,,MISSING,, +Now Compress method,,codec,,,R,,,,,1,,NowCompress,Y,,,OK,, WinZip JPEG (ZIP method 96),,codec,,,R,,,,,1,,,,,,MISSING,, WinZip WavPack (ZIP method 97),,codec,,,R,,,,,1,,,,,,MISSING,, tar sparse files,,codec,,,R,,,,,1,,Tar,Y,Y,Y,PARTIAL,,GNU/PAX sparse members are not hole-reconstructed (TarReader.cs)